@serfab/cadre-core 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/README.md +8 -6
  2. package/dist/cadre-node.d.ts +386 -114
  3. package/dist/cadre-node.js +887 -359
  4. package/dist/cadre-node.js.map +1 -1
  5. package/dist/control-authorization.d.ts +2 -2
  6. package/dist/control-authorization.js +1 -0
  7. package/dist/control-authorization.js.map +1 -1
  8. package/dist/control-cohort.d.ts +6 -0
  9. package/dist/control-cohort.js +6 -0
  10. package/dist/control-cohort.js.map +1 -1
  11. package/dist/control-database.d.ts +46 -8
  12. package/dist/control-database.js +97 -18
  13. package/dist/control-database.js.map +1 -1
  14. package/dist/control-formation-recorder.d.ts +37 -14
  15. package/dist/control-formation-recorder.js +60 -10
  16. package/dist/control-formation-recorder.js.map +1 -1
  17. package/dist/control-read-retry.d.ts +13 -12
  18. package/dist/control-read-retry.js +14 -12
  19. package/dist/control-read-retry.js.map +1 -1
  20. package/dist/control-schema.d.ts +1 -1
  21. package/dist/control-schema.js +67 -5
  22. package/dist/control-schema.js.map +1 -1
  23. package/dist/control-stream.d.ts +9 -1
  24. package/dist/control-stream.js +24 -1
  25. package/dist/control-stream.js.map +1 -1
  26. package/dist/control-write-retry.d.ts +12 -2
  27. package/dist/control-write-retry.js +13 -2
  28. package/dist/control-write-retry.js.map +1 -1
  29. package/dist/delegate-admission.d.ts +11 -8
  30. package/dist/delegate-admission.js +19 -11
  31. package/dist/delegate-admission.js.map +1 -1
  32. package/dist/diagnostics/connection-path.d.ts +0 -7
  33. package/dist/diagnostics/connection-path.js +0 -7
  34. package/dist/diagnostics/connection-path.js.map +1 -1
  35. package/dist/formation-approval.d.ts +7 -0
  36. package/dist/formation-approval.js +9 -3
  37. package/dist/formation-approval.js.map +1 -1
  38. package/dist/fs-atomic.d.ts +11 -2
  39. package/dist/fs-atomic.js +13 -3
  40. package/dist/fs-atomic.js.map +1 -1
  41. package/dist/hibernation-manager.d.ts +111 -8
  42. package/dist/hibernation-manager.js +234 -49
  43. package/dist/hibernation-manager.js.map +1 -1
  44. package/dist/index.d.ts +8 -6
  45. package/dist/index.js +15 -9
  46. package/dist/index.js.map +1 -1
  47. package/dist/joined-strand-store.d.ts +90 -25
  48. package/dist/joined-strand-store.js +223 -55
  49. package/dist/joined-strand-store.js.map +1 -1
  50. package/dist/key-store-file.js +14 -5
  51. package/dist/key-store-file.js.map +1 -1
  52. package/dist/link-budget.d.ts +367 -69
  53. package/dist/link-budget.js +375 -72
  54. package/dist/link-budget.js.map +1 -1
  55. package/dist/membership-connection-gater.d.ts +30 -43
  56. package/dist/membership-connection-gater.js +31 -42
  57. package/dist/membership-connection-gater.js.map +1 -1
  58. package/dist/peer-dial.d.ts +24 -4
  59. package/dist/peer-dial.js +74 -5
  60. package/dist/peer-dial.js.map +1 -1
  61. package/dist/peer-join-backfill.d.ts +2 -1
  62. package/dist/peer-join-backfill.js +6 -5
  63. package/dist/peer-join-backfill.js.map +1 -1
  64. package/dist/push-fanout.d.ts +6 -0
  65. package/dist/push-fanout.js +6 -0
  66. package/dist/push-fanout.js.map +1 -1
  67. package/dist/relay-reservation.d.ts +3 -1
  68. package/dist/relay-reservation.js +3 -1
  69. package/dist/relay-reservation.js.map +1 -1
  70. package/dist/relay-server.d.ts +79 -0
  71. package/dist/relay-server.js +74 -0
  72. package/dist/relay-server.js.map +1 -0
  73. package/dist/seed-bootstrap.d.ts +57 -4
  74. package/dist/seed-bootstrap.js +107 -83
  75. package/dist/seed-bootstrap.js.map +1 -1
  76. package/dist/storage-scope.d.ts +33 -17
  77. package/dist/storage-scope.js +44 -23
  78. package/dist/storage-scope.js.map +1 -1
  79. package/dist/strand-addr-protocol.d.ts +58 -21
  80. package/dist/strand-addr-protocol.js +133 -85
  81. package/dist/strand-addr-protocol.js.map +1 -1
  82. package/dist/strand-database.d.ts +20 -2
  83. package/dist/strand-database.js +42 -5
  84. package/dist/strand-database.js.map +1 -1
  85. package/dist/strand-first-sync-gate.d.ts +59 -21
  86. package/dist/strand-first-sync-gate.js +58 -19
  87. package/dist/strand-first-sync-gate.js.map +1 -1
  88. package/dist/strand-formation-deadlines.d.ts +151 -0
  89. package/dist/strand-formation-deadlines.js +171 -0
  90. package/dist/strand-formation-deadlines.js.map +1 -0
  91. package/dist/strand-formation-manager.d.ts +63 -30
  92. package/dist/strand-formation-manager.js +67 -41
  93. package/dist/strand-formation-manager.js.map +1 -1
  94. package/dist/strand-formation-protocol.d.ts +45 -35
  95. package/dist/strand-formation-protocol.js +59 -122
  96. package/dist/strand-formation-protocol.js.map +1 -1
  97. package/dist/strand-instance-manager.d.ts +113 -6
  98. package/dist/strand-instance-manager.js +257 -41
  99. package/dist/strand-instance-manager.js.map +1 -1
  100. package/dist/strand-membership-writer.d.ts +6 -0
  101. package/dist/strand-membership-writer.js +11 -0
  102. package/dist/strand-membership-writer.js.map +1 -1
  103. package/dist/strand-peer-book-protocol.d.ts +13 -4
  104. package/dist/strand-peer-book-protocol.js +3 -7
  105. package/dist/strand-peer-book-protocol.js.map +1 -1
  106. package/dist/strand-peer-book.d.ts +1 -1
  107. package/dist/strand-peer-book.js +8 -1
  108. package/dist/strand-peer-book.js.map +1 -1
  109. package/dist/strand-peer-observer.js +1 -1
  110. package/dist/strand-revocation-enforcer.js +2 -1
  111. package/dist/strand-revocation-enforcer.js.map +1 -1
  112. package/dist/strand-solicitation.d.ts +36 -20
  113. package/dist/strand-solicitation.js +4 -4
  114. package/dist/strand-solicitation.js.map +1 -1
  115. package/dist/strand-wake-protocol.d.ts +80 -23
  116. package/dist/strand-wake-protocol.js +89 -64
  117. package/dist/strand-wake-protocol.js.map +1 -1
  118. package/dist/types.d.ts +128 -38
  119. package/dist/types.js +4 -3
  120. package/dist/types.js.map +1 -1
  121. package/package.json +7 -7
  122. package/src/cadre-node.ts +8367 -7776
  123. package/src/control-authorization.ts +2 -1
  124. package/src/control-cohort.ts +6 -0
  125. package/src/control-database.ts +122 -18
  126. package/src/control-formation-recorder.ts +354 -305
  127. package/src/control-read-retry.ts +14 -12
  128. package/src/control-schema.ts +67 -5
  129. package/src/control-stream.ts +25 -1
  130. package/src/control-write-retry.ts +13 -2
  131. package/src/delegate-admission.ts +23 -13
  132. package/src/diagnostics/connection-path.ts +0 -7
  133. package/src/formation-approval.ts +610 -604
  134. package/src/fs-atomic.ts +14 -4
  135. package/src/hibernation-manager.ts +638 -437
  136. package/src/index.ts +28 -10
  137. package/src/joined-strand-store.ts +247 -57
  138. package/src/key-store-file.ts +14 -5
  139. package/src/link-budget.ts +393 -74
  140. package/src/membership-connection-gater.ts +31 -45
  141. package/src/peer-dial.ts +81 -7
  142. package/src/peer-join-backfill.ts +8 -6
  143. package/src/push-fanout.ts +6 -0
  144. package/src/relay-reservation.ts +3 -1
  145. package/src/relay-server.ts +98 -0
  146. package/src/seed-bootstrap.ts +133 -91
  147. package/src/storage-scope.ts +44 -23
  148. package/src/strand-addr-protocol.ts +547 -458
  149. package/src/strand-database.ts +287 -242
  150. package/src/strand-first-sync-gate.ts +58 -19
  151. package/src/strand-formation-deadlines.ts +227 -0
  152. package/src/strand-formation-manager.ts +700 -647
  153. package/src/strand-formation-protocol.ts +103 -150
  154. package/src/strand-instance-manager.ts +287 -44
  155. package/src/strand-membership-writer.ts +12 -0
  156. package/src/strand-peer-book-protocol.ts +16 -13
  157. package/src/strand-peer-book.ts +8 -1
  158. package/src/strand-peer-observer.ts +1 -1
  159. package/src/strand-revocation-enforcer.ts +2 -1
  160. package/src/strand-solicitation.ts +519 -500
  161. package/src/strand-wake-protocol.ts +401 -355
  162. package/src/types.ts +130 -38
@@ -34,8 +34,15 @@ export declare function liveStrandStatus(instance: StrandInstance): 'active' | '
34
34
  */
35
35
  export interface StartStrandConfig {
36
36
  strandRow: StrandRow;
37
- /** sApp configuration provided by the hosting application */
38
- sAppConfig: SAppConfig;
37
+ /**
38
+ * sApp configuration provided by the hosting application. Absent for a **storage
39
+ * replica** — a strand this node stores and serves without the app installed
40
+ * (`CadreNodeConfig.hostUnclaimedStrands`): no schema signature check, no `App`
41
+ * schema, no `sAppInfo` on the instance, and the default latency hint. A replica is
42
+ * always a joiner; `StrandDatabase` refuses to found one. An app that later claims the
43
+ * strand upgrades the running replica in place ({@link StrandInstanceManager.attachSApp}).
44
+ */
45
+ sAppConfig?: SAppConfig;
39
46
  storage?: StorageConfig;
40
47
  network?: NetworkConfig;
41
48
  profile: NodeProfile;
@@ -364,6 +371,23 @@ export declare class StrandInstanceManager {
364
371
  * wake becomes the better trade.
365
372
  */
366
373
  private strandStorages;
374
+ /**
375
+ * The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
376
+ * `resumeStrand`), present until it settles. The instance is tracked for the whole build,
377
+ * with its database not yet constructed or not yet initialized, so a caller that must act
378
+ * on the FINISHED runtime ({@link attachSApp}, {@link whenRuntimeBuilt}) waits on this instead. A failed build's
379
+ * rejection belongs to the call that started it.
380
+ */
381
+ private runtimeBuilds;
382
+ /**
383
+ * The latest resume or quiesce requested per strand id, present until it settles. Each one
384
+ * starts only once the one requested before it has settled, so a wake that lands during a
385
+ * quiesce rebuilds after it, and a quiesce requested during a rebuild releases what that
386
+ * rebuild finished. The WHOLE operation, not its entry in {@link runtimeBuilds}: that settles
387
+ * before a resume's own catch records `'error'`, so a joiner awaiting it could read a stale
388
+ * status.
389
+ */
390
+ private transitions;
367
391
  private stopping;
368
392
  constructor();
369
393
  /**
@@ -395,6 +419,18 @@ export declare class StrandInstanceManager {
395
419
  * hibernation wake never re-enters the embedder's provider.
396
420
  */
397
421
  startStrand(config: StartStrandConfig): Promise<StrandInstance>;
422
+ /** Hold `build` in {@link runtimeBuilds} until it settles, and settle as it does. */
423
+ private trackRuntimeBuild;
424
+ /**
425
+ * Wait until no runtime build of `strandId` is in flight. Never rejects: a failed build
426
+ * is reported by the call that started it, and the caller re-reads the instance after.
427
+ *
428
+ * NOTE: after a failed launch the re-read sees the instance gone only because
429
+ * `startStrand`'s catch deletes it synchronously, and its continuation is queued before
430
+ * this waiter's. If that cleanup ever awaits before `instances.delete`, a waiter can act
431
+ * on the dying record; move the cleanup inside the tracked promise then.
432
+ */
433
+ private settleRuntimeBuilds;
398
434
  /**
399
435
  * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
400
436
  * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
@@ -509,11 +545,26 @@ export declare class StrandInstanceManager {
509
545
  * Quiesce a strand: release its strand-network resources (stop the libp2p node,
510
546
  * close the StrandDatabase) while RETAINING the instance record — identity,
511
547
  * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
512
- * resumed later. Mechanically this is `stopStrand` minus the instance/config
513
- * deletion. The caller sets the post-quiesce status (e.g. `hibernating`).
514
- * No-ops when the strand is missing or already quiesced.
548
+ * resumed later, then mark it `hibernating`. Mechanically this is `stopStrand` minus
549
+ * the instance/config deletion. A tracked strand with no runtime left (a failed
550
+ * rebuild already released it) is only marked; an untracked one is left alone.
551
+ *
552
+ * Ordered with {@link resumeStrand} (see {@link transitions}): it starts once the resume
553
+ * or launch build before it has settled, so it releases a finished runtime, and the
554
+ * status write is its last step, so a resume requested meanwhile writes after it. A call
555
+ * made while another quiesce is the latest request joins it. A failed release leaves the
556
+ * status as it was and rejects.
515
557
  */
516
558
  quiesceStrand(strandId: string): Promise<void>;
559
+ /** Body of {@link quiesceStrand}. */
560
+ private runQuiesce;
561
+ /**
562
+ * Whether the latest runtime transition requested for `strandId` is a quiesce that has
563
+ * not finished. Its handles may still be attached and its status still live, but it is on
564
+ * its way to `hibernating`, so a caller that needs it up must {@link resumeStrand} it — the
565
+ * resume runs after the quiesce — rather than treat it as live.
566
+ */
567
+ isQuiescing(strandId: string): boolean;
517
568
  /**
518
569
  * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
519
570
  * from the retained launch config and re-attach them, transitioning it back to
@@ -521,8 +572,22 @@ export declare class StrandInstanceManager {
521
572
  * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
522
573
  * count) and updates the retained config so a later resume reuses the latest
523
574
  * values. Returns the live instance unchanged if it is already running.
575
+ *
576
+ * Overlapping calls share one rebuild: a call made while a resume is the latest request
577
+ * for the strand — a wake landing during a check-in — joins it and settles exactly as it
578
+ * does, `'error'` status included. The joiner's own `overrides` are ignored; the first
579
+ * resume's seed wins, and both callers resolved the cohort seed moments apart. A call made
580
+ * while a quiesce is the latest request runs after that quiesce and rebuilds (see
581
+ * {@link transitions}), and one made while `startStrand` is still building waits for that
582
+ * build rather than starting a second.
524
583
  */
525
584
  resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance>;
585
+ /** Record `transition` as the latest for `strandId` until it settles. */
586
+ private holdTransition;
587
+ /** Wait for the transition requested before this one. Never rejects: its failure belongs to its caller. */
588
+ private settleTransition;
589
+ /** Body of {@link resumeStrand}; at most one runs per strand at a time. */
590
+ private runResume;
526
591
  /**
527
592
  * Honor a founder request against an ALREADY-TRACKED strand — the seam that
528
593
  * closes the "whoever launches first decides whether the bootstrap runs" gap:
@@ -539,7 +604,8 @@ export declare class StrandInstanceManager {
539
604
  * - `'already-founder'` — the retained config already founds; nothing to do.
540
605
  * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
541
606
  * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
542
- * database), so the bootstrap could not run here: the CALLER must wake the
607
+ * database) or being quiesced (its database is closing), so the bootstrap could
608
+ * not run here: the CALLER must wake the
543
609
  * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
544
610
  * manager does not) so the rebuild — which now founds — runs it.
545
611
  * @param resolvePartyKey - Asked for the party's own membership key when (and only
@@ -555,6 +621,39 @@ export declare class StrandInstanceManager {
555
621
  * whose `'needs-resume'` wake fails owes the same withdrawal.
556
622
  */
557
623
  foundExistingStrand(strandId: string, resolvePartyKey?: () => Promise<string | undefined>): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'>;
624
+ /**
625
+ * Give a tracked storage replica ({@link StartStrandConfig.sAppConfig} absent) the app's
626
+ * schema in place — the seam for an app on this machine claiming a strand this node was
627
+ * already hosting. No runtime rebuild: the libp2p node, its peer id and connections, the
628
+ * store and its warm cache all stay, and the new `App` tables read the blocks the replica
629
+ * already holds. Callers attach BEFORE {@link foundExistingStrand}: the founder bootstrap
630
+ * writes the sApp into `Strand.Header` and refuses a database that has none.
631
+ *
632
+ * A runtime build in flight (the replica's launch, or a hibernation wake) is waited out
633
+ * first, so the attach acts on the database that build produced. Then, in order: the
634
+ * schema signature is checked before anything changes; the retained launch config takes
635
+ * the sApp, so every later rebuild applies it through `composeStrand`; the live database —
636
+ * published, or still held by the first-sync gate — gets the schema; and only then does
637
+ * the instance record `sAppInfo` and the sApp's latency hint. A quiesced instance gets the
638
+ * config alone, and its next resume applies the schema.
639
+ *
640
+ * @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
641
+ * is logged and left as it is, as a second claim of a claimed strand always has been —
642
+ * else `'attached'`.
643
+ * @throws when the strand is not tracked (a caller bug, or a launch that failed while
644
+ * this waited on it); when the schema signature is refused, having changed nothing; and
645
+ * whatever the live apply throws (e.g. a quiesce closed the database mid-apply) — the
646
+ * retained config then still carries the sApp and `sAppInfo` stays unset, so the next
647
+ * claim retries the apply.
648
+ *
649
+ * NOTE: two claims of one replica racing each other both pass the `sAppInfo` check and
650
+ * apply concurrently; the loser of the declarative diff can reject, and its `addStrand`
651
+ * retry then resolves `'already-attached'`. Needs an app calling `addStrand` twice at once
652
+ * (or beside a watcher retry); if it is ever seen, chain attaches per strand id.
653
+ */
654
+ attachSApp(strandId: string, sAppConfig: SAppConfig, options?: {
655
+ requireSignedSchemas?: boolean;
656
+ }): Promise<'attached' | 'already-attached'>;
558
657
  /**
559
658
  * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
560
659
  * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
@@ -575,6 +674,14 @@ export declare class StrandInstanceManager {
575
674
  whenWritable(strandId: string, options?: {
576
675
  timeoutMs?: number;
577
676
  }): Promise<StrandInstance>;
677
+ /**
678
+ * The tracked instance once no runtime build of it is in flight, or `undefined` when none
679
+ * is tracked by then — never launched, or the launch waited on failed and dropped its
680
+ * record. For a caller about to hand a tracked instance to an app: until its build settles
681
+ * the instance is `'starting'` with no database, and a failed launch leaves no instance.
682
+ * Never rejects; a failed build is reported by the call that started it.
683
+ */
684
+ whenRuntimeBuilt(strandId: string): Promise<StrandInstance | undefined>;
578
685
  /**
579
686
  * Hand a launch's database to the app: set `instance.database`, retire the gate that
580
687
  * held it, release every {@link whenWritable} waiter, and — when the launch had already
@@ -14,8 +14,9 @@ import { assertStrandScopeKey } from './storage-scope.js';
14
14
  import { StrandFirstSyncGate, StrandAwaitingFirstSyncError, strandFirstSyncComplete, DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS } from './strand-first-sync-gate.js';
15
15
  import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
16
16
  import { strandNodeAddrs } from './strand-network-config.js';
17
+ import { resolveRelayServer } from './relay-server.js';
17
18
  import { superviseRelayReservation } from './relay-reservation.js';
18
- import { connectionManagerTimeouts, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
19
+ import { declaredCohortReadDeadlineMs, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
19
20
  const log = debug('sereus:cadre:strand-manager');
20
21
  const timing = debug('sereus:cadre:timing');
21
22
  /**
@@ -99,6 +100,24 @@ function resolveStrandStorage(provider, strandId) {
99
100
  // lifetime — so the wrap survives every runtime rebuild the instance goes through.
100
101
  return wrapStorageWithCache(storage, strandId);
101
102
  }
103
+ /**
104
+ * Verify an sApp's schema signature (fail-closed unless `requireSignedSchemas` is
105
+ * false) and project the config onto the instance's `sAppInfo`.
106
+ */
107
+ function verifiedSAppInfo(strandId, sAppConfig, requireSignedSchemas) {
108
+ assertSchemaSignature(sAppConfig, { requireSignature: requireSignedSchemas ?? true });
109
+ log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
110
+ return {
111
+ id: sAppConfig.id,
112
+ version: sAppConfig.version,
113
+ schema: sAppConfig.schema,
114
+ signature: sAppConfig.signature
115
+ };
116
+ }
117
+ /** Log wording for what a launch runs: the sApp and its version, or a storage replica. */
118
+ function describeSApp(sAppConfig) {
119
+ return sAppConfig ? `sApp: ${sAppConfig.id} v${sAppConfig.version}` : 'storage replica';
120
+ }
102
121
  /**
103
122
  * Manages individual strand instances - creates and destroys isolated libp2p nodes
104
123
  * for each strand the cadre participates in.
@@ -204,6 +223,23 @@ export class StrandInstanceManager {
204
223
  * wake becomes the better trade.
205
224
  */
206
225
  this.strandStorages = new Map();
226
+ /**
227
+ * The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
228
+ * `resumeStrand`), present until it settles. The instance is tracked for the whole build,
229
+ * with its database not yet constructed or not yet initialized, so a caller that must act
230
+ * on the FINISHED runtime ({@link attachSApp}, {@link whenRuntimeBuilt}) waits on this instead. A failed build's
231
+ * rejection belongs to the call that started it.
232
+ */
233
+ this.runtimeBuilds = new Map();
234
+ /**
235
+ * The latest resume or quiesce requested per strand id, present until it settles. Each one
236
+ * starts only once the one requested before it has settled, so a wake that lands during a
237
+ * quiesce rebuilds after it, and a quiesce requested during a rebuild releases what that
238
+ * rebuild finished. The WHOLE operation, not its entry in {@link runtimeBuilds}: that settles
239
+ * before a resume's own catch records `'error'`, so a joiner awaiting it could read a stale
240
+ * status.
241
+ */
242
+ this.transitions = new Map();
207
243
  this.stopping = false;
208
244
  log('StrandInstanceManager created');
209
245
  }
@@ -259,7 +295,7 @@ export class StrandInstanceManager {
259
295
  log('Strand %s already running', strandId);
260
296
  return this.instances.get(strandId);
261
297
  }
262
- log('Starting strand instance: %s (sApp: %s v%s)', strandId, sAppConfig.id, sAppConfig.version);
298
+ log('Starting strand instance: %s (%s)', strandId, describeSApp(sAppConfig));
263
299
  const tTotal = performance.now();
264
300
  // The id becomes two names below: the storage scope key the embedder's provider
265
301
  // turns into a directory or database name, and `networkName` in buildStrandRuntime,
@@ -269,17 +305,7 @@ export class StrandInstanceManager {
269
305
  // function returns early when no provider is configured — a node with no storage
270
306
  // still reaches buildStrandRuntime and still mints the protocol prefix.
271
307
  assertStrandScopeKey(strandId);
272
- // Verify schema signature before proceeding (fail-closed by default)
273
- const requireSignature = config.requireSignedSchemas ?? true;
274
- assertSchemaSignature(sAppConfig, { requireSignature });
275
- log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
276
- // Convert SAppConfig to SAppInfo for the instance
277
- const sAppInfo = {
278
- id: sAppConfig.id,
279
- version: sAppConfig.version,
280
- schema: sAppConfig.schema,
281
- signature: sAppConfig.signature
282
- };
308
+ const sAppInfo = sAppConfig ? verifiedSAppInfo(strandId, sAppConfig, config.requireSignedSchemas) : undefined;
283
309
  // Resolve this strand's storage ONCE, before anything is recorded. If a factory
284
310
  // function is provided, it is called with the strandId to create strand-specific
285
311
  // storage (e.g. strand-isolated directories). A provider that throws therefore
@@ -289,7 +315,7 @@ export class StrandInstanceManager {
289
315
  log('Strand %s using provided storage provider', strandId);
290
316
  }
291
317
  // Determine latency hint: sApp config > default
292
- const latencyHint = sAppConfig.latencyHint ?? config.defaultLatencyHint;
318
+ const latencyHint = sAppConfig?.latencyHint ?? config.defaultLatencyHint;
293
319
  const instance = {
294
320
  strandId,
295
321
  status: 'starting',
@@ -305,9 +331,9 @@ export class StrandInstanceManager {
305
331
  this.strandStorages.set(strandId, strandStorage);
306
332
  }
307
333
  try {
308
- await this.buildStrandRuntime(instance, config);
334
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, config));
309
335
  timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
310
- log('Strand %s started successfully with sApp %s', strandId, sAppConfig.id);
336
+ log('Strand %s started successfully (%s)', strandId, describeSApp(sAppConfig));
311
337
  return instance;
312
338
  }
313
339
  catch (error) {
@@ -326,6 +352,34 @@ export class StrandInstanceManager {
326
352
  throw error;
327
353
  }
328
354
  }
355
+ /** Hold `build` in {@link runtimeBuilds} until it settles, and settle as it does. */
356
+ async trackRuntimeBuild(strandId, build) {
357
+ this.runtimeBuilds.set(strandId, build);
358
+ try {
359
+ await build;
360
+ }
361
+ finally {
362
+ if (this.runtimeBuilds.get(strandId) === build) {
363
+ this.runtimeBuilds.delete(strandId);
364
+ }
365
+ }
366
+ }
367
+ /**
368
+ * Wait until no runtime build of `strandId` is in flight. Never rejects: a failed build
369
+ * is reported by the call that started it, and the caller re-reads the instance after.
370
+ *
371
+ * NOTE: after a failed launch the re-read sees the instance gone only because
372
+ * `startStrand`'s catch deletes it synchronously, and its continuation is queued before
373
+ * this waiter's. If that cleanup ever awaits before `instances.delete`, a waiter can act
374
+ * on the dying record; move the cleanup inside the tracked promise then.
375
+ */
376
+ async settleRuntimeBuilds(strandId) {
377
+ for (let build = this.runtimeBuilds.get(strandId); build; build = this.runtimeBuilds.get(strandId)) {
378
+ await build.catch((error) => {
379
+ log('Strand %s: the runtime build waited on failed (reported by its launcher): %o', strandId, error);
380
+ });
381
+ }
382
+ }
329
383
  /**
330
384
  * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
331
385
  * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
@@ -353,9 +407,9 @@ export class StrandInstanceManager {
353
407
  // with — hence one binding for both, not two literals that can drift.
354
408
  const networkName = `strand-${strandId}`;
355
409
  const protocolPrefix = `/optimystic/${networkName}`;
356
- // Determine relay mode: if explicitly set in config, use that;
357
- // otherwise default to true for storage profile nodes.
358
- const enableRelay = config.network?.enableRelay ?? (config.profile === 'storage');
410
+ // Whether this node runs the circuit-relay server, and its init — the same
411
+ // resolution the control node takes from the same config (`relay-server.ts`).
412
+ const relayServer = resolveRelayServer(config.network, config.profile);
359
413
  // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
360
414
  // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
361
415
  // bind one port), the announce config is dropped (it names the control node's
@@ -423,7 +477,12 @@ export class StrandInstanceManager {
423
477
  networkName,
424
478
  storage: strandStorage,
425
479
  fretProfile: config.profile === 'storage' ? 'core' : 'edge',
426
- relay: enableRelay,
480
+ relay: relayServer.enabled,
481
+ // NOTE: unlike the control node, a strand node has no unauthorized-reservation budget,
482
+ // so with the per-connection cap off any peer that reaches this node can hold one of
483
+ // `maxReservations` slots and forward uncapped through it. If strand relays are ever
484
+ // abused for bandwidth, give strand nodes a budget or a capped init of their own.
485
+ ...(relayServer.enabled && { relayServerInit: relayServer.init }),
427
486
  clusterSize: strandClusterSize,
428
487
  // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
429
488
  // breadth reasoning, and the shape match with the control policy is a coincidence.
@@ -435,11 +494,12 @@ export class StrandInstanceManager {
435
494
  // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
436
495
  // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
437
496
  // serving set that changed while the strand slept.
438
- // The read deadline is the same field the control node reads, because the two
439
- // networks ride one link; absent, the base policy's COHORT_READ_DEADLINE_MS stands.
497
+ // The read deadline is settled by the same helper the control node uses, because the
498
+ // two networks ride one link: the host's own, else derived from its declared
499
+ // `linkRoundTripMs`, else nothing, and the base policy's COHORT_READ_DEADLINE_MS stands.
440
500
  clusterPolicy: strandClusterPolicy(strandClusterSize, {
441
501
  servingMachines: config.servingMachines,
442
- cohortQueryTimeoutMs: config.network?.cohortQueryTimeoutMs
502
+ cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(config.network)
443
503
  }),
444
504
  arachnode: {
445
505
  enableRingZulu: config.profile === 'storage'
@@ -452,10 +512,10 @@ export class StrandInstanceManager {
452
512
  // widened ping deadline has to reach the strand nodes too (see
453
513
  // DEFAULT_CONNECTION_MONITOR).
454
514
  connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
455
- // The same declared-link limits the control node takes, and for the same reason: a
456
- // strand node is the listener for every other member's strand node (`link-budget.ts`,
457
- // `connectionManagerTimeouts`).
458
- connectionManager: connectionManagerTimeouts(config.network?.linkRoundTripMs),
515
+ // The same declared link the control node states, and for the same reason: a strand
516
+ // node is also the listener for every other member's strand node, and Optimystic
517
+ // derives that listener's limit from this (`link-budget.ts`).
518
+ linkRoundTripMs: resolveLinkRoundTripMs(config.network?.linkRoundTripMs),
459
519
  // Listen entries plus the WebSocket transport switch they imply — a strand node
460
520
  // announces nothing the operator configured (`strand-network-config.ts`), and
461
521
  // spreads AFTER `transports` above because the switch is a no-op whenever the
@@ -663,7 +723,7 @@ export class StrandInstanceManager {
663
723
  // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
664
724
  //
665
725
  // NOTE: a relay that is down costs this launch one full drive — the reservation budget
666
- // counted from the declared link round trip, 14 s at its default (`link-budget.ts`) — and
726
+ // counted from the declared link round trip, 18 s at its default (`link-budget.ts`) — and
667
727
  // `StrandWatcher` launches strands one at a time, so N strands cost N of those in
668
728
  // bring-up during a relay outage, and MORE on a host that declared a slower link. If that
669
729
  // ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
@@ -671,8 +731,9 @@ export class StrandInstanceManager {
671
731
  t0 = performance.now();
672
732
  await this.awaitFirstRelayAttempts(strandId);
673
733
  timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
734
+ // No `lastActivity` stamp: bringing a runtime up is not activity. A check-in marks
735
+ // activity before its resume and must see only what landed while the strand was up.
674
736
  instance.status = liveStrandStatus(instance);
675
- instance.lastActivity = new Date();
676
737
  if (instance.status === 'syncing') {
677
738
  log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
678
739
  'a member of the strand is reached', strandId);
@@ -934,23 +995,53 @@ export class StrandInstanceManager {
934
995
  * Quiesce a strand: release its strand-network resources (stop the libp2p node,
935
996
  * close the StrandDatabase) while RETAINING the instance record — identity,
936
997
  * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
937
- * resumed later. Mechanically this is `stopStrand` minus the instance/config
938
- * deletion. The caller sets the post-quiesce status (e.g. `hibernating`).
939
- * No-ops when the strand is missing or already quiesced.
998
+ * resumed later, then mark it `hibernating`. Mechanically this is `stopStrand` minus
999
+ * the instance/config deletion. A tracked strand with no runtime left (a failed
1000
+ * rebuild already released it) is only marked; an untracked one is left alone.
1001
+ *
1002
+ * Ordered with {@link resumeStrand} (see {@link transitions}): it starts once the resume
1003
+ * or launch build before it has settled, so it releases a finished runtime, and the
1004
+ * status write is its last step, so a resume requested meanwhile writes after it. A call
1005
+ * made while another quiesce is the latest request joins it. A failed release leaves the
1006
+ * status as it was and rejects.
940
1007
  */
941
1008
  async quiesceStrand(strandId) {
1009
+ const previous = this.transitions.get(strandId);
1010
+ if (previous?.kind === 'quiesce') {
1011
+ log('quiesceStrand: strand %s — joining the quiesce already requested', strandId);
1012
+ return previous.operation;
1013
+ }
1014
+ const operation = this.runQuiesce(strandId, previous);
1015
+ this.holdTransition(strandId, { kind: 'quiesce', operation });
1016
+ return operation;
1017
+ }
1018
+ /** Body of {@link quiesceStrand}. */
1019
+ async runQuiesce(strandId, previous) {
1020
+ await this.settleTransition(strandId, previous);
1021
+ await this.settleRuntimeBuilds(strandId);
942
1022
  const instance = this.instances.get(strandId);
943
1023
  if (!instance) {
944
1024
  log('quiesceStrand: strand %s not found', strandId);
945
1025
  return;
946
1026
  }
947
- if (!instance.libp2pNode && !instance.database) {
1027
+ if (instance.libp2pNode || instance.database) {
1028
+ log('Quiescing strand instance: %s', strandId);
1029
+ await this.releaseRuntime(instance);
1030
+ log('Strand %s quiesced (resources released, instance retained)', strandId);
1031
+ }
1032
+ else {
948
1033
  log('quiesceStrand: strand %s already quiesced', strandId);
949
- return;
950
1034
  }
951
- log('Quiescing strand instance: %s', strandId);
952
- await this.releaseRuntime(instance);
953
- log('Strand %s quiesced (resources released, instance retained)', strandId);
1035
+ instance.status = 'hibernating';
1036
+ }
1037
+ /**
1038
+ * Whether the latest runtime transition requested for `strandId` is a quiesce that has
1039
+ * not finished. Its handles may still be attached and its status still live, but it is on
1040
+ * its way to `hibernating`, so a caller that needs it up must {@link resumeStrand} it — the
1041
+ * resume runs after the quiesce — rather than treat it as live.
1042
+ */
1043
+ isQuiescing(strandId) {
1044
+ return this.transitions.get(strandId)?.kind === 'quiesce';
954
1045
  }
955
1046
  /**
956
1047
  * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
@@ -959,11 +1050,58 @@ export class StrandInstanceManager {
959
1050
  * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
960
1051
  * count) and updates the retained config so a later resume reuses the latest
961
1052
  * values. Returns the live instance unchanged if it is already running.
1053
+ *
1054
+ * Overlapping calls share one rebuild: a call made while a resume is the latest request
1055
+ * for the strand — a wake landing during a check-in — joins it and settles exactly as it
1056
+ * does, `'error'` status included. The joiner's own `overrides` are ignored; the first
1057
+ * resume's seed wins, and both callers resolved the cohort seed moments apart. A call made
1058
+ * while a quiesce is the latest request runs after that quiesce and rebuilds (see
1059
+ * {@link transitions}), and one made while `startStrand` is still building waits for that
1060
+ * build rather than starting a second.
962
1061
  */
963
1062
  async resumeStrand(strandId, overrides) {
964
1063
  if (this.stopping) {
965
1064
  throw new Error('StrandInstanceManager is stopping');
966
1065
  }
1066
+ // No `await` between this read and `holdTransition`, or a second caller slips through
1067
+ // and builds.
1068
+ const previous = this.transitions.get(strandId);
1069
+ if (previous?.kind === 'resume') {
1070
+ log('resumeStrand: strand %s — joining the resume already in flight', strandId);
1071
+ return previous.operation;
1072
+ }
1073
+ const operation = this.runResume(strandId, previous, overrides);
1074
+ this.holdTransition(strandId, { kind: 'resume', operation });
1075
+ return operation;
1076
+ }
1077
+ /** Record `transition` as the latest for `strandId` until it settles. */
1078
+ holdTransition(strandId, transition) {
1079
+ this.transitions.set(strandId, transition);
1080
+ const release = () => {
1081
+ if (this.transitions.get(strandId) === transition) {
1082
+ this.transitions.delete(strandId);
1083
+ }
1084
+ };
1085
+ // The rejection itself reaches the caller through the operation returned to it.
1086
+ void transition.operation.then(release, release);
1087
+ }
1088
+ /** Wait for the transition requested before this one. Never rejects: its failure belongs to its caller. */
1089
+ async settleTransition(strandId, previous) {
1090
+ if (!previous) {
1091
+ return;
1092
+ }
1093
+ await previous.operation.catch((error) => {
1094
+ log('Strand %s: the %s queued ahead failed (reported by its caller): %o', strandId, previous.kind, error);
1095
+ });
1096
+ }
1097
+ /** Body of {@link resumeStrand}; at most one runs per strand at a time. */
1098
+ async runResume(strandId, previous, overrides) {
1099
+ // Before any "already live" test: a quiesce ahead still has the handles attached, and
1100
+ // mid-build (a launch still building) the libp2p node is attached while the database is
1101
+ // not. A failed launch has already dropped the instance, and the check below reports it
1102
+ // untracked.
1103
+ await this.settleTransition(strandId, previous);
1104
+ await this.settleRuntimeBuilds(strandId);
967
1105
  const instance = this.instances.get(strandId);
968
1106
  if (!instance) {
969
1107
  throw new Error(`Cannot resume strand ${strandId}: not tracked`);
@@ -997,7 +1135,7 @@ export class StrandInstanceManager {
997
1135
  this.launchConfigs.set(strandId, resumeConfig);
998
1136
  instance.status = 'starting';
999
1137
  try {
1000
- await this.buildStrandRuntime(instance, resumeConfig);
1138
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, resumeConfig));
1001
1139
  timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
1002
1140
  log('Strand %s resumed successfully', strandId);
1003
1141
  return instance;
@@ -1025,7 +1163,8 @@ export class StrandInstanceManager {
1025
1163
  * - `'already-founder'` — the retained config already founds; nothing to do.
1026
1164
  * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1027
1165
  * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1028
- * database), so the bootstrap could not run here: the CALLER must wake the
1166
+ * database) or being quiesced (its database is closing), so the bootstrap could
1167
+ * not run here: the CALLER must wake the
1029
1168
  * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1030
1169
  * manager does not) so the rebuild — which now founds — runs it.
1031
1170
  * @param resolvePartyKey - Asked for the party's own membership key when (and only
@@ -1054,7 +1193,7 @@ export class StrandInstanceManager {
1054
1193
  // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1055
1194
  // config object, which is not ours to rewrite.
1056
1195
  this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1057
- if (!instance.database && !this.firstSyncGates.has(strandId)) {
1196
+ if (this.isQuiescing(strandId) || (!instance.database && !this.firstSyncGates.has(strandId))) {
1058
1197
  return 'needs-resume';
1059
1198
  }
1060
1199
  try {
@@ -1066,6 +1205,70 @@ export class StrandInstanceManager {
1066
1205
  }
1067
1206
  return 'bootstrapped';
1068
1207
  }
1208
+ /**
1209
+ * Give a tracked storage replica ({@link StartStrandConfig.sAppConfig} absent) the app's
1210
+ * schema in place — the seam for an app on this machine claiming a strand this node was
1211
+ * already hosting. No runtime rebuild: the libp2p node, its peer id and connections, the
1212
+ * store and its warm cache all stay, and the new `App` tables read the blocks the replica
1213
+ * already holds. Callers attach BEFORE {@link foundExistingStrand}: the founder bootstrap
1214
+ * writes the sApp into `Strand.Header` and refuses a database that has none.
1215
+ *
1216
+ * A runtime build in flight (the replica's launch, or a hibernation wake) is waited out
1217
+ * first, so the attach acts on the database that build produced. Then, in order: the
1218
+ * schema signature is checked before anything changes; the retained launch config takes
1219
+ * the sApp, so every later rebuild applies it through `composeStrand`; the live database —
1220
+ * published, or still held by the first-sync gate — gets the schema; and only then does
1221
+ * the instance record `sAppInfo` and the sApp's latency hint. A quiesced instance gets the
1222
+ * config alone, and its next resume applies the schema.
1223
+ *
1224
+ * @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
1225
+ * is logged and left as it is, as a second claim of a claimed strand always has been —
1226
+ * else `'attached'`.
1227
+ * @throws when the strand is not tracked (a caller bug, or a launch that failed while
1228
+ * this waited on it); when the schema signature is refused, having changed nothing; and
1229
+ * whatever the live apply throws (e.g. a quiesce closed the database mid-apply) — the
1230
+ * retained config then still carries the sApp and `sAppInfo` stays unset, so the next
1231
+ * claim retries the apply.
1232
+ *
1233
+ * NOTE: two claims of one replica racing each other both pass the `sAppInfo` check and
1234
+ * apply concurrently; the loser of the declarative diff can reject, and its `addStrand`
1235
+ * retry then resolves `'already-attached'`. Needs an app calling `addStrand` twice at once
1236
+ * (or beside a watcher retry); if it is ever seen, chain attaches per strand id.
1237
+ */
1238
+ async attachSApp(strandId, sAppConfig, options = {}) {
1239
+ await this.settleRuntimeBuilds(strandId);
1240
+ const instance = this.instances.get(strandId);
1241
+ const config = this.launchConfigs.get(strandId);
1242
+ if (!instance || !config) {
1243
+ throw new Error(`Cannot attach an sApp to strand ${strandId}: not tracked`);
1244
+ }
1245
+ if (instance.sAppInfo) {
1246
+ if (instance.sAppInfo.id !== sAppConfig.id) {
1247
+ log('attachSApp: strand %s already runs sApp %s — the claim for sApp %s is ignored', strandId, instance.sAppInfo.id, sAppConfig.id);
1248
+ }
1249
+ return 'already-attached';
1250
+ }
1251
+ const sAppInfo = verifiedSAppInfo(strandId, sAppConfig, options.requireSignedSchemas);
1252
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1253
+ // config object, which is not ours to rewrite.
1254
+ this.launchConfigs.set(strandId, { ...config, sAppConfig });
1255
+ const database = instance.database ?? this.firstSyncGates.get(strandId)?.database;
1256
+ if (database) {
1257
+ await database.attachAppSchema(sAppConfig);
1258
+ }
1259
+ else {
1260
+ log('attachSApp: strand %s is quiesced — its next resume applies the sApp schema', strandId);
1261
+ }
1262
+ instance.sAppInfo = sAppInfo;
1263
+ if (sAppConfig.latencyHint) {
1264
+ // HibernationManager reads the hint whenever it arms a timer, so the app's hint governs
1265
+ // from the next one: a timer already armed runs once at the old duration, and a replica
1266
+ // launched under a realtime default was never tracked, so it stays up until relaunched.
1267
+ instance.latencyHint = sAppConfig.latencyHint;
1268
+ }
1269
+ log('Strand %s upgraded from storage replica to %s', strandId, describeSApp(sAppConfig));
1270
+ return 'attached';
1271
+ }
1069
1272
  /**
1070
1273
  * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1071
1274
  * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
@@ -1117,6 +1320,17 @@ export class StrandInstanceManager {
1117
1320
  waiters.add(waiter);
1118
1321
  });
1119
1322
  }
1323
+ /**
1324
+ * The tracked instance once no runtime build of it is in flight, or `undefined` when none
1325
+ * is tracked by then — never launched, or the launch waited on failed and dropped its
1326
+ * record. For a caller about to hand a tracked instance to an app: until its build settles
1327
+ * the instance is `'starting'` with no database, and a failed launch leaves no instance.
1328
+ * Never rejects; a failed build is reported by the call that started it.
1329
+ */
1330
+ async whenRuntimeBuilt(strandId) {
1331
+ await this.settleRuntimeBuilds(strandId);
1332
+ return this.instances.get(strandId);
1333
+ }
1120
1334
  /**
1121
1335
  * Hand a launch's database to the app: set `instance.database`, retire the gate that
1122
1336
  * held it, release every {@link whenWritable} waiter, and — when the launch had already
@@ -1146,9 +1360,11 @@ export class StrandInstanceManager {
1146
1360
  this.firstSyncGates.delete(strandId);
1147
1361
  }
1148
1362
  instance.database = database;
1149
- instance.lastActivity = new Date();
1150
1363
  const wasGated = instance.status !== 'starting';
1151
1364
  if (wasGated) {
1365
+ // A Header delivered by a peer is strand activity; a publish during bring-up is not
1366
+ // (see the build's end in `buildStrandRuntime`).
1367
+ instance.lastActivity = new Date();
1152
1368
  instance.status = 'active';
1153
1369
  log('Strand %s is now writable: Strand.Header held', strandId);
1154
1370
  }