@serfab/cadre-core 1.16.0 → 1.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/README.md +4 -3
  2. package/dist/bootstrap-peer-store-file.js +1 -1
  3. package/dist/bootstrap-peer-store.d.ts +1 -1
  4. package/dist/bootstrap-peer-store.js +4 -4
  5. package/dist/cadre-invite-protocol.d.ts +81 -142
  6. package/dist/cadre-invite-protocol.js +168 -458
  7. package/dist/cadre-invite-protocol.js.map +1 -1
  8. package/dist/cadre-join-request.d.ts +15 -2
  9. package/dist/cadre-join-request.js +22 -3
  10. package/dist/cadre-join-request.js.map +1 -1
  11. package/dist/cadre-node.d.ts +1089 -475
  12. package/dist/cadre-node.js +2170 -1037
  13. package/dist/cadre-node.js.map +1 -1
  14. package/dist/control-authorization.d.ts +36 -14
  15. package/dist/control-authorization.js +20 -3
  16. package/dist/control-authorization.js.map +1 -1
  17. package/dist/control-cohort.d.ts +88 -32
  18. package/dist/control-cohort.js +91 -33
  19. package/dist/control-cohort.js.map +1 -1
  20. package/dist/control-database.d.ts +856 -318
  21. package/dist/control-database.js +1422 -701
  22. package/dist/control-database.js.map +1 -1
  23. package/dist/control-formation-recorder.d.ts +46 -43
  24. package/dist/control-formation-recorder.js +77 -78
  25. package/dist/control-formation-recorder.js.map +1 -1
  26. package/dist/control-protocol-guard.d.ts +11 -4
  27. package/dist/control-protocol-guard.js +20 -5
  28. package/dist/control-protocol-guard.js.map +1 -1
  29. package/dist/control-schema-version.d.ts +97 -0
  30. package/dist/control-schema-version.js +225 -0
  31. package/dist/control-schema-version.js.map +1 -0
  32. package/dist/control-schema.d.ts +45 -5
  33. package/dist/control-schema.js +262 -78
  34. package/dist/control-schema.js.map +1 -1
  35. package/dist/control-write-retry.d.ts +45 -10
  36. package/dist/control-write-retry.js +92 -22
  37. package/dist/control-write-retry.js.map +1 -1
  38. package/dist/delegate-admission.d.ts +13 -0
  39. package/dist/delegate-admission.js +27 -0
  40. package/dist/delegate-admission.js.map +1 -1
  41. package/dist/ed25519-key.d.ts +10 -7
  42. package/dist/ed25519-key.js +23 -20
  43. package/dist/ed25519-key.js.map +1 -1
  44. package/dist/edge-node/config.d.ts +122 -0
  45. package/dist/edge-node/config.js +199 -0
  46. package/dist/edge-node/config.js.map +1 -0
  47. package/dist/edge-node/dissolve.d.ts +95 -0
  48. package/dist/edge-node/dissolve.js +184 -0
  49. package/dist/edge-node/dissolve.js.map +1 -0
  50. package/dist/edge-node/edge-join.d.ts +44 -0
  51. package/dist/edge-node/edge-join.js +227 -0
  52. package/dist/edge-node/edge-join.js.map +1 -0
  53. package/dist/edge-node/edge-node.d.ts +295 -0
  54. package/dist/edge-node/edge-node.js +704 -0
  55. package/dist/edge-node/edge-node.js.map +1 -0
  56. package/dist/edge-node/index.d.ts +18 -0
  57. package/dist/edge-node/index.js +18 -0
  58. package/dist/edge-node/index.js.map +1 -0
  59. package/dist/edge-node/join-credential.d.ts +11 -0
  60. package/dist/edge-node/join-credential.js +11 -0
  61. package/dist/edge-node/join-credential.js.map +1 -0
  62. package/dist/edge-node/lazy-raw-storage.d.ts +40 -0
  63. package/dist/edge-node/lazy-raw-storage.js +134 -0
  64. package/dist/edge-node/lazy-raw-storage.js.map +1 -0
  65. package/dist/edge-node/node-local.d.ts +60 -0
  66. package/dist/edge-node/node-local.js +58 -0
  67. package/dist/edge-node/node-local.js.map +1 -0
  68. package/dist/edge-node/options.d.ts +83 -0
  69. package/dist/edge-node/options.js +110 -0
  70. package/dist/edge-node/options.js.map +1 -0
  71. package/dist/edge-node/owner-grant.d.ts +41 -0
  72. package/dist/edge-node/owner-grant.js +38 -0
  73. package/dist/edge-node/owner-grant.js.map +1 -0
  74. package/dist/edge-node/platform.d.ts +140 -0
  75. package/dist/edge-node/platform.js +12 -0
  76. package/dist/edge-node/platform.js.map +1 -0
  77. package/dist/edge-node/strand-move.d.ts +115 -0
  78. package/dist/edge-node/strand-move.js +223 -0
  79. package/dist/edge-node/strand-move.js.map +1 -0
  80. package/dist/edge-node/strands.d.ts +28 -0
  81. package/dist/edge-node/strands.js +60 -0
  82. package/dist/edge-node/strands.js.map +1 -0
  83. package/dist/enrolled-machine-store.d.ts +3 -3
  84. package/dist/enrolled-machine-store.js +3 -3
  85. package/dist/formation-token.d.ts +48 -0
  86. package/dist/formation-token.js +26 -0
  87. package/dist/formation-token.js.map +1 -0
  88. package/dist/index.d.ts +27 -17
  89. package/dist/index.js +39 -17
  90. package/dist/index.js.map +1 -1
  91. package/dist/joined-strand-store.d.ts +29 -12
  92. package/dist/joined-strand-store.js +66 -27
  93. package/dist/joined-strand-store.js.map +1 -1
  94. package/dist/key-store.d.ts +9 -4
  95. package/dist/key-store.js +9 -4
  96. package/dist/key-store.js.map +1 -1
  97. package/dist/link-budget.d.ts +9 -7
  98. package/dist/link-budget.js +9 -7
  99. package/dist/link-budget.js.map +1 -1
  100. package/dist/machine-removal.d.ts +19 -39
  101. package/dist/machine-removal.js +24 -98
  102. package/dist/machine-removal.js.map +1 -1
  103. package/dist/member-redemption.d.ts +256 -0
  104. package/dist/member-redemption.js +490 -0
  105. package/dist/member-redemption.js.map +1 -0
  106. package/dist/membership-view.d.ts +91 -0
  107. package/dist/membership-view.js +110 -0
  108. package/dist/membership-view.js.map +1 -0
  109. package/dist/owner-anchor-sync.d.ts +4 -4
  110. package/dist/owner-anchor-sync.js +16 -15
  111. package/dist/owner-anchor-sync.js.map +1 -1
  112. package/dist/owner-grant-protocol.d.ts +251 -0
  113. package/dist/owner-grant-protocol.js +344 -0
  114. package/dist/owner-grant-protocol.js.map +1 -0
  115. package/dist/owner-key-export.d.ts +70 -0
  116. package/dist/owner-key-export.js +279 -0
  117. package/dist/owner-key-export.js.map +1 -0
  118. package/dist/owner-key-flows.d.ts +55 -0
  119. package/dist/owner-key-flows.js +60 -0
  120. package/dist/owner-key-flows.js.map +1 -0
  121. package/dist/owner-key-store.d.ts +31 -0
  122. package/dist/owner-key-store.js +81 -0
  123. package/dist/owner-key-store.js.map +1 -0
  124. package/dist/owner-keys.d.ts +78 -0
  125. package/dist/owner-keys.js +133 -0
  126. package/dist/owner-keys.js.map +1 -0
  127. package/dist/owner-signer.d.ts +54 -0
  128. package/dist/owner-signer.js +51 -0
  129. package/dist/owner-signer.js.map +1 -0
  130. package/dist/peer-addr-book.d.ts +10 -1
  131. package/dist/peer-addr-book.js +4 -3
  132. package/dist/peer-addr-book.js.map +1 -1
  133. package/dist/peer-authorization.d.ts +83 -18
  134. package/dist/peer-authorization.js +123 -53
  135. package/dist/peer-authorization.js.map +1 -1
  136. package/dist/peer-join-backfill.d.ts +89 -20
  137. package/dist/peer-join-backfill.js +265 -108
  138. package/dist/peer-join-backfill.js.map +1 -1
  139. package/dist/pending-join-runner.d.ts +48 -18
  140. package/dist/pending-join-runner.js +74 -56
  141. package/dist/pending-join-runner.js.map +1 -1
  142. package/dist/provider/rent-node.js +1 -1
  143. package/dist/provider/rent-node.js.map +1 -1
  144. package/dist/seed-bootstrap.d.ts +162 -133
  145. package/dist/seed-bootstrap.js +300 -283
  146. package/dist/seed-bootstrap.js.map +1 -1
  147. package/dist/seed-trust-policy.js +4 -4
  148. package/dist/strand-addr-protocol.d.ts +4 -1
  149. package/dist/strand-addr-protocol.js +8 -2
  150. package/dist/strand-addr-protocol.js.map +1 -1
  151. package/dist/strand-cohort.d.ts +3 -0
  152. package/dist/strand-cohort.js.map +1 -1
  153. package/dist/strand-database.d.ts +24 -2
  154. package/dist/strand-database.js +29 -6
  155. package/dist/strand-database.js.map +1 -1
  156. package/dist/strand-first-sync-gate.d.ts +176 -88
  157. package/dist/strand-first-sync-gate.js +288 -121
  158. package/dist/strand-first-sync-gate.js.map +1 -1
  159. package/dist/strand-formation-deadlines.d.ts +9 -9
  160. package/dist/strand-formation-deadlines.js +9 -9
  161. package/dist/strand-formation-manager.d.ts +23 -17
  162. package/dist/strand-formation-manager.js +46 -44
  163. package/dist/strand-formation-manager.js.map +1 -1
  164. package/dist/strand-formation-protocol.d.ts +5 -8
  165. package/dist/strand-formation-protocol.js +101 -16
  166. package/dist/strand-formation-protocol.js.map +1 -1
  167. package/dist/strand-instance-manager.d.ts +27 -18
  168. package/dist/strand-instance-manager.js +119 -50
  169. package/dist/strand-instance-manager.js.map +1 -1
  170. package/dist/strand-membership-reconciler.d.ts +16 -13
  171. package/dist/strand-membership-reconciler.js +30 -22
  172. package/dist/strand-membership-reconciler.js.map +1 -1
  173. package/dist/strand-membership-writer.d.ts +45 -22
  174. package/dist/strand-membership-writer.js +226 -131
  175. package/dist/strand-membership-writer.js.map +1 -1
  176. package/dist/strand-move.d.ts +6 -5
  177. package/dist/strand-move.js +36 -32
  178. package/dist/strand-move.js.map +1 -1
  179. package/dist/strand-peer-redial.js +2 -3
  180. package/dist/strand-peer-redial.js.map +1 -1
  181. package/dist/strand-revocation-enforcer.d.ts +134 -60
  182. package/dist/strand-revocation-enforcer.js +144 -71
  183. package/dist/strand-revocation-enforcer.js.map +1 -1
  184. package/dist/strand-solicitation.d.ts +50 -39
  185. package/dist/strand-solicitation.js +10 -3
  186. package/dist/strand-solicitation.js.map +1 -1
  187. package/dist/strand-watcher.d.ts +80 -13
  188. package/dist/strand-watcher.js +152 -48
  189. package/dist/strand-watcher.js.map +1 -1
  190. package/dist/table-change-watch.d.ts +63 -0
  191. package/dist/table-change-watch.js +209 -0
  192. package/dist/table-change-watch.js.map +1 -0
  193. package/dist/timeout-scheduler.d.ts +4 -3
  194. package/dist/timeout-scheduler.js +4 -3
  195. package/dist/timeout-scheduler.js.map +1 -1
  196. package/dist/types.d.ts +374 -150
  197. package/dist/types.js +4 -4
  198. package/dist/types.js.map +1 -1
  199. package/package.json +17 -8
  200. package/src/bootstrap-peer-store-file.ts +44 -44
  201. package/src/bootstrap-peer-store.ts +214 -214
  202. package/src/cadre-invite-protocol.ts +740 -1052
  203. package/src/cadre-join-request.ts +24 -3
  204. package/src/cadre-node.ts +2402 -1079
  205. package/src/control-authorization.ts +47 -14
  206. package/src/control-cohort.ts +204 -128
  207. package/src/control-database.ts +1749 -820
  208. package/src/control-formation-recorder.ts +359 -353
  209. package/src/control-protocol-guard.ts +23 -6
  210. package/src/control-schema-version.ts +301 -0
  211. package/src/control-schema.ts +273 -78
  212. package/src/control-write-retry.ts +641 -567
  213. package/src/delegate-admission.ts +351 -314
  214. package/src/ed25519-key.ts +174 -171
  215. package/src/edge-node/config.ts +277 -0
  216. package/src/edge-node/dissolve.ts +251 -0
  217. package/src/edge-node/edge-join.ts +319 -0
  218. package/src/edge-node/edge-node.ts +1045 -0
  219. package/src/edge-node/index.ts +59 -0
  220. package/src/edge-node/join-credential.ts +16 -0
  221. package/src/edge-node/lazy-raw-storage.ts +178 -0
  222. package/src/edge-node/node-local.ts +84 -0
  223. package/src/edge-node/options.ts +179 -0
  224. package/src/edge-node/owner-grant.ts +63 -0
  225. package/src/edge-node/platform.ts +151 -0
  226. package/src/edge-node/strand-move.ts +270 -0
  227. package/src/edge-node/strands.ts +72 -0
  228. package/src/enrolled-machine-store.ts +3 -3
  229. package/src/formation-token.ts +63 -0
  230. package/src/index.ts +899 -789
  231. package/src/joined-strand-store.ts +73 -28
  232. package/src/key-store.ts +10 -4
  233. package/src/link-budget.ts +9 -7
  234. package/src/machine-removal.ts +26 -111
  235. package/src/member-redemption.ts +649 -0
  236. package/src/membership-view.ts +144 -0
  237. package/src/owner-anchor-sync.ts +19 -18
  238. package/src/owner-grant-protocol.ts +547 -0
  239. package/src/owner-key-export.ts +348 -0
  240. package/src/owner-key-flows.ts +109 -0
  241. package/src/owner-key-store.ts +85 -0
  242. package/src/owner-keys.ts +178 -0
  243. package/src/owner-signer.ts +94 -0
  244. package/src/peer-addr-book.ts +4 -3
  245. package/src/peer-authorization.ts +558 -488
  246. package/src/peer-join-backfill.ts +971 -761
  247. package/src/pending-join-runner.ts +76 -53
  248. package/src/provider/rent-node.ts +1 -1
  249. package/src/seed-bootstrap.ts +343 -349
  250. package/src/seed-trust-policy.ts +4 -4
  251. package/src/strand-addr-protocol.ts +547 -538
  252. package/src/strand-cohort.ts +4 -0
  253. package/src/strand-database.ts +352 -319
  254. package/src/strand-first-sync-gate.ts +334 -133
  255. package/src/strand-formation-deadlines.ts +227 -227
  256. package/src/strand-formation-manager.ts +731 -725
  257. package/src/strand-formation-protocol.ts +122 -24
  258. package/src/strand-instance-manager.ts +2092 -2026
  259. package/src/strand-membership-reconciler.ts +33 -25
  260. package/src/strand-membership-writer.ts +2093 -1974
  261. package/src/strand-move.ts +50 -38
  262. package/src/strand-peer-redial.ts +2 -3
  263. package/src/strand-revocation-enforcer.ts +740 -654
  264. package/src/strand-solicitation.ts +61 -34
  265. package/src/strand-watcher.ts +175 -56
  266. package/src/table-change-watch.ts +285 -0
  267. package/src/timeout-scheduler.ts +29 -28
  268. package/src/types.ts +391 -152
@@ -1,2026 +1,2092 @@
1
- import debug from 'debug';
2
- import type { Libp2p, PrivateKey } from '@libp2p/interface';
3
- import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
4
- import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
5
- import { StrandDatabase } from './strand-database.js';
6
- import type { StrandCommitDurability, StrandCommitHeld } from './strand-commit-durability.js';
7
- import { PeerJoinBackfill, type PeerJoinBackfillConfig } from './peer-join-backfill.js';
8
- import { strandNetworkStatePersistence, type StrandNetworkStateStore } from './strand-network-state.js';
9
- import {
10
- StrandRevocationEnforcer,
11
- createRevocationConnectionGater,
12
- readStrandRevocationRows,
13
- type StrandRevocationEnforcementConfig
14
- } from './strand-revocation-enforcer.js';
15
- import {
16
- StrandMembershipReconciler,
17
- type PendingMembershipInviteSource,
18
- type StrandMembershipReconciliationConfig
19
- } from './strand-membership-reconciler.js';
20
- import { removeMemberPeer } from './strand-membership-writer.js';
21
- import { strandMemberKeyPair } from './strand-member-key.js';
22
- import { assertSchemaSignature } from './schema-verification.js';
23
- import { assertStrandScopeKey } from './storage-scope.js';
24
- import {
25
- StrandFirstSyncGate,
26
- StrandAwaitingFirstSyncError,
27
- strandFirstSyncComplete,
28
- DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS,
29
- type StrandFirstSyncConfig
30
- } from './strand-first-sync-gate.js';
31
- import type {
32
- StrandInstance,
33
- StrandRow,
34
- StorageConfig,
35
- NetworkConfig,
36
- LatencyHint,
37
- NodeProfile,
38
- SAppConfig,
39
- SAppInfo,
40
- RawStorageProvider,
41
- Libp2pNodeWithRepo
42
- } from './types.js';
43
- import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
44
- import { strandNodeAddrs } from './strand-network-config.js';
45
- import { nodeTransportsOption } from './node-transports.js';
46
- import { resolveRelayServer } from './relay-server.js';
47
- import { strandCohortTopicOption, type StrandReactivityConfig } from './strand-reactivity.js';
48
- import { superviseRelayReservation, type RelayReservationSupervisor } from './relay-reservation.js';
49
- import { strandRedialTimings, superviseStrandPeerRedial, type StrandPeerRedial } from './strand-peer-redial.js';
50
- import { groupAddrsByPeerId } from './peer-addr-book.js';
51
- import { isCohortUnreachableRead } from './control-read-retry.js';
52
- import { declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
53
-
54
- const log = debug('sereus:cadre:strand-manager');
55
-
56
- /**
57
- * How often a joining launch that could not build its database, because no seed peer had
58
- * answered, retries the build on its running libp2p node
59
- * (`StrandInstanceManager.deferUnreachableJoin`). Each attempt composes a fresh database and
60
- * re-reads the schema catalog over the network, so it is spaced well above the first-sync
61
- * gate's probe cadence.
62
- *
63
- * NOTE: the 5 s dates from when each attempt also rebuilt the libp2p node. A joiner whose seed
64
- * peer answered just after a failed attempt waits up to this long to become writable; if that
65
- * wait shows up in a measured join, try a shorter first retry.
66
- */
67
- export const UNREACHABLE_JOIN_RETRY_MS = 5_000;
68
-
69
- /**
70
- * How one database build of a runtime ended ({@link StrandInstanceManager.buildStrandDatabase}):
71
- * published or held by the first-sync gate; failed only because no seed peer has answered yet;
72
- * or overtaken by a release of the runtime it was building on.
73
- */
74
- type StrandDatabaseBuild = 'built' | 'awaiting-seed' | 'released';
75
-
76
- const timing = debug('sereus:cadre:timing');
77
-
78
- /** One caller blocked in {@link StrandInstanceManager.whenWritable}. */
79
- interface WritableWaiter {
80
- resolve: () => void;
81
- reject: (error: Error) => void;
82
- }
83
-
84
- /**
85
- * Whether a tracked instance is launched but waiting for its first sync: the runtime is
86
- * up (`libp2pNode` set) while the database is still withheld (`database` unset). That
87
- * is the first-sync gate's state regardless of what `status` says — the hibernation
88
- * manager's idle timer can flip a long-gated joiner to `'idle'` without changing it.
89
- *
90
- * NOTE: `status` is therefore not the source of truth for "gated": a joiner that reaches
91
- * nobody for `idleTimeout` (5 min at the interactive hint) reads `'idle'`, then
92
- * `'hibernating'`, while still holding no database. Harmless today — every decision the
93
- * runtime makes goes through this predicate, {@link StrandInstanceManager.publishDatabase}
94
- * announces on the idle-flipped instance exactly as on a `'syncing'` one, and an app that
95
- * shows "waiting for the other member" on `'syncing'` merely sees "idle" instead after
96
- * five minutes. If an app ever needs the gated LABEL to survive the idle flip, make the
97
- * hibernation manager's idle transition preserve `'syncing'` rather than adding a flag.
98
- */
99
- export function isAwaitingFirstSync(instance: StrandInstance): boolean {
100
- return instance.libp2pNode !== undefined && instance.database === undefined;
101
- }
102
-
103
- /**
104
- * The status a LIVE instance (runtime rebuilt or woken) should report: `'active'` once its
105
- * database is published, `'syncing'` while the first-sync gate still withholds it. Every
106
- * site that would otherwise write `'active'` after a build or a wake goes through this,
107
- * so a gated joiner never reads as writable.
108
- */
109
- export function liveStrandStatus(instance: StrandInstance): 'active' | 'syncing' {
110
- return instance.database ? 'active' : 'syncing';
111
- }
112
-
113
- /**
114
- * Configuration for starting a strand instance
115
- */
116
- export interface StartStrandConfig {
117
- strandRow: StrandRow;
118
- /**
119
- * sApp configuration provided by the hosting application. Absent for a **storage
120
- * replica** — a strand this node stores and serves without the app installed
121
- * (`CadreNodeConfig.hostUnclaimedStrands`): no schema signature check, no `App`
122
- * schema, no `sAppInfo` on the instance, and the default latency hint. A replica is
123
- * always a joiner; `StrandDatabase` refuses to found one. An app that later claims the
124
- * strand upgrades the running replica in place ({@link StrandInstanceManager.attachSApp}).
125
- */
126
- sAppConfig?: SAppConfig;
127
- storage?: StorageConfig;
128
- network?: NetworkConfig;
129
- profile: NodeProfile;
130
- defaultLatencyHint: LatencyHint;
131
- privateKey?: PrivateKey;
132
- /** Cohort-derived discovery seed (multiaddr strings). Defaults to [] when omitted. */
133
- bootstrapNodes?: string[];
134
- /**
135
- * Require a valid author signature on the sApp schema before bring-up.
136
- * Defaults to true (fail closed) when omitted; set false only for dev/test
137
- * with unsigned demo schemas. Mirrors {@link CadreNodeConfig.requireSignedSchemas}.
138
- */
139
- requireSignedSchemas?: boolean;
140
- /**
141
- * Whether this node founds the strand (vs. joins it). Forwarded to the
142
- * StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
143
- * runs once at bring-up. Joiners leave this unset and write nothing. Callers
144
- * resolve it BEFORE calling `startStrand` (`CadreNode.launchStrand` derives it
145
- * from the row's `FounderOwnerKey` when no explicit flag is given — see
146
- * {@link StrandConfig.founder}); a founder request that arrives while the
147
- * instance is already tracked goes through {@link StrandInstanceManager.foundExistingStrand},
148
- * never through a repeat `startStrand`.
149
- */
150
- founder?: boolean;
151
- /**
152
- * THIS party's own strand membership private key (base64 protobuf) for a closed
153
- * strand — the identity the founder bootstrap derives `Member.Key` /
154
- * `Manager.MemberKey` from. Resolved by the caller (`CadreNode.launchStrand`: an
155
- * explicit attach-time key, else the control-layer `StrandPartyKey` row, healed by
156
- * mint-on-launch for the founding machine) and retained with the launch config, so a
157
- * hibernation wake rebuilds under the same identity. Deliberately NOT the strand
158
- * row's shared `MemberPrivateKey` (see `StrandDatabaseConfig.partyMemberPrivateKey`).
159
- * Absent for open strands, and for joiners with no persisted party key.
160
- */
161
- partyMemberPrivateKey?: string;
162
- /**
163
- * Number of nodes Optimystic is told this strand's replication cluster should
164
- * have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
165
- * forwards here: every node on the strand should use the same value, and it is
166
- * frozen when the strand's libp2p node is created. Defaults to
167
- * `DEFAULT_STRAND_CLUSTER_SIZE` (4); values below `MIN_CLUSTER_SIZE` (2) are
168
- * rejected by `resolveStrandClusterSize`. Strand-only — the
169
- * control network's breadth is the fixed `CONTROL_REPLICATION_BREADTH`.
170
- */
171
- clusterSize?: number;
172
- /**
173
- * Machines that SERVE THIS STRAND (this node included), for the strand node's
174
- * block-repair corroboration yardstick — see `resolveRepairYardstick`. It must be
175
- * an authenticated per-strand count: the machines that actually run a node for this
176
- * strand, not the machines that exist.
177
- *
178
- * **The party's enrolled-machine count is NOT this number and must never be passed
179
- * here.** A strand launches only on machines whose embedding app registered its sApp
180
- * config (`CadreNode.addStrand`), so a closed strand shared by two machines of a
181
- * three-machine party is served by two. Passing three over-declares, and
182
- * over-declaring is the unsafe direction: at a declaration of 3 or more Optimystic
183
- * pins the repair corroboration floor at two corroborating peers, which a cohort
184
- * that can only ever field one peer can never reach — so the strand can never repair
185
- * a block (`cluster-fetch:no-quorum`, surfacing as reads failing with `Missing
186
- * block`). That regression is why this field was renamed off "enrolledMachines":
187
- * `bug-strand-yardstick-counts-party-machines`.
188
- *
189
- * **Nothing feeds it in production today.** No authenticated per-strand serving count
190
- * exists yet, so `CadreNode` passes nothing and the strand node runs the frozen
191
- * `STRAND_CLUSTER_POLICY` — declaring no yardstick, which leaves the known,
192
- * upstream-tracked single-voter exposure
193
- * (`backlog/debt-read-repair-single-voter-corroboration`). Building the count is
194
- * `backlog/feat-strand-yardstick-from-serving-machines`; this field and its
195
- * threading are the seam it plugs into.
196
- *
197
- * Volatile when a source does exist: re-resolve it on every resume beside the cohort
198
- * seed, since machines join and leave a strand while it hibernates. Omitting it means
199
- * "this node does not know", which declares nothing.
200
- */
201
- servingMachines?: number;
202
- /**
203
- * Tuning for the strand peer-join block catch-up ({@link PeerJoinBackfill}),
204
- * forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
205
- * libp2p node connects to a peer this runtime has not yet caught up, every
206
- * block in the strand's own raw store is pushed to it. Runs only on strands
207
- * with per-strand storage; `{ enabled: false }` restores the pre-existing
208
- * no-backfill behaviour.
209
- */
210
- backfill?: PeerJoinBackfillConfig;
211
- /**
212
- * Whether this strand's libp2p node is built with Optimystic's change notifications,
213
- * forwarded from {@link CadreNodeConfig.strandReactivity} and resolved per strand by
214
- * `strandCohortTopicOption`. Retained with the launch config, so a hibernation wake
215
- * rebuilds the node with the same setting.
216
- */
217
- reactivity?: StrandReactivityConfig;
218
- /**
219
- * Tuning for the CLOSED-strand revoked-peer gate
220
- * ({@link StrandRevocationEnforcer}), forwarded from
221
- * {@link CadreNodeConfig.strandRevocationEnforcement}. Armed only for
222
- * `strandRow.Type === 'c'` (an open strand has no membership rows to derive a
223
- * deny set from); `{ enabled: false }` restores the pre-existing behaviour
224
- * (a removed party's peers keep being served).
225
- */
226
- revocationEnforcement?: StrandRevocationEnforcementConfig;
227
-
228
- /**
229
- * The staged formation membership invitation for THIS strand — the seam the
230
- * bring-up membership reconciler (`strand-membership-reconciler.ts`) reads to
231
- * redeem the party's `Strand.Member` seat, and clears once the invitation is
232
- * spent, burned, or dead. `CadreNode.launchStrand` wires it over its in-memory
233
- * `pendingMembershipInvites` map; read lazily per pass, so a re-formation that
234
- * replaces the entry between passes is picked up. Only consulted for closed
235
- * strands launched with a {@link partyMemberPrivateKey}; harmless otherwise.
236
- */
237
- pendingMembershipInvite?: PendingMembershipInviteSource;
238
-
239
- /**
240
- * Tuning for the CLOSED-strand membership reconciler, forwarded from
241
- * {@link CadreNodeConfig.strandMembershipReconciliation}. Armed only for
242
- * closed strands launched with a {@link partyMemberPrivateKey};
243
- * `{ enabled: false }` disarms the loop (test fixtures that hand-drive the
244
- * membership writers). When it names no `pollIntervalMs` the reconciler
245
- * mirrors {@link revocationEnforcement}'s cadence — as its IDLE cadence and
246
- * the cap of its unfinished-join retry ladder, not as a flat retry interval.
247
- */
248
- membershipReconciliation?: StrandMembershipReconciliationConfig;
249
-
250
- /**
251
- * Called when THIS node's own peer id turns up in the strand's revoked set —
252
- * this node's party was removed from the closed strand, or left it. Fires once
253
- * per removal (re-armed if the party is re-admitted; a resume also rebuilds the
254
- * enforcer and may legitimately re-fire), and only for closed strands with the
255
- * gate armed. `CadreNode`
256
- * wires it to its `strand:revoked` event; nothing is stopped or torn down on
257
- * this node's behalf — what to do about it is the app's call.
258
- */
259
- onSelfRevoked?: (strandId: string) => void;
260
-
261
- /**
262
- * Called when the membership reconciler reports a blocked re-join — this node holds a
263
- * staged membership invitation for the strand that its writes cannot redeem, which is
264
- * what a REMOVED party handed a fresh invitation hits (see "Reporting a blocked
265
- * re-join" in `strand-membership-reconciler.ts` for the two triggers, one of them a
266
- * suspicion rather than a verdict). At most once per re-arm of the loop — a further
267
- * invitation staged while the loop is still running does not reset the report — and
268
- * only for closed strands with a party key. `CadreNode` wires it to its `strand:rejoin-blocked`
269
- * event; nothing is stopped or torn down — a remaining manager admitting this party's
270
- * key directly is the remedy, and the loop finishes the join by itself once that lands.
271
- */
272
- onRejoinBlocked?: (strandId: string) => void;
273
-
274
- /**
275
- * Tuning for the JOINING machine's first-sync write gate (`strand-first-sync-gate.ts`),
276
- * forwarded from {@link CadreNodeConfig.strandFirstSync}: the Header probe cadence while
277
- * a non-founder launch is `'syncing'`, and the default budget {@link whenWritable} waits
278
- * before rejecting. Founders and machines that already hold the Header are never gated.
279
- */
280
- firstSync?: StrandFirstSyncConfig;
281
-
282
- /**
283
- * Called when a launch that came up `'syncing'` (see {@link firstSync}) becomes
284
- * writable: the strand's `Strand.Header` arrived from a peer — or a founder request
285
- * ran the bootstrap against the gated database — and `instance.database` is now set.
286
- * Never called for a launch that was writable when `startStrand`/`resumeStrand`
287
- * resolved. `CadreNode` wires it to its `strand:writable` event. Retained with the
288
- * launch config, so a hibernation resume that comes up gated announces too.
289
- */
290
- onWritable?: (strandId: string) => void;
291
-
292
- /**
293
- * Called synchronously inside every commit on the strand's database that wrote something,
294
- * with who holds it — the app's commits and cadre-core's own membership writes alike.
295
- * `CadreNode` wires it to its `strand:commit-durability` event. Retained with the launch
296
- * config and passed to every database built for the launch, so a hibernation wake's rebuilt
297
- * database reports too. A throw is logged and does not affect the commit.
298
- */
299
- onCommitDurability?: (commit: StrandCommitDurability) => void;
300
-
301
- /**
302
- * Called when a commit reported `followed` below full replication becomes held by every member
303
- * of its cohort, while the database that made it is still open. `CadreNode` wires it to its
304
- * `strand:commit-held` event. Retained and isolated like {@link onCommitDurability}.
305
- */
306
- onCommitHeld?: (event: StrandCommitHeld) => void;
307
-
308
- /**
309
- * Re-announce this strand's delegate peer id (`delegatePeerId` — the strand
310
- * node's own transport peer id) to ONE relay (`relayAddr`, a direct dial addr
311
- * with a trailing `/p2p/<relayPeerId>`), unthrottled. Awaited before every
312
- * relay-reservation RE-drive of that relay — never before the first attempt,
313
- * which the launch/resume seed pass has already announced for.
314
- *
315
- * Why: a party control node running the relay server admits the strand node on
316
- * an in-memory delegate grant that a relay restart drops, so a re-drive that did
317
- * not re-announce first would be denied at the relay's gate. `CadreNode` wires
318
- * its own control-mesh announce here; a throw or rejection is logged by the
319
- * supervisor and the re-drive still runs. Retained with the launch config, so a
320
- * hibernation wake's rebuilt supervisors carry it too.
321
- */
322
- announceDelegateToRelay?: (strandId: string, relayAddr: string, delegatePeerId: string) => Promise<void>;
323
-
324
- /**
325
- * The node-local strand network state (`strand-network-state.ts`): this strand's
326
- * slice of it is handed to db-p2p as `persistence`, so the strand node saves its
327
- * FRET routing table (with each peer's signed address record) on every connection
328
- * and re-imports it when it is next built. Absent ⇒ nothing is saved and the node
329
- * starts with an empty table. `CadreNode` passes its store. Retained with the launch
330
- * config, so a hibernation wake rebuilds the node over the same state.
331
- */
332
- networkState?: StrandNetworkStateStore;
333
- }
334
-
335
- /**
336
- * Volatile inputs re-resolved when resuming a quiesced strand — the ones that can
337
- * have moved since it last ran, and that the rebuilt libp2p node freezes again. Each
338
- * one omitted keeps the value the retained launch config already holds, so a resume
339
- * that passes nothing rebuilds the strand exactly as it last ran.
340
- */
341
- export interface ResumeStrandOverrides {
342
- /**
343
- * Freshly-resolved cohort discovery seed (multiaddr strings). Grows as peers are
344
- * learned since the strand first launched.
345
- */
346
- bootstrapNodes?: string[];
347
- /**
348
- * Freshly-read count of the machines serving this strand, for the repair yardstick
349
- * (see {@link StartStrandConfig.servingMachines} — including why the party's
350
- * enrolled-machine count is not it). Moves whenever a machine starts or stops
351
- * serving the strand, which a hibernating strand does not otherwise notice. Nothing
352
- * passes it today. Omitting it here does NOT mean "unknown" the way omitting it from
353
- * {@link StartStrandConfig} does — it retains the last value; see `resumeStrand`.
354
- */
355
- servingMachines?: number;
356
- }
357
-
358
- /** A {@link StrandInstanceManager.resumeStrand} or {@link StrandInstanceManager.quiesceStrand} of one strand. */
359
- type RuntimeTransition =
360
- | { kind: 'resume'; operation: Promise<StrandInstance> }
361
- | { kind: 'quiesce'; operation: Promise<void> };
362
-
363
- /**
364
- * Get the isolated storage path for a specific strand.
365
- *
366
- * @deprecated This helper is Node-only and throws in React Native (it assumes a
367
- * filesystem layout). Use a storage provider factory function instead, which
368
- * receives the strandId and can create strand-specific storage paths using
369
- * platform-appropriate methods.
370
- *
371
- * @example
372
- * // Instead of using getStrandStoragePath, use a storage provider factory:
373
- * const storage = {
374
- * provider: (strandId: string) => new FileRawStorage(`./data/strands/${strandId}`)
375
- * };
376
- */
377
- export function getStrandStoragePath(basePath: string, strandId: string): string {
378
- // Check if we're in a Node.js environment
379
- if (typeof process === 'undefined' || !process.versions?.node) {
380
- throw new Error(
381
- 'getStrandStoragePath is not available in React Native. ' +
382
- 'Use a storage provider factory function instead.'
383
- );
384
- }
385
-
386
- // Sanitize strandId for filesystem safety (UUIDs should be safe, but just in case)
387
- const safeId = strandId.replace(/[^a-zA-Z0-9-]/g, '_');
388
-
389
- // Build the path with plain string joins rather than the Node `path` module.
390
- // A static `require('path')` forces RN bundlers (e.g. Metro) to *resolve* the
391
- // module at bundle time even though this Node-only helper throws above before
392
- // ever reaching here — joining by hand keeps the module free of any Node
393
- // built-in reference, so RN bundles need no `path` shim.
394
- const trimmedBase = basePath.replace(/[\\/]+$/, '');
395
- return `${trimmedBase}/strands/${safeId}`;
396
- }
397
-
398
- /**
399
- * Resolve a storage provider for a specific strand.
400
- * If the provider is a factory function, call it with the strandId.
401
- *
402
- * Called only from {@link StrandInstanceManager.startStrand}, which owns the result
403
- * for the instance's lifetime — see `strandStorages`, and which has already asserted
404
- * the id is usable as a scope key. Do not move that assertion here: this function
405
- * returns early when no provider is configured, and the id becomes a libp2p protocol
406
- * prefix on that path too.
407
- *
408
- * @param provider - Storage provider (instance or factory)
409
- * @param strandId - The strand ID to create storage for
410
- * @returns The resolved IRawStorage instance, or undefined if no provider
411
- */
412
- function resolveStrandStorage(
413
- provider: RawStorageProvider | undefined,
414
- strandId: string
415
- ): IRawStorage | undefined {
416
- if (!provider) {
417
- return undefined;
418
- }
419
- const storage = typeof provider === 'function' ? provider(strandId) : provider;
420
- // Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's cached-storage.ts).
421
- // Called ONCE per strand launch — `startStrand` keeps the result for the instance's
422
- // lifetime — so the wrap survives every runtime rebuild the instance goes through.
423
- return wrapStorageWithCache(storage, strandId);
424
- }
425
-
426
- /**
427
- * Verify an sApp's schema signature (fail-closed unless `requireSignedSchemas` is
428
- * false) and project the config onto the instance's `sAppInfo`.
429
- */
430
- function verifiedSAppInfo(strandId: string, sAppConfig: SAppConfig, requireSignedSchemas: boolean | undefined): SAppInfo {
431
- assertSchemaSignature(sAppConfig, { requireSignature: requireSignedSchemas ?? true });
432
- log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
433
- return {
434
- id: sAppConfig.id,
435
- version: sAppConfig.version,
436
- schema: sAppConfig.schema,
437
- signature: sAppConfig.signature
438
- };
439
- }
440
-
441
- /** Log wording for what a launch runs: the sApp and its version, or a storage replica. */
442
- function describeSApp(sAppConfig: SAppConfig | undefined): string {
443
- return sAppConfig ? `sApp: ${sAppConfig.id} v${sAppConfig.version}` : 'storage replica';
444
- }
445
-
446
- /**
447
- * Manages individual strand instances - creates and destroys isolated libp2p nodes
448
- * for each strand the cadre participates in.
449
- */
450
- export class StrandInstanceManager {
451
- private instances: Map<string, StrandInstance> = new Map();
452
- /**
453
- * Retained launch config per strand, captured in `startStrand` and cleared in
454
- * `stopStrand`. `resumeStrand` reuses it to rebuild a quiesced strand's runtime
455
- * without the caller re-threading storage/network/profile/key/sApp config.
456
- */
457
- private launchConfigs: Map<string, StartStrandConfig> = new Map();
458
- /**
459
- * The per-strand peer-join block catch-up, keyed by strand id. Private (not on
460
- * the public {@link StrandInstance}) because it is runtime plumbing with the
461
- * same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
462
- * stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
463
- * with a fresh caught-up-peer memo, which is intended (a resumed node may have
464
- * missed writes).
465
- */
466
- private backfills: Map<string, PeerJoinBackfill> = new Map();
467
- /**
468
- * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
469
- * id. Same lifecycle rationale as {@link backfills}: created in
470
- * `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so quiesce →
471
- * resume rebuilds it with a fresh (initially empty, fail-open) snapshot.
472
- */
473
- private revocationEnforcers: Map<string, StrandRevocationEnforcer> = new Map();
474
- /**
475
- * The per-strand membership reconciler (closed strands launched with a party
476
- * key only), keyed by strand id. Same lifecycle rationale as {@link backfills}:
477
- * created in `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so
478
- * quiesce → resume rebuilds it and re-runs the idempotent join ladder (redeem a
479
- * staged invitation, write this machine's own `MemberPeer` binding) from scratch.
480
- */
481
- private membershipReconcilers: Map<string, StrandMembershipReconciler> = new Map();
482
- /**
483
- * The per-strand first-sync gate (`strand-first-sync-gate.ts`), keyed by strand id:
484
- * holds a launch's `StrandDatabase` while it is NOT yet published on the instance.
485
- * Present from the moment the database object exists in `buildStrandRuntime` until
486
- * {@link publishDatabase} hands it to the app — so it doubles as the rollback handle
487
- * for a failed `initialize()` — and, for a joiner that holds no `Strand.Header` yet,
488
- * for as long as the launch stays `'syncing'`. Same lifecycle as {@link backfills}:
489
- * stopped (its database closed) and dropped in `releaseRuntime`, so a quiesce →
490
- * resume rebuild re-probes over the same store. An entry and `instance.database`
491
- * are mutually exclusive.
492
- */
493
- private firstSyncGates: Map<string, StrandFirstSyncGate> = new Map();
494
- /**
495
- * Callers blocked in {@link whenWritable}, keyed by strand id. Resolved by
496
- * {@link publishDatabase}, rejected by {@link stopStrand}; each waiter also carries
497
- * its own timeout. Deliberately NOT cleared by `releaseRuntime`: a waiter outlives a
498
- * quiesce → resume cycle, since the rebuild is what may finally publish the database.
499
- */
500
- private writableWaiters: Map<string, Set<WritableWaiter>> = new Map();
501
- /**
502
- * The per-strand relay-reservation supervisors — ONE PER CONFIGURED RELAY, each
503
- * over the node's own bare `/p2p-circuit` listener for that relay
504
- * (`strand-network-config.ts`) — keyed by strand id. Same lifecycle rationale as
505
- * {@link backfills}: started in `buildStrandRuntime` right after the libp2p node
506
- * exists, stopped FIRST and dropped in `releaseRuntime` (before the database
507
- * closes and the node stops, so no re-drive dials a node being torn down), so
508
- * quiesce → resume rebuilds them over the new node. What makes a strand node's
509
- * lost relay slot come back on its own — the control node has the same loop in
510
- * `CadreNode.reserveRelays`.
511
- */
512
- private relaySupervisors: Map<string, RelayReservationSupervisor[]> = new Map();
513
- /**
514
- * The per-strand redial of the strand's other members (`strand-peer-redial.ts`), keyed by
515
- * strand id. Same lifecycle as {@link relaySupervisors}: started in `buildStrandRuntime` once
516
- * the libp2p node exists, stopped first in `releaseRuntime`, so a hibernating strand dials
517
- * nobody and a resume redials over the new node.
518
- */
519
- private peerRedials: Map<string, StrandPeerRedial> = new Map();
520
- /**
521
- * The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
522
- * store, resolved once in `startStrand` and held until `stopStrand` disposes it.
523
- * Private for the same reason `backfills` is: runtime plumbing, not part of the
524
- * public {@link StrandInstance}.
525
- *
526
- * Deliberately NOT touched by `releaseRuntime`: that is what lets a quiesce →
527
- * resume cycle rebuild the libp2p node over the SAME store, keeping its
528
- * write-through cache warm (and, on an in-memory backend, keeping the strand's
529
- * blocks at all). An entry exists iff `instances` does AND the launch config
530
- * supplied a storage provider.
531
- *
532
- * NOTE: a hibernating strand therefore keeps its store — and its share of the
533
- * process-wide cache pool — resident for as long as the instance is tracked. That
534
- * is the point (a warm wake), and the pool evicts under pressure, so the cost is
535
- * one map entry per hibernating strand today. If a device ever hibernates strands
536
- * by the hundred, revisit: dropping the store at quiesce and paying for a cold
537
- * wake becomes the better trade.
538
- */
539
- private strandStorages: Map<string, IRawStorage> = new Map();
540
- /**
541
- * The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
542
- * `resumeStrand`, or a deferred join's database retry), present until it settles. The
543
- * instance is tracked for the whole build, with its database not yet constructed or not yet
544
- * initialized, so a caller that must act on the FINISHED runtime ({@link attachSApp},
545
- * {@link whenRuntimeBuilt}) waits on this instead. A failed build's rejection belongs to the
546
- * call that started it.
547
- */
548
- private runtimeBuilds: Map<string, Promise<unknown>> = new Map();
549
- /**
550
- * The latest resume or quiesce requested per strand id, present until it settles. Each one
551
- * starts only once the one requested before it has settled, so a wake that lands during a
552
- * quiesce rebuilds after it, and a quiesce requested during a rebuild releases what that
553
- * rebuild finished. The WHOLE operation, not its entry in {@link runtimeBuilds}: that settles
554
- * before a resume's own catch records `'error'`, so a joiner awaiting it could read a stale
555
- * status.
556
- */
557
- private transitions: Map<string, RuntimeTransition> = new Map();
558
- /**
559
- * The pending database retry per strand id of a launch whose database could not be built
560
- * because no seed peer had answered yet ({@link deferUnreachableJoin}). Cleared by
561
- * {@link releaseRuntime}, so a stop or a quiesce cancels it.
562
- */
563
- private unreachableJoinRetries: Map<string, unknown> = new Map();
564
- private stopping = false;
565
-
566
- constructor() {
567
- log('StrandInstanceManager created');
568
- }
569
-
570
- /**
571
- * Get all current strand instances
572
- */
573
- getInstances(): Map<string, StrandInstance> {
574
- return new Map(this.instances);
575
- }
576
-
577
- /**
578
- * Get a specific strand instance
579
- */
580
- getInstance(strandId: string): StrandInstance | undefined {
581
- return this.instances.get(strandId);
582
- }
583
-
584
- /**
585
- * Check if a strand is currently running
586
- */
587
- hasStrand(strandId: string): boolean {
588
- return this.instances.has(strandId);
589
- }
590
-
591
- /**
592
- * Start a new strand instance.
593
- *
594
- * A failed launch leaves NOTHING tracked: the instance and its retained launch
595
- * config are both dropped before the error is rethrown, so the strand id is
596
- * free for a genuine retry. This matches the pre-registration failure path
597
- * (a rejected schema signature, which throws before anything is recorded) —
598
- * both failure modes of this call leave the same residue: none. Callers learn
599
- * of the failure from the rejected promise (and, on the control-discovered
600
- * path, from CadreNode's `strand:error` event), not from an error record left
601
- * behind in `instances`.
602
- *
603
- * The strand's raw storage is resolved HERE, once, and held for the instance's
604
- * lifetime (see `strandStorages`) — `buildStrandRuntime` only reads it, so a
605
- * hibernation wake never re-enters the embedder's provider.
606
- */
607
- async startStrand(config: StartStrandConfig): Promise<StrandInstance> {
608
- const { strandRow, sAppConfig } = config;
609
- const strandId = strandRow.Id;
610
-
611
- if (this.stopping) {
612
- throw new Error('StrandInstanceManager is stopping');
613
- }
614
-
615
- if (this.instances.has(strandId)) {
616
- // Callers resolve founder-ness BEFORE reaching here (CadreNode.launchStrand) and
617
- // honor a founder request on a tracked instance via foundExistingStrand — so a
618
- // founder flag arriving at this early return against a non-founder retained
619
- // config is a dropped bootstrap, and must never again be silent.
620
- if (config.founder === true && this.launchConfigs.get(strandId)?.founder !== true) {
621
- log('startStrand: strand %s is already running but was NOT launched as founder — ' +
622
- 'this early return DROPS the founder request; use foundExistingStrand', strandId);
623
- }
624
- log('Strand %s already running', strandId);
625
- return this.instances.get(strandId)!;
626
- }
627
-
628
- log('Starting strand instance: %s (%s)', strandId, describeSApp(sAppConfig));
629
- const tTotal = performance.now();
630
-
631
- // The id becomes two names below: the storage scope key the embedder's provider
632
- // turns into a directory or database name, and `networkName` in buildStrandRuntime,
633
- // from which the libp2p protocol prefix `/optimystic/strand-<id>` is built. A
634
- // replicated strand row carries whatever id the founding node wrote, so neither is
635
- // safe unchecked. Asserted HERE rather than in `resolveStrandStorage` because that
636
- // function returns early when no provider is configured — a node with no storage
637
- // still reaches buildStrandRuntime and still mints the protocol prefix.
638
- assertStrandScopeKey(strandId);
639
-
640
- const sAppInfo = sAppConfig ? verifiedSAppInfo(strandId, sAppConfig, config.requireSignedSchemas) : undefined;
641
-
642
- // Resolve this strand's storage ONCE, before anything is recorded. If a factory
643
- // function is provided, it is called with the strandId to create strand-specific
644
- // storage (e.g. strand-isolated directories). A provider that throws therefore
645
- // fails the launch alongside the schema-signature check, leaving nothing tracked.
646
- const strandStorage = resolveStrandStorage(config.storage?.provider, strandId);
647
- if (strandStorage) {
648
- log('Strand %s using provided storage provider', strandId);
649
- }
650
-
651
- // Determine latency hint: sApp config > default
652
- const latencyHint = sAppConfig?.latencyHint ?? config.defaultLatencyHint;
653
-
654
- const instance: StrandInstance = {
655
- strandId,
656
- status: 'starting',
657
- sAppInfo,
658
- memberPrivateKey: strandRow.MemberPrivateKey ?? undefined,
659
- connectedPeers: 0,
660
- lastActivity: new Date(),
661
- latencyHint
662
- };
663
-
664
- this.instances.set(strandId, instance);
665
- this.launchConfigs.set(strandId, config);
666
- if (strandStorage) {
667
- this.strandStorages.set(strandId, strandStorage);
668
- }
669
-
670
- try {
671
- await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, config));
672
- timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
673
- log('Strand %s started successfully (%s)', strandId, describeSApp(sAppConfig));
674
- return instance;
675
- } catch (error) {
676
- // Status/error first — the (now discarded) record is still what `log` reports on.
677
- instance.status = 'error';
678
- instance.error = error instanceof Error ? error.message : String(error);
679
- log('Failed to start strand %s: %s', strandId, instance.error);
680
- // Drop the dead record so this strand id can be launched again. Keep the
681
- // `launchConfigs` has an entry iff `instances` does invariant — resumeStrand
682
- // reads both, and a config without an instance would strand the config.
683
- this.instances.delete(strandId);
684
- this.launchConfigs.delete(strandId);
685
- // The store this launch resolved goes with it: nothing owns it any more, and a
686
- // retained cache wrapper would be handed back (already retired) on a retry.
687
- await this.disposeStrandStorage(strandId);
688
- throw error;
689
- }
690
- }
691
-
692
- /** Hold `build` in {@link runtimeBuilds} until it settles, and settle as it does. */
693
- private async trackRuntimeBuild<T>(strandId: string, build: Promise<T>): Promise<T> {
694
- this.runtimeBuilds.set(strandId, build);
695
- try {
696
- return await build;
697
- } finally {
698
- if (this.runtimeBuilds.get(strandId) === build) {
699
- this.runtimeBuilds.delete(strandId);
700
- }
701
- }
702
- }
703
-
704
- /**
705
- * Wait until no runtime build of `strandId` is in flight. Never rejects: a failed build
706
- * is reported by the call that started it, and the caller re-reads the instance after.
707
- *
708
- * NOTE: after a failed launch the re-read sees the instance gone only because
709
- * `startStrand`'s catch deletes it synchronously, and its continuation is queued before
710
- * this waiter's. If that cleanup ever awaits before `instances.delete`, a waiter can act
711
- * on the dying record; move the cleanup inside the tracked promise then.
712
- */
713
- private async settleRuntimeBuilds(strandId: string): Promise<void> {
714
- for (let build = this.runtimeBuilds.get(strandId); build; build = this.runtimeBuilds.get(strandId)) {
715
- await build.catch((error: unknown) => {
716
- log('Strand %s: the runtime build waited on failed (reported by its launcher): %o', strandId, error);
717
- });
718
- }
719
- }
720
-
721
- /**
722
- * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
723
- * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
724
- * launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
725
- * volatile inputs (bootstrapNodes, servingMachines, network, profile,
726
- * privateKey, sApp config) from `config`, so the caller controls the
727
- * cohort-derived values. Storage is the
728
- * one input it does NOT re-read from `config`: that belongs to the instance and
729
- * comes from `strandStorages`.
730
- *
731
- * A database that cannot be built because no seed peer has answered yet does not fail the
732
- * build: the runtime comes up `'syncing'` without it and the database phase alone is
733
- * retried on the same node ({@link deferUnreachableJoin}).
734
- */
735
- private async buildStrandRuntime(instance: StrandInstance, config: StartStrandConfig): Promise<void> {
736
- const strandId = instance.strandId;
737
-
738
- // Refuse a bad `linkRoundTripMs` before any strand bring-up, for the reason
739
- // `CadreNode.start()` does: every budget derived from it here is behind a condition
740
- // (backfill disabled, no relay addrs), so a zero or NaN declaration would otherwise
741
- // surface later inside a path that logs and carries on. `link-budget.ts`.
742
- resolveLinkRoundTripMs(config.network?.linkRoundTripMs);
743
-
744
- // The store the instance OWNS (resolved once in `startStrand`), not a fresh
745
- // resolution: a rebuild must reach the same backend through the same warm cache.
746
- const strandStorage = this.strandStorages.get(strandId);
747
-
748
- // db-p2p namespaces every one of the node's protocol ids by network name
749
- // (`/optimystic/<networkName>/...`), so anything dialing this node's own
750
- // services must derive its prefix from the SAME string the node was built
751
- // with — hence one binding for both, not two literals that can drift.
752
- const networkName = `strand-${strandId}`;
753
- const protocolPrefix = `/optimystic/${networkName}`;
754
-
755
- // Whether this node runs the circuit-relay server, and its init — the same
756
- // resolution the control node takes from the same config (`relay-server.ts`).
757
- const relayServer = resolveRelayServer(config.network, config.profile);
758
- // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
759
- // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
760
- // bind one port), the announce config is dropped (it names the control node's
761
- // address), and every configured relay becomes one bare `/p2p-circuit` SEARCH
762
- // listener plus a dial addr in `relayAddrs` for the per-relay supervisor started
763
- // below — the same route the control node takes, because the configured
764
- // `<relay>/p2p-circuit` shape loses its address on a relay restart, a hangup, or
765
- // libp2p's own reservation refresh and recovers from none of them. See
766
- // `strand-network-config.ts` for all three, and `relay-reservation.ts`.
767
- //
768
- // `relayAddrs` is destructured OFF: it is not a `createLibp2pNode` option, and
769
- // spreading it would hand db-p2p a key it does not know.
770
- const { relayAddrs: relayDialAddrs = [], ...addrOptions } = strandNodeAddrs(config.network);
771
-
772
- // The CLOSED-strand revoked-peer gate (see strand-revocation-enforcer.ts).
773
- // Constructed BEFORE the libp2p node because the node's options embed its
774
- // predicates; its deny set starts empty (admit everything — fail-open, same
775
- // posture as bring-up everywhere here) and is first populated after the
776
- // strand database initializes below. Registered in the map immediately so
777
- // the failure rollback (releaseRuntime) tears it down like every other
778
- // runtime component. Open strands skip it entirely: their Member/MemberPeer
779
- // tables are empty by schema (`OnlyClosed`), so there is nothing to derive
780
- // a deny set from — arming would be pointless polling.
781
- const revocationEnforcer =
782
- config.strandRow.Type === 'c' && config.revocationEnforcement?.enabled !== false
783
- ? new StrandRevocationEnforcer({
784
- label: strandId,
785
- readRows: () => {
786
- const database = instance.database;
787
- if (!database) {
788
- throw new Error(`strand ${strandId} has no live database`);
789
- }
790
- return readStrandRevocationRows(database.getDatabase());
791
- },
792
- // Read per sweep, never captured: this closure is built BEFORE the
793
- // libp2p node exists (its options embed the enforcer's predicates),
794
- // and `releaseRuntime` clears the field again on quiesce. Undefined
795
- // therefore means "no transport right now", which the enforcer
796
- // treats as "nothing to tear down".
797
- getNetwork: () => instance.libp2pNode,
798
- onSelfRevoked: () => config.onSelfRevoked?.(strandId)
799
- }, config.revocationEnforcement)
800
- : undefined;
801
- if (revocationEnforcer) {
802
- this.revocationEnforcers.set(strandId, revocationEnforcer);
803
- }
804
-
805
- // Closed strands compose revoked-peer denial onto the caller's gater and arm
806
- // the fail-closed per-stream gate; open strands keep the raw configured gater
807
- // (their peers are legitimately cross-party and nothing is ever revoked).
808
- const revocationGateOptions = revocationEnforcer
809
- ? {
810
- connectionGater: createRevocationConnectionGater(revocationEnforcer, config.network?.connectionGater),
811
- authorizeInboundStream: (remotePeerId: string, protocol: string) =>
812
- revocationEnforcer.authorizeStream(remotePeerId, protocol)
813
- }
814
- : config.network?.connectionGater
815
- ? { connectionGater: config.network.connectionGater }
816
- : {};
817
-
818
- const reactivityOption = strandCohortTopicOption(config.reactivity, strandId);
819
- if (reactivityOption.cohortTopic) {
820
- log('strand %s: change notifications on', strandId);
821
- }
822
-
823
- try {
824
- // Bound once: the breadth is also the ceiling on the repair yardstick below, and the
825
- // two must be derived from the same resolution. Inside the `try` deliberately — a
826
- // rejected clusterSize is a build failure that runs the same cleanup as any other.
827
- const strandClusterSize = resolveStrandClusterSize(config.clusterSize);
828
-
829
- let t0 = performance.now();
830
- const node = await createLibp2pNode({
831
- port: 0, // Random port
832
- bootstrapNodes: config.bootstrapNodes ?? [],
833
- networkName,
834
- storage: strandStorage,
835
- fretProfile: config.profile === 'storage' ? 'core' : 'edge',
836
- relay: relayServer.enabled,
837
- // NOTE: unlike the control node, a strand node has no unauthorized-reservation budget,
838
- // so with the per-connection cap off any peer that reaches this node can hold one of
839
- // `maxReservations` slots and forward uncapped through it. If strand relays are ever
840
- // abused for bandwidth, give strand nodes a budget or a capped init of their own.
841
- ...(relayServer.enabled && { relayServerInit: relayServer.init }),
842
- clusterSize: strandClusterSize,
843
- // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
844
- // breadth reasoning, and the shape match with the control policy is a coincidence.
845
- //
846
- // The builder declares this node's block-repair corroboration yardstick from the
847
- // count of machines SERVING this strand, capped at the breadth above (a block
848
- // never lives on more machines than the cohort is wide). Given no count — the
849
- // production path today, since no per-strand serving count exists yet — it
850
- // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
851
- // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
852
- // serving set that changed while the strand slept.
853
- // The read deadline is settled by the same helper the control node uses, because the
854
- // two networks ride one link: the host's own, else derived from its declared
855
- // `linkRoundTripMs`, else nothing, and the base policy's COHORT_READ_DEADLINE_MS stands.
856
- clusterPolicy: strandClusterPolicy(strandClusterSize, {
857
- servingMachines: config.servingMachines,
858
- cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(config.network)
859
- }),
860
- arachnode: {
861
- enableRingZulu: config.profile === 'storage'
862
- },
863
- ...(config.privateKey && { privateKey: config.privateKey }),
864
- // Built per runtime, not per launch: the adapter stops saving once the strand's
865
- // state is forgotten, and a rebuilt runtime is what starts it saving again.
866
- //
867
- // NOTE: the saved state also restores which peers db-p2p saw serving the strand,
868
- // and it counts them in the cohort until FRET finds them unreachable. A node built
869
- // while every other member is offline therefore fails its first writes ("Failed to
870
- // get super-majority") — for 4-6 s on loopback, the window a running node already
871
- // sees when a member drops. If apps report failed sends after relaunching alone on
872
- // a slow relayed link, retry the write in the app or raise it upstream (db-p2p's
873
- // remembered-serving fallback); dropping `persistence` brings back the split
874
- // restart.
875
- ...(config.networkState && { persistence: strandNetworkStatePersistence(config.networkState, strandId) }),
876
- // With relays configured, built with libp2p's relay discovery off (given a transport
877
- // list to wrap — `node-transports.ts`), so the supervisors below are the only thing
878
- // that fills the bare `/p2p-circuit` slots: otherwise a cohort seed running the relay
879
- // server takes one as soon as it connects (`relay-reservation.ts`, "WHY LIBP2P'S
880
- // RELAY DISCOVERY IS SWITCHED OFF").
881
- ...nodeTransportsOption(config.network, addrOptions, relayDialAddrs.length > 0),
882
- ...(config.network?.noiseCrypto && { noiseCrypto: config.network.noiseCrypto }),
883
- // Unconditional, and the same default the control node takes: every node of
884
- // every party runs the monitor over its connection to a slow phone, so the
885
- // widened ping deadline has to reach the strand nodes too (see
886
- // DEFAULT_CONNECTION_MONITOR).
887
- connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
888
- // The same declared link the control node states, and for the same reason: a strand
889
- // node is also the listener for every other member's strand node, and Optimystic
890
- // derives that listener's limit from this (`link-budget.ts`).
891
- linkRoundTripMs: resolveLinkRoundTripMs(config.network?.linkRoundTripMs),
892
- // The same dial limits as the control node: a strand node dials the same machines,
893
- // through the same relays and gates.
894
- ...optimysticDialLimits(config.network?.linkRoundTripMs),
895
- // Listen entries plus the WebSocket transport switch they imply — a strand node
896
- // announces nothing the operator configured (`strand-network-config.ts`), and
897
- // spreads AFTER `transports` above because the switch is a no-op whenever a
898
- // transport list was passed. The bare `/p2p-circuit` entries (one per relay) open
899
- // no connection here; each registers a pending reservation that the matching
900
- // supervisor below fills. That is what gives a NAT'd strand node a reachable relay
901
- // slot — and works against a party-run relay because the launch path announced
902
- // this strand's derived peerId to it first (delegate admission; see cadre-node.ts).
903
- ...addrOptions,
904
- // The raw configured gater (open strands), or the revocation-composed
905
- // gater plus the fail-closed per-stream revoked-peer gate (closed
906
- // strands) — resolved above, before the try.
907
- ...revocationGateOptions,
908
- // `{}` unless the operator enabled change notifications for this strand (`strand-reactivity.ts`).
909
- ...reactivityOption
910
- }) as Libp2pNodeWithRepo;
911
- timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
912
-
913
- instance.libp2pNode = node;
914
-
915
- // One reservation supervisor PER RELAY, started now so the first attempts
916
- // overlap the database bring-up below (the relay is never in a strand's
917
- // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
918
- // — so an open relay connection cannot disturb that bring-up). Their first
919
- // attempts are awaited before the strand goes `active`, below. Registered in the
920
- // map immediately so the failure rollback (releaseRuntime) stops them like every
921
- // other runtime component.
922
- this.relaySupervisors.set(strandId, this.startRelaySupervisors(strandId, node, relayDialAddrs, config));
923
- // Redials the strand's other members whenever none is connected, on the budget the
924
- // declared link gives a relayed dial — what reconnects two restarted phones on a slow
925
- // link, where FRET's own redial cannot (`strand-peer-redial.ts`). The relays are
926
- // excluded by peer id: the node is connected to them throughout, and they are no member.
927
- this.peerRedials.set(strandId, superviseStrandPeerRedial({
928
- label: strandId,
929
- node,
930
- seed: config.bootstrapNodes ?? [],
931
- excluded: new Set(groupAddrsByPeerId([...relayDialAddrs]).keys()),
932
- ...(revocationEnforcer && { isRevoked: (peerId: string) => revocationEnforcer.isRevoked(peerId) })
933
- }, strandRedialTimings(config.network?.linkRoundTripMs)));
934
-
935
- const databaseBuild = await this.buildStrandDatabase(instance, config, node);
936
- if (!this.isCurrentRuntime(instance, node)) {
937
- // A stop released this runtime during the database phase (`stopStrand` does not
938
- // wait for builds); arming the loops below would restart them on a stopped node.
939
- log('Strand %s was stopped while its runtime was being built', strandId);
940
- return;
941
- }
942
-
943
- // The membership reconciler: finish this party's join on every machine —
944
- // redeem a staged formation invitation if one is pending, then write the
945
- // durable machine→party binding (Strand.MemberPeer). Armed here, the one
946
- // seam where the strand's transport peer id exists and its database has been
947
- // attempted, for launch AND hibernation resume, for closed strands that carry the
948
- // party's own membership key. start() kicks an immediate pass WITHOUT
949
- // awaiting it — a joiner at this instant has not synced the founder's rows
950
- // and may lack write quorum, so bring-up is never blocked; an unfinished join
951
- // then retries on a short doubling ladder capped at the configured
952
- // pollIntervalMs (which is also the flat cadence while nobody has admitted this
953
- // party yet), and the loop stops once the member row and this machine's own
954
- // binding are both in place. Armed AFTER the database phase above, so a gated or
955
- // deferred launch always has a reconciler registered by the time it can publish.
956
- if (config.strandRow.Type === 'c' && config.partyMemberPrivateKey
957
- && config.membershipReconciliation?.enabled !== false) {
958
- const reconciler = new StrandMembershipReconciler({
959
- label: strandId,
960
- partyMemberPrivateKey: config.partyMemberPrivateKey,
961
- // Read per pass, never captured — same lifecycle argument as the
962
- // enforcer's getNetwork: quiesce drops both handles.
963
- getDatabase: () => instance.database?.getDatabase(),
964
- getOwnPeerId: () => instance.libp2pNode?.peerId.toString(),
965
- pendingInvite: config.pendingMembershipInvite,
966
- // The enforcer's CURRENT snapshot view of this node's own peer id — the
967
- // reconciler stops rather than fight a self-revocation. Absent when the
968
- // gate is disarmed (fail-open: the loop just runs).
969
- isSelfRevoked: revocationEnforcer
970
- ? () => {
971
- const ownPeerId = instance.libp2pNode?.peerId.toString();
972
- return ownPeerId !== undefined && revocationEnforcer.isRevoked(ownPeerId);
973
- }
974
- : undefined,
975
- onRejoinBlocked: () => config.onRejoinBlocked?.(strandId)
976
- }, {
977
- pollIntervalMs: config.membershipReconciliation?.pollIntervalMs
978
- ?? config.revocationEnforcement?.pollIntervalMs
979
- });
980
- this.membershipReconcilers.set(strandId, reconciler);
981
- reconciler.start();
982
- }
983
-
984
- // Arm the revoked-peer deny-set poll now that the database it reads
985
- // exists. start() kicks an immediate refresh WITHOUT awaiting it —
986
- // bring-up and resume are never blocked on a membership read; until the
987
- // first read lands the empty snapshot admits everyone (fail-open).
988
- revocationEnforcer?.start();
989
-
990
- // Peer-join block catch-up: push this strand's own blocks to each newly
991
- // connected peer, so a machine that joined after blocks were committed
992
- // still ends up physically holding them (without per-strand storage there
993
- // is nothing to copy). Armed for EVERY stored strand: `PeerJoinBackfill`
994
- // only does work when the strand's libp2p node reports a peer connection,
995
- // so on a device that is genuinely alone it is inert, and arming it at
996
- // launch is what closes the "founded alone, never replicates" hole — a
997
- // peer that joins later gets the founder's blocks without any relaunch.
998
- // No `authorizePeer` gate, deliberately — see the module comment in
999
- // peer-join-backfill.ts for the strand-side argument (and why the control
1000
- // network, which DOES gate, is different).
1001
- //
1002
- // NOTE: cost is one PeerJoinBackfill object + one `peer:identify` listener
1003
- // per running strand — linear in strand count, negligible at the handful a
1004
- // device or host runs today. If a node ever hosts strands by the hundred,
1005
- // move to one shared listener that dispatches by strand id.
1006
- if (strandStorage && config.backfill?.enabled !== false) {
1007
- if (node.keyNetwork) {
1008
- const backfill = new PeerJoinBackfill({
1009
- label: strandId,
1010
- libp2p: node,
1011
- peerNetwork: node.keyNetwork,
1012
- storage: strandStorage,
1013
- // The same prefix the receiver registered its block-transfer handler
1014
- // under — derived from networkName above, never re-spelled here.
1015
- protocolPrefix
1016
- }, {
1017
- // Dial and response deadlines counted in link round trips, not fixed milliseconds:
1018
- // a relayed dial costs a fixed number of exchanges, so a fixed budget has a link
1019
- // speed above which this catch-up can never reach the peer at all. See
1020
- // `link-budget.ts`. Spread BEFORE the host's own config so an explicit
1021
- // `strandBackfill.dialTimeoutMs` still wins.
1022
- ...peerJoinPushBudget(config.network?.linkRoundTripMs),
1023
- ...config.backfill
1024
- });
1025
- backfill.start();
1026
- this.backfills.set(strandId, backfill);
1027
- } else {
1028
- log('Strand %s: libp2p node exposes no keyNetwork; peer-join block catch-up is inert', strandId);
1029
- }
1030
- }
1031
-
1032
- // The relay supervisors' FIRST attempts, all of them, before `active`: the happy
1033
- // path still publishes its circuit addr before `addStrand` resolves (the
1034
- // strand-addr RPC answers read it, and the same-party circuit scenario asserts
1035
- // it). Fail-SOFT — a first attempt that lands nothing does not fail the launch.
1036
- // The strand's database is up, and the supervisor keeps trying on its backoff;
1037
- // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
1038
- //
1039
- // NOTE: a relay that is down costs this launch one full drive — the reservation budget
1040
- // counted from the declared link round trip, 18 s at its default (`link-budget.ts`) — and
1041
- // `StrandWatcher` launches strands one at a time, so N strands cost N of those in
1042
- // bring-up during a relay outage, and MORE on a host that declared a slower link. If that
1043
- // ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
1044
- // than shortening the drive.
1045
- t0 = performance.now();
1046
- await this.awaitFirstRelayAttempts(strandId);
1047
- timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
1048
-
1049
- // No `lastActivity` stamp: bringing a runtime up is not activity. A check-in marks
1050
- // activity before its resume and must see only what landed while the strand was up.
1051
- instance.status = liveStrandStatus(instance);
1052
- if (databaseBuild === 'awaiting-seed') {
1053
- // Armed only now that the build is over: a retry firing mid-build would find the
1054
- // build in flight.
1055
- this.deferUnreachableJoin(instance);
1056
- } else if (instance.status === 'syncing') {
1057
- log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
1058
- 'a member of the strand is reached', strandId);
1059
- }
1060
- } catch (error) {
1061
- // Roll back any partially-attached runtime so the instance is left with
1062
- // NEITHER handle. Otherwise the `libp2pNode || database` "already live"
1063
- // guard in resumeStrand/handleStrandWake would treat a half-built strand
1064
- // as healthy — leaking the libp2p node and never retrying the rebuild.
1065
- await this.releaseRuntime(instance).catch((cleanupErr) => {
1066
- log('buildStrandRuntime cleanup for strand %s also failed: %o', strandId, cleanupErr);
1067
- });
1068
- throw error;
1069
- }
1070
- }
1071
-
1072
- /**
1073
- * The database phase of a runtime build, on the libp2p node already attached: create the
1074
- * StrandDatabase, initialize it, then publish it or arm its first-sync gate. Run by
1075
- * {@link buildStrandRuntime} and again by each {@link retryUnreachableJoin}, with a new
1076
- * StrandDatabase per attempt.
1077
- *
1078
- * Returns `'awaiting-seed'` when initialize failed only because no seed peer has answered
1079
- * (`cohort-unreachable`): that attempt's database is closed and the node keeps running. Any
1080
- * other failure throws, with the attempt's database left with its gate for
1081
- * {@link releaseRuntime} to close.
1082
- */
1083
- private async buildStrandDatabase(
1084
- instance: StrandInstance,
1085
- config: StartStrandConfig,
1086
- node: Libp2pNodeWithRepo
1087
- ): Promise<StrandDatabaseBuild> {
1088
- const strandId = instance.strandId;
1089
- // Held by the first-sync gate (registered before initialize) rather than attached
1090
- // to the instance: a failed init is then cleaned up through the gate (close() is safe on
1091
- // a partially-initialized db), and the app only ever sees `instance.database` once this
1092
- // machine is allowed to write to the strand.
1093
- let t0 = performance.now();
1094
- const strandDb = new StrandDatabase({
1095
- strandId,
1096
- sAppConfig: config.sAppConfig,
1097
- libp2pNode: node,
1098
- coordinatedRepo: node.coordinatedRepo,
1099
- // Founder bootstrap inputs: the strand's type drives which membership rows
1100
- // are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
1101
- // caller from the control-layer StrandPartyKey row) derives the founding
1102
- // Member/Manager key. The row's shared MemberPrivateKey rides along as the
1103
- // read-gating secret only — it derives nobody's identity.
1104
- strandType: config.strandRow.Type,
1105
- memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
1106
- partyMemberPrivateKey: config.partyMemberPrivateKey,
1107
- founder: config.founder,
1108
- onCommitDurability: config.onCommitDurability,
1109
- onCommitHeld: config.onCommitHeld
1110
- });
1111
- const gate = new StrandFirstSyncGate({
1112
- label: strandId,
1113
- database: strandDb,
1114
- onHeaderHeld: () => this.publishDatabase(instance, strandDb)
1115
- }, config.firstSync);
1116
- this.firstSyncGates.set(strandId, gate);
1117
- try {
1118
- await strandDb.initialize();
1119
- } catch (error) {
1120
- if (!isCohortUnreachableRead(error)) {
1121
- throw error;
1122
- }
1123
- await this.discardDatabaseAttempt(strandId, gate);
1124
- log('Strand %s: no seed peer has answered yet, so this joiner cannot build its database (%s); ' +
1125
- 'its runtime stays up and the build is retried every %dms',
1126
- strandId, error instanceof Error ? error.message : String(error), UNREACHABLE_JOIN_RETRY_MS);
1127
- return 'awaiting-seed';
1128
- }
1129
- timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
1130
- if (this.firstSyncGates.get(strandId) !== gate) {
1131
- // A stop released the runtime while initialize ran, closing this database through its
1132
- // gate part-way; close what initialize finished after that.
1133
- await strandDb.close();
1134
- return 'released';
1135
- }
1136
-
1137
- // The first-sync write gate. A founder just wrote the Header in its bootstrap; any
1138
- // other machine may commit only once it holds the Header — which it can only have
1139
- // received from a peer (now, or on an earlier run over the same store) — and has read
1140
- // each App table once, so their collections are fetched too. Publishing the database
1141
- // is what makes the strand writable to the app; until then the instance stays
1142
- // `'syncing'` and the gate re-probes on its cadence. Everything armed after the database
1143
- // phase reads `instance.database` lazily, so the reconciler and enforcer wait with it —
1144
- // and `publishDatabase` kicks the reconciler when the wait ends, so a joiner's membership
1145
- // rows land with the strand rather than one retry later.
1146
- t0 = performance.now();
1147
- const synced = config.founder === true || await strandFirstSyncComplete(strandDb.getDatabase(), strandId);
1148
- timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
1149
- if (this.firstSyncGates.get(strandId) !== gate) {
1150
- // A stop released the runtime during the probe and closed this database through its gate.
1151
- return 'released';
1152
- }
1153
- if (synced) {
1154
- this.publishDatabase(instance, strandDb);
1155
- } else {
1156
- gate.start();
1157
- }
1158
- return 'built';
1159
- }
1160
-
1161
- /** Close a database attempt that failed to initialize, and drop its gate if it is still the registered one. */
1162
- private async discardDatabaseAttempt(strandId: string, gate: StrandFirstSyncGate): Promise<void> {
1163
- gate.stop();
1164
- if (this.firstSyncGates.get(strandId) === gate) {
1165
- this.firstSyncGates.delete(strandId);
1166
- }
1167
- await gate.database.close();
1168
- }
1169
-
1170
- /**
1171
- * Keep a launch `'syncing'` when its database could not be built because no seed peer has
1172
- * answered yet, and retry the database build every {@link UNREACHABLE_JOIN_RETRY_MS} on the
1173
- * runtime already up.
1174
- *
1175
- * Optimystic (since 1.11.0) will not rule a block never-created while any of a node's
1176
- * `bootstrapNodes` is unheard, so a joiner alone with an unreachable seed cannot build even
1177
- * the schema catalog: `strandDb.initialize()` fails with `cohort-unreachable`. A founder
1178
- * given a seed is refused the same way (normally it has none: no sibling runs the strand yet). Refusing is
1179
- * right (building it alone would fork the catalog), but failing the launch would drop the
1180
- * strand until the watcher relaunched it, and `addStrand` would reject with a raw Optimystic
1181
- * error. Deferring gives a joiner the same contract the first-sync gate gives one that did
1182
- * build: the strand stays tracked, {@link whenWritable} waits and rejects with the retryable
1183
- * `StrandAwaitingFirstSyncError`, and the launch proceeds once a seed peer answers.
1184
- *
1185
- * The libp2p node, its connections and the background loops stay up meanwhile. A seed peer
1186
- * can answer just after the schema read gave up on it — on a slow link it connects and its
1187
- * peer-join backfill delivers the whole store — and the retry then builds the database, and
1188
- * runs the first-sync probe, while that peer is still connected.
1189
- */
1190
- private deferUnreachableJoin(instance: StrandInstance): void {
1191
- const strandId = instance.strandId;
1192
- const timer = setTimeout(() => {
1193
- this.unreachableJoinRetries.delete(strandId);
1194
- void this.retryUnreachableJoin(instance);
1195
- }, UNREACHABLE_JOIN_RETRY_MS);
1196
- (timer as { unref?: () => void }).unref?.();
1197
- this.unreachableJoinRetries.set(strandId, timer);
1198
- }
1199
-
1200
- /**
1201
- * One retry scheduled by {@link deferUnreachableJoin}: the database phase again, on the
1202
- * runtime still attached. Skipped when the strand was stopped or released since — a quiesce
1203
- * releases it and the resume after rebuilds the whole runtime — or already has a database.
1204
- * A failure other than an unanswered seed releases the runtime and marks the strand
1205
- * `'error'`, as a failed resume does.
1206
- */
1207
- private async retryUnreachableJoin(instance: StrandInstance): Promise<void> {
1208
- const strandId = instance.strandId;
1209
- const config = this.launchConfigs.get(strandId);
1210
- // Every strand node is built by `buildStrandRuntime`, as a node with its coordinated repo.
1211
- const node = instance.libp2pNode as Libp2pNodeWithRepo | undefined;
1212
- if (!config || !node || !this.isCurrentRuntime(instance, node) || instance.database
1213
- || this.firstSyncGates.has(strandId) || this.isQuiescing(strandId)) {
1214
- return;
1215
- }
1216
- let build: StrandDatabaseBuild;
1217
- try {
1218
- // Tracked like any runtime build, so a wake, a quiesce or an sApp attach waits for it
1219
- // rather than acting on a database that is still initializing.
1220
- build = await this.trackRuntimeBuild(strandId, this.buildStrandDatabase(instance, config, node));
1221
- } catch (error) {
1222
- if (!this.isCurrentRuntime(instance, node)) {
1223
- log('Strand %s: a database retry failed after its runtime was released: %o', strandId, error);
1224
- return;
1225
- }
1226
- await this.releaseRuntime(instance).catch((cleanupErr) => {
1227
- log('Strand %s: releasing the runtime after a failed database retry also failed: %o', strandId, cleanupErr);
1228
- });
1229
- instance.status = 'error';
1230
- instance.error = error instanceof Error ? error.message : String(error);
1231
- log('Strand %s: building the database of a joiner that was waiting for a seed peer failed: %s', strandId, instance.error);
1232
- return;
1233
- }
1234
- if (build === 'awaiting-seed' && this.isCurrentRuntime(instance, node) && !this.isQuiescing(strandId)) {
1235
- this.deferUnreachableJoin(instance);
1236
- }
1237
- }
1238
-
1239
- /** Whether `instance` is still tracked, with `node` still its runtime's libp2p node. */
1240
- private isCurrentRuntime(instance: StrandInstance, node: Libp2p): boolean {
1241
- return !this.stopping && this.instances.get(instance.strandId) === instance && instance.libp2pNode === node;
1242
- }
1243
-
1244
- /**
1245
- * Release an instance's strand-network runtime: stop the relay supervisors and
1246
- * the other background loops, close the StrandDatabase, then stop the libp2p
1247
- * node (construction order in reverse), clearing both fields and zeroing
1248
- * connectedPeers. Tolerant of partially-built state — either handle may be
1249
- * absent — so it doubles as rollback for a failed `buildStrandRuntime`. Shared
1250
- * by `quiesceStrand`, `stopStrand`, and that rollback path.
1251
- *
1252
- * Leaves `strandStorages` untouched by design — the store outlives the runtime it
1253
- * was built into, which is what makes a resume warm. Only `stopStrand` disposes it.
1254
- */
1255
- private async releaseRuntime(instance: StrandInstance): Promise<void> {
1256
- // Relay supervisors FIRST of all — before anything below is torn down — so no
1257
- // re-drive is scheduled against a node being stopped. `stop()` is synchronous
1258
- // and never awaits a drive, so a relay that is down cannot delay this teardown.
1259
- // `stop()` also cancels the drive already in flight, so an attempt started
1260
- // moments before this does not keep dialing and polling the node the lines
1261
- // below are stopping; its result is discarded either way.
1262
- const unreachableJoinRetry = this.unreachableJoinRetries.get(instance.strandId);
1263
- if (unreachableJoinRetry !== undefined) {
1264
- clearTimeout(unreachableJoinRetry as ReturnType<typeof setTimeout>);
1265
- this.unreachableJoinRetries.delete(instance.strandId);
1266
- }
1267
- const relaySupervisors = this.relaySupervisors.get(instance.strandId);
1268
- if (relaySupervisors) {
1269
- relaySupervisors.forEach((supervisor) => supervisor.stop());
1270
- this.relaySupervisors.delete(instance.strandId);
1271
- }
1272
- this.peerRedials.get(instance.strandId)?.stop();
1273
- this.peerRedials.delete(instance.strandId);
1274
- // Backfill next — before the database closes and the libp2p node stops — so
1275
- // no NEW catch-up push is issued against a torn-down transport. A push already
1276
- // in flight is not awaited; it fails into the module's own per-chunk catch.
1277
- const backfill = this.backfills.get(instance.strandId);
1278
- if (backfill) {
1279
- backfill.stop();
1280
- this.backfills.delete(instance.strandId);
1281
- }
1282
- // The revoked-peer enforcer goes with the runtime it gated: a resume
1283
- // rebuilds it with a fresh (initially empty, fail-open) snapshot.
1284
- const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
1285
- if (revocationEnforcer) {
1286
- revocationEnforcer.stop();
1287
- this.revocationEnforcers.delete(instance.strandId);
1288
- }
1289
- // The membership reconciler likewise: a resume rebuilds it and re-runs the
1290
- // idempotent ladder, which is what makes resume-after-partial-join heal.
1291
- const membershipReconciler = this.membershipReconcilers.get(instance.strandId);
1292
- if (membershipReconciler) {
1293
- membershipReconciler.stop();
1294
- this.membershipReconcilers.delete(instance.strandId);
1295
- }
1296
- // A still-gated database (a joiner that never received the Header, or a launch whose
1297
- // initialize() failed) is closed through its gate; a published one through the
1298
- // instance. Never both — publishing moves the handle from the gate to the instance.
1299
- const gate = this.firstSyncGates.get(instance.strandId);
1300
- if (gate) {
1301
- gate.stop();
1302
- this.firstSyncGates.delete(instance.strandId);
1303
- await gate.database.close();
1304
- }
1305
- if (instance.database) {
1306
- await instance.database.close();
1307
- instance.database = undefined;
1308
- }
1309
- if (instance.libp2pNode) {
1310
- await instance.libp2pNode.stop();
1311
- instance.libp2pNode = undefined;
1312
- }
1313
- instance.connectedPeers = 0;
1314
- }
1315
-
1316
- /**
1317
- * One {@link superviseRelayReservation} per relay dial addr, each over exactly
1318
- * that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
1319
- * one relay re-drives only that one. They are the only thing that fills the node's
1320
- * search slots once the node is built with libp2p's relay discovery off
1321
- * (`nodeTransportsOption`). Default retry timings — the control node's (2 s
1322
- * doubling to 60 s between failed attempts, a 5 s liveness check while held). Each DRIVE's
1323
- * own deadline is counted in link round trips at this host's declared
1324
- * `network.linkRoundTripMs` (`link-budget.ts`), so the same declaration that lengthens the
1325
- * control node's drive lengthens these.
1326
- *
1327
- * Each supervisor's `beforeRedrive` is the caller's
1328
- * {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
1329
- * node's peer id, so a re-drive after a relay restart is preceded by a fresh
1330
- * delegate grant on the relay about to be dialed. Returns `[]` when no relay is
1331
- * configured — the strand then simply has no circuit listener to fill.
1332
- */
1333
- private startRelaySupervisors(
1334
- strandId: string,
1335
- node: Libp2pNodeWithRepo,
1336
- relayDialAddrs: readonly string[],
1337
- config: StartStrandConfig
1338
- ): RelayReservationSupervisor[] {
1339
- if (relayDialAddrs.length === 0) {
1340
- return [];
1341
- }
1342
- const announce = config.announceDelegateToRelay;
1343
- const delegatePeerId = node.peerId.toString();
1344
- const timeoutMs = relayReservationBudgetMs(config.network?.linkRoundTripMs);
1345
- return relayDialAddrs.map((relayAddr) => superviseRelayReservation(node, [relayAddr], {
1346
- timeoutMs,
1347
- ...(announce && { beforeRedrive: () => announce(strandId, relayAddr, delegatePeerId) })
1348
- }));
1349
- }
1350
-
1351
- /**
1352
- * Wait for every relay supervisor's first attempt to settle, concurrently, and log
1353
- * the ones that landed nothing. Never throws — a strand whose relay is unreachable
1354
- * at launch still comes up, with its supervisors retrying in the background.
1355
- */
1356
- private async awaitFirstRelayAttempts(strandId: string): Promise<void> {
1357
- const supervisors = this.relaySupervisors.get(strandId) ?? [];
1358
- if (supervisors.length === 0) {
1359
- return;
1360
- }
1361
- await Promise.all(supervisors.map((supervisor) => supervisor.firstAttempt));
1362
- supervisors.forEach((supervisor) => {
1363
- if (supervisor.lastError !== null) {
1364
- log('Strand %s: relay reservation not held after the first attempt (retrying in the background): %s',
1365
- strandId, supervisor.lastError);
1366
- }
1367
- });
1368
- }
1369
-
1370
- /**
1371
- * Drop the strand's owned store and release this strand's claim on its cache. The
1372
- * wrapper counts holders, so the cache is emptied and unregistered from the shared
1373
- * pool only if no other scope still holds it. Called only where the strand's whole
1374
- * lifetime ends (`stopStrand`, and the failed-launch rollback) — never on a quiesce.
1375
- *
1376
- * A dispose failure is logged, not thrown: the store is already unreferenced here,
1377
- * and failing the stop over a cache-bookkeeping error would leave the caller unable
1378
- * to tear the strand down. `disposeStorageCache` no-ops for an unwrapped store
1379
- * (e.g. `MemoryRawStorage`), so no instanceof test is needed.
1380
- */
1381
- private async disposeStrandStorage(strandId: string): Promise<void> {
1382
- const storage = this.strandStorages.get(strandId);
1383
- if (!storage) {
1384
- return;
1385
- }
1386
- this.strandStorages.delete(strandId);
1387
- try {
1388
- await disposeStorageCache(storage);
1389
- } catch (error) {
1390
- log('Failed to dispose storage cache for strand %s: %o', strandId, error);
1391
- }
1392
- }
1393
-
1394
- /**
1395
- * Refresh a strand's revoked-peer deny set NOW — and, with it, run the
1396
- * teardown sweep that hangs up every connected revoked peer.
1397
- *
1398
- * The gate is otherwise poll-driven (default
1399
- * `DEFAULT_REVOCATION_POLL_INTERVAL_MS`), because neither a replicated
1400
- * revocation nor a local `revokeMember`/`leaveStrand` raises anything this
1401
- * runtime can hook. A caller that just wrote a revocation against the
1402
- * strand's database should follow it with this call, which makes the cut
1403
- * immediate instead of waiting out the interval.
1404
- *
1405
- * Quiet no-op — matching `quiesceStrand`'s posture — when the strand is not
1406
- * tracked, is quiesced, is open, or has the gate disabled: in every one of
1407
- * those cases there is no enforcer to refresh. Never rejects (the enforcer's
1408
- * refresh contains its own failures), and resolves only once the sweep has
1409
- * finished.
1410
- */
1411
- async refreshRevocationEnforcement(strandId: string): Promise<void> {
1412
- const enforcer = this.revocationEnforcers.get(strandId);
1413
- if (!enforcer) {
1414
- log('refreshRevocationEnforcement: strand %s has no armed revocation enforcer', strandId);
1415
- return;
1416
- }
1417
- await enforcer.refresh();
1418
- }
1419
-
1420
- /**
1421
- * A fresh membership invitation was staged for `strandId` (`CadreNode`'s
1422
- * `adoptFormationMembershipInvite`): re-arm the strand's membership reconciler so the
1423
- * invitation is attempted now rather than never — the loop finished during the first
1424
- * join and would otherwise stay stopped until a relaunch. Quiet no-op when no reconciler
1425
- * is armed: a first formation stages before the strand is added, and bring-up then arms
1426
- * a loop that finds the invitation by itself. Returns at once; the pass runs on the
1427
- * loop's own serialized chain and never rejects.
1428
- */
1429
- notifyMembershipInviteStaged(strandId: string): void {
1430
- const reconciler = this.membershipReconcilers.get(strandId);
1431
- if (!reconciler) {
1432
- log('notifyMembershipInviteStaged: strand %s has no armed membership reconciler — bring-up will find the invitation', strandId);
1433
- return;
1434
- }
1435
- void reconciler.rearm();
1436
- }
1437
-
1438
- /**
1439
- * Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
1440
- * self arm of `removeMemberPeer`, signed with the retained launch config's party
1441
- * key. Called by `CadreNode.unpublishStrand` BEFORE the local stop, while the
1442
- * strand database and transport are still live: after the unpublish commits, the
1443
- * party's `StrandPartyKey` control row is gone, so no restart could ever sign
1444
- * this removal again — the in-memory retained key is the last chance.
1445
- *
1446
- * Never throws. Quiet no-op for a strand that is untracked, open, or launched
1447
- * without a party key; a quiesced (hibernating) strand, or a strand whose write
1448
- * quorum is already unreachable, leaves the stale binding behind with a log —
1449
- * it grants nothing today (admission is not allowlist-gated) and only
1450
- * mis-credits diversity. The strand's own membership reconciler is stopped —
1451
- * and its in-flight pass awaited — FIRST, so a pass that was already past its
1452
- * own stopped check cannot re-register the binding this is about to clear.
1453
- *
1454
- * NOTE: deliberately NOT handled here (or anywhere yet): clearing the bindings
1455
- * of a machine removed from the CADRE at the control layer, or of a party's
1456
- * OTHER machines when the party unpublishes — the party still holds the key on
1457
- * every surviving machine and can clear from any of them, and the stale rows
1458
- * grant nothing while admission is deny-list only. Revisit when
1459
- * `feat-strand-member-allowlist-admission` lands and a stale binding starts
1460
- * granting admission rather than merely mis-crediting diversity.
1461
- */
1462
- async clearOwnMemberPeerBinding(strandId: string): Promise<void> {
1463
- const instance = this.instances.get(strandId);
1464
- const config = this.launchConfigs.get(strandId);
1465
- if (!instance || !config || config.strandRow.Type !== 'c' || !config.partyMemberPrivateKey) {
1466
- return;
1467
- }
1468
- const reconciler = this.membershipReconcilers.get(strandId);
1469
- if (reconciler) {
1470
- reconciler.stop();
1471
- await reconciler.settle();
1472
- }
1473
- const db = instance.database?.getDatabase();
1474
- const peerId = instance.libp2pNode?.peerId.toString();
1475
- if (!db || !peerId) {
1476
- log('clearOwnMemberPeerBinding: strand %s has no live runtime — own MemberPeer binding left in place', strandId);
1477
- return;
1478
- }
1479
- try {
1480
- // Background write on the app's database: never join a transaction the app has open.
1481
- await removeMemberPeer(db, {
1482
- memberKeyPair: strandMemberKeyPair(config.partyMemberPrivateKey),
1483
- peerId
1484
- }, { joinOpenTransaction: false });
1485
- log('clearOwnMemberPeerBinding: strand %s — own binding for peer %s cleared', strandId, peerId);
1486
- } catch (error) {
1487
- log('clearOwnMemberPeerBinding: strand %s — best-effort clear failed (strand may already be unreachable): %o',
1488
- strandId, error);
1489
- }
1490
- }
1491
-
1492
- /**
1493
- * Quiesce a strand: release its strand-network resources (stop the libp2p node,
1494
- * close the StrandDatabase) while RETAINING the instance record — identity,
1495
- * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
1496
- * resumed later, then mark it `hibernating`. Mechanically this is `stopStrand` minus
1497
- * the instance/config deletion. A tracked strand with no runtime left (a failed
1498
- * rebuild already released it) is only marked; an untracked one is left alone.
1499
- *
1500
- * Ordered with {@link resumeStrand} (see {@link transitions}): it starts once the resume
1501
- * or launch build before it has settled, so it releases a finished runtime, and the
1502
- * status write is its last step, so a resume requested meanwhile writes after it. A call
1503
- * made while another quiesce is the latest request joins it. A failed release leaves the
1504
- * status as it was and rejects.
1505
- */
1506
- async quiesceStrand(strandId: string): Promise<void> {
1507
- const previous = this.transitions.get(strandId);
1508
- if (previous?.kind === 'quiesce') {
1509
- log('quiesceStrand: strand %s — joining the quiesce already requested', strandId);
1510
- return previous.operation;
1511
- }
1512
- const operation = this.runQuiesce(strandId, previous);
1513
- this.holdTransition(strandId, { kind: 'quiesce', operation });
1514
- return operation;
1515
- }
1516
-
1517
- /** Body of {@link quiesceStrand}. */
1518
- private async runQuiesce(strandId: string, previous: RuntimeTransition | undefined): Promise<void> {
1519
- await this.settleTransition(strandId, previous);
1520
- await this.settleRuntimeBuilds(strandId);
1521
- const instance = this.instances.get(strandId);
1522
- if (!instance) {
1523
- log('quiesceStrand: strand %s not found', strandId);
1524
- return;
1525
- }
1526
- if (instance.libp2pNode || instance.database) {
1527
- log('Quiescing strand instance: %s', strandId);
1528
- await this.releaseRuntime(instance);
1529
- log('Strand %s quiesced (resources released, instance retained)', strandId);
1530
- } else {
1531
- log('quiesceStrand: strand %s already quiesced', strandId);
1532
- }
1533
- instance.status = 'hibernating';
1534
- }
1535
-
1536
- /**
1537
- * Whether the latest runtime transition requested for `strandId` is a quiesce that has
1538
- * not finished. Its handles may still be attached and its status still live, but it is on
1539
- * its way to `hibernating`, so a caller that needs it up must {@link resumeStrand} it — the
1540
- * resume runs after the quiesce — rather than treat it as live.
1541
- */
1542
- isQuiescing(strandId: string): boolean {
1543
- return this.transitions.get(strandId)?.kind === 'quiesce';
1544
- }
1545
-
1546
- /**
1547
- * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
1548
- * from the retained launch config and re-attach them, transitioning it back to
1549
- * `active`. `overrides` re-applies volatile inputs that may have changed since
1550
- * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
1551
- * count) and updates the retained config so a later resume reuses the latest
1552
- * values. Returns the live instance unchanged if it is already running.
1553
- *
1554
- * Overlapping calls share one rebuild: a call made while a resume is the latest request
1555
- * for the strand — a wake landing during a check-in — joins it and settles exactly as it
1556
- * does, `'error'` status included. The joiner's own `overrides` are ignored; the first
1557
- * resume's seed wins, and both callers resolved the cohort seed moments apart. A call made
1558
- * while a quiesce is the latest request runs after that quiesce and rebuilds (see
1559
- * {@link transitions}), and one made while `startStrand` is still building waits for that
1560
- * build rather than starting a second.
1561
- */
1562
- async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
1563
- if (this.stopping) {
1564
- throw new Error('StrandInstanceManager is stopping');
1565
- }
1566
- // No `await` between this read and `holdTransition`, or a second caller slips through
1567
- // and builds.
1568
- const previous = this.transitions.get(strandId);
1569
- if (previous?.kind === 'resume') {
1570
- log('resumeStrand: strand %s — joining the resume already in flight', strandId);
1571
- return previous.operation;
1572
- }
1573
- const operation = this.runResume(strandId, previous, overrides);
1574
- this.holdTransition(strandId, { kind: 'resume', operation });
1575
- return operation;
1576
- }
1577
-
1578
- /** Record `transition` as the latest for `strandId` until it settles. */
1579
- private holdTransition(strandId: string, transition: RuntimeTransition): void {
1580
- this.transitions.set(strandId, transition);
1581
- const release = (): void => {
1582
- if (this.transitions.get(strandId) === transition) {
1583
- this.transitions.delete(strandId);
1584
- }
1585
- };
1586
- // The rejection itself reaches the caller through the operation returned to it.
1587
- void transition.operation.then(release, release);
1588
- }
1589
-
1590
- /** Wait for the transition requested before this one. Never rejects: its failure belongs to its caller. */
1591
- private async settleTransition(strandId: string, previous: RuntimeTransition | undefined): Promise<void> {
1592
- if (!previous) {
1593
- return;
1594
- }
1595
- await (previous.operation as Promise<unknown>).catch((error: unknown) => {
1596
- log('Strand %s: the %s queued ahead failed (reported by its caller): %o', strandId, previous.kind, error);
1597
- });
1598
- }
1599
-
1600
- /** Body of {@link resumeStrand}; at most one runs per strand at a time. */
1601
- private async runResume(
1602
- strandId: string,
1603
- previous: RuntimeTransition | undefined,
1604
- overrides?: ResumeStrandOverrides
1605
- ): Promise<StrandInstance> {
1606
- // Before any "already live" test: a quiesce ahead still has the handles attached, and
1607
- // mid-build (a launch still building) the libp2p node is attached while the database is
1608
- // not. A failed launch has already dropped the instance, and the check below reports it
1609
- // untracked.
1610
- await this.settleTransition(strandId, previous);
1611
- await this.settleRuntimeBuilds(strandId);
1612
- const instance = this.instances.get(strandId);
1613
- if (!instance) {
1614
- throw new Error(`Cannot resume strand ${strandId}: not tracked`);
1615
- }
1616
- const launchConfig = this.launchConfigs.get(strandId);
1617
- if (!launchConfig) {
1618
- throw new Error(`Cannot resume strand ${strandId}: no retained launch config`);
1619
- }
1620
- if (instance.libp2pNode || instance.database) {
1621
- log('resumeStrand: strand %s already live', strandId);
1622
- return instance;
1623
- }
1624
-
1625
- log('Resuming strand instance: %s', strandId);
1626
- const tTotal = performance.now();
1627
-
1628
- // Re-apply volatile inputs and persist them so a subsequent resume reuses them.
1629
- // Each `??` matters: a resume that passes no override must keep the retained value,
1630
- // and a resume that passes one must leave it retained for the next resume — otherwise
1631
- // a later no-override wake silently reverts to whatever launch time saw.
1632
- const resumeConfig: StartStrandConfig = {
1633
- ...launchConfig,
1634
- bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes,
1635
- // NOTE: `??` retains, so an override can raise or lower the count but cannot CLEAR it
1636
- // back to "this node no longer knows" — the direction that would declare nothing. Moot
1637
- // while nothing feeds `servingMachines` at all; if a source lands
1638
- // (`backlog/feat-strand-yardstick-from-serving-machines`) that can legitimately lose the
1639
- // count — a strand whose member rows became unreadable — this merge must gain an explicit
1640
- // clear rather than silently declaring a stale number over a serving set it can no longer
1641
- // see. Same applies to `bootstrapNodes` above, where a stale seed is harmless.
1642
- servingMachines: overrides?.servingMachines ?? launchConfig.servingMachines
1643
- };
1644
- this.launchConfigs.set(strandId, resumeConfig);
1645
-
1646
- instance.status = 'starting';
1647
- try {
1648
- await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, resumeConfig));
1649
- timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
1650
- log('Strand %s resumed successfully', strandId);
1651
- return instance;
1652
- } catch (error) {
1653
- instance.status = 'error';
1654
- instance.error = error instanceof Error ? error.message : String(error);
1655
- log('Failed to resume strand %s: %s', strandId, instance.error);
1656
- throw error;
1657
- }
1658
- }
1659
-
1660
- /**
1661
- * Honor a founder request against an ALREADY-TRACKED strand — the seam that
1662
- * closes the "whoever launches first decides whether the bootstrap runs" gap:
1663
- * an instance first launched as a joiner (an app's own attach, or a watcher
1664
- * poll winning the launch race) used to swallow a later founder request
1665
- * silently, leaving the strand active with no `Strand.Header`.
1666
- *
1667
- * Flips the RETAINED launch config's `founder` to true, so every later
1668
- * quiesce → resume rebuild founds as well (the bootstrap is insert-if-absent —
1669
- * {@link StrandDatabase.ensureFounderBootstrap} — so re-running it per rebuild
1670
- * writes nothing twice), and runs the bootstrap against the live database now.
1671
- *
1672
- * @returns how the request resolved:
1673
- * - `'already-founder'` — the retained config already founds; nothing to do.
1674
- * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1675
- * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1676
- * database) or being quiesced (its database is closing), so the bootstrap could
1677
- * not run here: the CALLER must wake the
1678
- * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1679
- * manager does not) so the rebuild — which now founds — runs it.
1680
- * @param resolvePartyKey - Asked for the party's own membership key when (and only
1681
- * when) a CLOSED strand's retained config carries none — the instance was launched
1682
- * as a joiner before its `StrandPartyKey` row existed or replicated. The resolved
1683
- * key is retained alongside the founder flip so the bootstrap (now, or on the
1684
- * caller's wake for `'needs-resume'`) can seat the founding Member/Manager.
1685
- * @throws when the strand is not tracked — this seam exists only for the
1686
- * tracked-instance launch path; an untracked id is a caller bug.
1687
- * @throws whatever the live bootstrap throws (e.g. `PreSplitStrandIdentityError`),
1688
- * after {@link withdrawFounderRequest} — so a retry re-attempts the founding rather
1689
- * than resolving `'already-founder'` over an instance that never founded. A caller
1690
- * whose `'needs-resume'` wake fails owes the same withdrawal.
1691
- */
1692
- async foundExistingStrand(
1693
- strandId: string,
1694
- resolvePartyKey?: () => Promise<string | undefined>
1695
- ): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'> {
1696
- const instance = this.instances.get(strandId);
1697
- const config = this.launchConfigs.get(strandId);
1698
- if (!instance || !config) {
1699
- throw new Error(`Cannot found strand ${strandId}: not tracked`);
1700
- }
1701
- if (config.founder === true) {
1702
- return 'already-founder';
1703
- }
1704
- const partyMemberPrivateKey =
1705
- config.partyMemberPrivateKey
1706
- ?? (config.strandRow.Type === 'c' ? await resolvePartyKey?.() : undefined);
1707
- // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1708
- // config object, which is not ours to rewrite.
1709
- this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1710
- if (this.isQuiescing(strandId) || (!instance.database && !this.firstSyncGates.has(strandId))) {
1711
- return 'needs-resume';
1712
- }
1713
- try {
1714
- await this.ensureFounderBootstrap(strandId);
1715
- } catch (error) {
1716
- this.withdrawFounderRequest(strandId);
1717
- throw error;
1718
- }
1719
- return 'bootstrapped';
1720
- }
1721
-
1722
- /**
1723
- * Give a tracked storage replica ({@link StartStrandConfig.sAppConfig} absent) the app's
1724
- * schema in place — the seam for an app on this machine claiming a strand this node was
1725
- * already hosting. No runtime rebuild: the libp2p node, its peer id and connections, the
1726
- * store and its warm cache all stay, and the new `App` tables read the blocks the replica
1727
- * already holds. Callers attach BEFORE {@link foundExistingStrand}: the founder bootstrap
1728
- * writes the sApp into `Strand.Header` and refuses a database that has none.
1729
- *
1730
- * A runtime build in flight (the replica's launch, or a hibernation wake) is waited out
1731
- * first, so the attach acts on the database that build produced. Then, in order: the
1732
- * schema signature is checked before anything changes; the retained launch config takes
1733
- * the sApp, so every later rebuild applies it through `composeStrand`; the live database —
1734
- * published, or still held by the first-sync gate — gets the schema; and only then does
1735
- * the instance record `sAppInfo` and the sApp's latency hint. An instance with no live
1736
- * database gets the config alone: a quiesced one applies the schema at its next resume, a
1737
- * launch waiting for a seed peer at its next database retry.
1738
- *
1739
- * @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
1740
- * is logged and left as it is, as a second claim of a claimed strand always has been —
1741
- * else `'attached'`.
1742
- * @throws when the strand is not tracked (a caller bug, or a launch that failed while
1743
- * this waited on it); when the schema signature is refused, having changed nothing; and
1744
- * whatever the live apply throws (e.g. a quiesce closed the database mid-apply) — the
1745
- * retained config then still carries the sApp and `sAppInfo` stays unset, so the next
1746
- * claim retries the apply.
1747
- *
1748
- * NOTE: two claims of one replica racing each other both pass the `sAppInfo` check and
1749
- * apply concurrently; the loser of the declarative diff can reject, and its `addStrand`
1750
- * retry then resolves `'already-attached'`. Needs an app calling `addStrand` twice at once
1751
- * (or beside a watcher retry); if it is ever seen, chain attaches per strand id.
1752
- */
1753
- async attachSApp(
1754
- strandId: string,
1755
- sAppConfig: SAppConfig,
1756
- options: { requireSignedSchemas?: boolean } = {}
1757
- ): Promise<'attached' | 'already-attached'> {
1758
- await this.settleRuntimeBuilds(strandId);
1759
- const instance = this.instances.get(strandId);
1760
- const config = this.launchConfigs.get(strandId);
1761
- if (!instance || !config) {
1762
- throw new Error(`Cannot attach an sApp to strand ${strandId}: not tracked`);
1763
- }
1764
- if (instance.sAppInfo) {
1765
- if (instance.sAppInfo.id !== sAppConfig.id) {
1766
- log('attachSApp: strand %s already runs sApp %s — the claim for sApp %s is ignored',
1767
- strandId, instance.sAppInfo.id, sAppConfig.id);
1768
- }
1769
- return 'already-attached';
1770
- }
1771
- const sAppInfo = verifiedSAppInfo(strandId, sAppConfig, options.requireSignedSchemas);
1772
- // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1773
- // config object, which is not ours to rewrite.
1774
- this.launchConfigs.set(strandId, { ...config, sAppConfig });
1775
- const database = instance.database ?? this.firstSyncGates.get(strandId)?.database;
1776
- if (database) {
1777
- await database.attachAppSchema(sAppConfig);
1778
- } else {
1779
- log('attachSApp: strand %s has no live database — its next build applies the sApp schema', strandId);
1780
- }
1781
- instance.sAppInfo = sAppInfo;
1782
- if (sAppConfig.latencyHint) {
1783
- // HibernationManager reads the hint whenever it arms a timer, so the app's hint governs
1784
- // from the next one: a timer already armed runs once at the old duration, and a replica
1785
- // launched under a realtime default was never tracked, so it stays up until relaunched.
1786
- instance.latencyHint = sAppConfig.latencyHint;
1787
- }
1788
- log('Strand %s upgraded from storage replica to %s', strandId, describeSApp(sAppConfig));
1789
- return 'attached';
1790
- }
1791
-
1792
- /**
1793
- * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1794
- * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
1795
- * the hibernation manager's idle timer fired on it), including one whose database build
1796
- * waits for a seed peer to answer ({@link deferUnreachableJoin}). `false` for a writable,
1797
- * quiesced, or untracked strand. The predicate {@link whenWritable}'s callers gate on.
1798
- */
1799
- isAwaitingFirstSync(strandId: string): boolean {
1800
- const instance = this.instances.get(strandId);
1801
- return instance !== undefined && isAwaitingFirstSync(instance);
1802
- }
1803
-
1804
- /**
1805
- * Resolve once the strand's database is published to the app — immediately for a
1806
- * writable strand — or reject with {@link StrandAwaitingFirstSyncError} after
1807
- * `timeoutMs` (default: the retained launch config's `firstSync.timeoutMs`, else
1808
- * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}). The rejection is RETRYABLE: nothing is
1809
- * torn down, the gate keeps probing, and a later call waits afresh. A quiesced
1810
- * (hibernating) strand is not woken here — the wait spans a resume, so a caller that
1811
- * wants one wakes it (`CadreNode.wakeStrand`) and waits. Rejects immediately for an
1812
- * untracked strand, and whenever the strand is stopped while a wait is pending.
1813
- */
1814
- whenWritable(strandId: string, options?: { timeoutMs?: number }): Promise<StrandInstance> {
1815
- const instance = this.instances.get(strandId);
1816
- if (!instance) {
1817
- return Promise.reject(new Error(`Cannot wait for strand ${strandId} to become writable: not tracked`));
1818
- }
1819
- if (instance.database) {
1820
- return Promise.resolve(instance);
1821
- }
1822
- const timeoutMs = options?.timeoutMs
1823
- ?? this.launchConfigs.get(strandId)?.firstSync?.timeoutMs
1824
- ?? DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS;
1825
- const startedAt = Date.now();
1826
- return new Promise<StrandInstance>((resolve, reject) => {
1827
- const waiters = this.writableWaiters.get(strandId) ?? new Set<WritableWaiter>();
1828
- this.writableWaiters.set(strandId, waiters);
1829
- const settle = (): void => {
1830
- clearTimeout(timer);
1831
- waiters.delete(waiter);
1832
- if (waiters.size === 0) {
1833
- this.writableWaiters.delete(strandId);
1834
- }
1835
- };
1836
- const waiter: WritableWaiter = {
1837
- resolve: () => { settle(); resolve(instance); },
1838
- reject: (error) => { settle(); reject(error); }
1839
- };
1840
- const timer = setTimeout(
1841
- () => waiter.reject(new StrandAwaitingFirstSyncError(strandId, Date.now() - startedAt)),
1842
- timeoutMs);
1843
- (timer as { unref?: () => void }).unref?.();
1844
- waiters.add(waiter);
1845
- });
1846
- }
1847
-
1848
- /**
1849
- * The tracked instance once no runtime build of it is in flight, or `undefined` when none
1850
- * is tracked by then — never launched, or the launch waited on failed and dropped its
1851
- * record. For a caller about to hand a tracked instance to an app: until its build settles
1852
- * the instance is `'starting'` with no database, and a failed launch leaves no instance.
1853
- * Never rejects; a failed build is reported by the call that started it.
1854
- */
1855
- async whenRuntimeBuilt(strandId: string): Promise<StrandInstance | undefined> {
1856
- await this.settleRuntimeBuilds(strandId);
1857
- return this.instances.get(strandId);
1858
- }
1859
-
1860
- /**
1861
- * Hand a launch's database to the app: set `instance.database`, retire the gate that
1862
- * held it, release every {@link whenWritable} waiter, and — when the launch had already
1863
- * been reported gated — flip it `'active'` and announce it through the retained config's
1864
- * `onWritable`. "Reported gated" is any status but `'starting'`: `'syncing'`, or the
1865
- * `'idle'` the hibernation manager's idle timer flips a long-gated joiner to (see
1866
- * {@link isAwaitingFirstSync}) — the Header arriving after that flip must announce too,
1867
- * or an app hanging on `strand:writable` never learns the strand opened. A publish that
1868
- * lands while the instance is still `'starting'` (the Header arrived during the rest of
1869
- * bring-up) announces nothing: `startStrand` / `resumeStrand` are about to report the
1870
- * strand `'active'`, and `strand:writable` is defined as the follow-up to a gated
1871
- * launch, never a duplicate of `strand:started`.
1872
- *
1873
- * It is also the one seam where a WITHHELD database becomes available, so it kicks the
1874
- * membership reconciler: on a gated launch that loop's own immediate pass ran while
1875
- * `instance.database` was still unset, found no database, and would otherwise not try
1876
- * again until its retry timer fired. A launch that was never gated publishes here BEFORE
1877
- * the reconciler is constructed, so there is nothing registered to kick and the loop's own
1878
- * immediate pass already sees the database — which is why the kick needs no `wasGated`
1879
- * test of its own.
1880
- */
1881
- private publishDatabase(instance: StrandInstance, database: StrandDatabase): void {
1882
- const strandId = instance.strandId;
1883
- const gate = this.firstSyncGates.get(strandId);
1884
- if (gate) {
1885
- gate.open();
1886
- this.firstSyncGates.delete(strandId);
1887
- }
1888
- instance.database = database;
1889
- const wasGated = instance.status !== 'starting';
1890
- if (wasGated) {
1891
- // A Header delivered by a peer is strand activity; a publish during bring-up is not
1892
- // (see the build's end in `buildStrandRuntime`).
1893
- instance.lastActivity = new Date();
1894
- instance.status = 'active';
1895
- log('Strand %s is now writable: Strand.Header held', strandId);
1896
- }
1897
- const waiters = this.writableWaiters.get(strandId);
1898
- if (waiters) {
1899
- this.writableWaiters.delete(strandId);
1900
- waiters.forEach((waiter) => waiter.resolve());
1901
- }
1902
- if (wasGated) {
1903
- this.launchConfigs.get(strandId)?.onWritable?.(strandId);
1904
- }
1905
- // Not awaited: publishing must stay synchronous, and the pass chains on the loop's own
1906
- // tail, so it cannot overlap a pass already in flight.
1907
- void this.membershipReconcilers.get(strandId)?.reconcile();
1908
- }
1909
-
1910
- /** Reject every {@link whenWritable} waiter for a strand that is going away. */
1911
- private rejectWritableWaiters(strandId: string, reason: string): void {
1912
- const waiters = this.writableWaiters.get(strandId);
1913
- if (!waiters) {
1914
- return;
1915
- }
1916
- this.writableWaiters.delete(strandId);
1917
- waiters.forEach((waiter) => waiter.reject(new Error(`Strand ${strandId} ${reason} before becoming writable`)));
1918
- }
1919
-
1920
- /**
1921
- * Undo {@link foundExistingStrand}'s founder flip after the founding it promised
1922
- * failed, so the retained config again says what this instance actually runs as (a
1923
- * joiner) and the next founder request re-attempts the bootstrap. The resolved party
1924
- * key stays retained — it is this party's identity for the strand either way. No-op
1925
- * when the strand is untracked or its config does not found.
1926
- */
1927
- withdrawFounderRequest(strandId: string): void {
1928
- const config = this.launchConfigs.get(strandId);
1929
- if (config?.founder === true) {
1930
- this.launchConfigs.set(strandId, { ...config, founder: false });
1931
- }
1932
- }
1933
-
1934
- /**
1935
- * Run the (idempotent) founder bootstrap against a tracked strand's LIVE
1936
- * database, independently of what the retained launch config says.
1937
- *
1938
- * Separate from {@link foundExistingStrand} because a caller that resolved
1939
- * `'needs-resume'` and woke the strand must not assume the wake's own rebuild
1940
- * founded it: `HibernationManager` COALESCES wakes, so a wake already in flight
1941
- * when the config flipped had already read the PRE-flip config and rebuilt as a
1942
- * joiner. Re-running the bootstrap costs one insert-if-absent probe per table
1943
- * and is the only thing that makes "founding resolves once the Header is
1944
- * written" true on that path.
1945
- *
1946
- * A strand still `'syncing'` (its database withheld behind the first-sync gate) is
1947
- * founded through the gate: the bootstrap writes the Header this machine was waiting
1948
- * to receive, so a successful run publishes the database and the strand goes
1949
- * `'active'` — the one legitimate way a joiner launch becomes writable without a peer.
1950
- *
1951
- * @throws when the strand is not tracked, or is still quiesced (no live
1952
- * database) — both mean the bootstrap did NOT run, which a founder request
1953
- * must never swallow.
1954
- */
1955
- async ensureFounderBootstrap(strandId: string): Promise<void> {
1956
- const instance = this.instances.get(strandId);
1957
- if (!instance) {
1958
- throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: not tracked`);
1959
- }
1960
- const gate = this.firstSyncGates.get(strandId);
1961
- const database = instance.database ?? gate?.database;
1962
- if (!database) {
1963
- throw new Error(
1964
- `Cannot run the founder bootstrap for strand ${strandId}: it has no live database to ` +
1965
- 'write to — it is quiesced (resume it first), or its database build is waiting for a ' +
1966
- 'seed peer to answer.'
1967
- );
1968
- }
1969
- // Forward the RETAINED config's party key: a foundExistingStrand that flipped a
1970
- // joiner launch to founder may have resolved a key the live database's captured
1971
- // config (built at the original launch) never saw.
1972
- await database.ensureFounderBootstrap(this.launchConfigs.get(strandId)?.partyMemberPrivateKey);
1973
- if (!instance.database) {
1974
- this.publishDatabase(instance, database);
1975
- }
1976
- }
1977
-
1978
- /**
1979
- * Stop a strand instance
1980
- */
1981
- async stopStrand(strandId: string): Promise<void> {
1982
- const instance = this.instances.get(strandId);
1983
- if (!instance) {
1984
- log('Strand %s not found', strandId);
1985
- return;
1986
- }
1987
-
1988
- log('Stopping strand instance: %s', strandId);
1989
- instance.status = 'stopping';
1990
- this.rejectWritableWaiters(strandId, 'was stopped');
1991
-
1992
- try {
1993
- await this.releaseRuntime(instance);
1994
- instance.status = 'stopped';
1995
- this.instances.delete(strandId);
1996
- this.launchConfigs.delete(strandId);
1997
- // Storage is released only here — NOT in releaseRuntime, which a quiesce shares.
1998
- await this.disposeStrandStorage(strandId);
1999
- log('Strand %s stopped successfully', strandId);
2000
- } catch (error) {
2001
- instance.status = 'error';
2002
- instance.error = error instanceof Error ? error.message : String(error);
2003
- log('Error stopping strand %s: %s', strandId, instance.error);
2004
- throw error;
2005
- }
2006
- }
2007
-
2008
- /**
2009
- * Stop all strand instances
2010
- */
2011
- async stopAll(): Promise<void> {
2012
- this.stopping = true;
2013
- log('Stopping all strand instances (%d)', this.instances.size);
2014
-
2015
- const stopPromises = Array.from(this.instances.keys()).map(id =>
2016
- this.stopStrand(id).catch(err => {
2017
- log('Error stopping strand %s during shutdown: %s', id, err);
2018
- })
2019
- );
2020
-
2021
- await Promise.all(stopPromises);
2022
- this.stopping = false;
2023
- log('All strand instances stopped');
2024
- }
2025
- }
2026
-
1
+ import debug from 'debug';
2
+ import type { Libp2p, PrivateKey } from '@libp2p/interface';
3
+ import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
4
+ import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
5
+ import { StrandDatabase } from './strand-database.js';
6
+ import type { StrandCommitDurability, StrandCommitHeld } from './strand-commit-durability.js';
7
+ import { PeerJoinBackfill, type PeerJoinBackfillConfig } from './peer-join-backfill.js';
8
+ import { strandNetworkStatePersistence, type StrandNetworkStateStore } from './strand-network-state.js';
9
+ import {
10
+ StrandRevocationEnforcer,
11
+ createRevocationConnectionGater,
12
+ readStrandRevocationRows,
13
+ type StrandRevocationEnforcementConfig
14
+ } from './strand-revocation-enforcer.js';
15
+ import {
16
+ StrandMembershipReconciler,
17
+ type PendingMembershipInviteSource,
18
+ type StrandMembershipReconciliationConfig
19
+ } from './strand-membership-reconciler.js';
20
+ import { removeMemberPeer } from './strand-membership-writer.js';
21
+ import { strandMemberKeyPair } from './strand-member-key.js';
22
+ import { assertSchemaSignature } from './schema-verification.js';
23
+ import { assertStrandScopeKey } from './storage-scope.js';
24
+ import {
25
+ StrandFirstSyncGate,
26
+ StrandAwaitingFirstSyncError,
27
+ DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS,
28
+ type StrandFirstSyncConfig
29
+ } from './strand-first-sync-gate.js';
30
+ import type {
31
+ StrandInstance,
32
+ StrandRow,
33
+ StorageConfig,
34
+ NetworkConfig,
35
+ LatencyHint,
36
+ NodeProfile,
37
+ SAppConfig,
38
+ SAppInfo,
39
+ RawStorageProvider,
40
+ Libp2pNodeWithRepo
41
+ } from './types.js';
42
+ import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
43
+ import { strandNodeAddrs } from './strand-network-config.js';
44
+ import { nodeTransportsOption } from './node-transports.js';
45
+ import { resolveRelayServer } from './relay-server.js';
46
+ import { strandCohortTopicOption, type StrandReactivityConfig } from './strand-reactivity.js';
47
+ import { superviseRelayReservation, type RelayReservationSupervisor } from './relay-reservation.js';
48
+ import { strandRedialTimings, superviseStrandPeerRedial, type StrandPeerRedial } from './strand-peer-redial.js';
49
+ import { groupAddrsByPeerId } from './peer-addr-book.js';
50
+ import { isCohortUnreachableRead } from './control-read-retry.js';
51
+ import { declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
52
+
53
+ const log = debug('sereus:cadre:strand-manager');
54
+
55
+ /**
56
+ * How often a joining launch that could not build its database, because no seed peer had
57
+ * answered, retries the build on its running libp2p node
58
+ * (`StrandInstanceManager.deferUnreachableJoin`). Each attempt composes a fresh database and
59
+ * re-reads the schema catalog over the network, so it is spaced well above the first-sync
60
+ * gate's probe cadence.
61
+ *
62
+ * NOTE: the 5 s dates from when each attempt also rebuilt the libp2p node. A joiner whose seed
63
+ * peer answered just after a failed attempt waits up to this long to become writable; if that
64
+ * wait shows up in a measured join, try a shorter first retry.
65
+ */
66
+ export const UNREACHABLE_JOIN_RETRY_MS = 5_000;
67
+
68
+ /**
69
+ * How one database build of a runtime ended ({@link StrandInstanceManager.buildStrandDatabase}):
70
+ * published or held by the first-sync gate; failed only because no seed peer has answered yet;
71
+ * or overtaken by a release of the runtime it was building on.
72
+ */
73
+ type StrandDatabaseBuild = 'built' | 'awaiting-seed' | 'released';
74
+
75
+ const timing = debug('sereus:cadre:timing');
76
+
77
+ /** One caller blocked in {@link StrandInstanceManager.whenWritable}. */
78
+ interface WritableWaiter {
79
+ resolve: () => void;
80
+ reject: (error: Error) => void;
81
+ }
82
+
83
+ /**
84
+ * Whether a tracked instance is launched but waiting for its first sync: the runtime is
85
+ * up (`libp2pNode` set) while the database is still withheld (`database` unset). That
86
+ * is the first-sync gate's state regardless of what `status` says — the hibernation
87
+ * manager's idle timer can flip a long-gated joiner to `'idle'` without changing it.
88
+ *
89
+ * NOTE: `status` is therefore not the source of truth for "gated": a joiner that reaches
90
+ * nobody for `idleTimeout` (5 min at the interactive hint) reads `'idle'`, then
91
+ * `'hibernating'`, while still holding no database. Harmless today — every decision the
92
+ * runtime makes goes through this predicate, {@link StrandInstanceManager.publishDatabase}
93
+ * announces on the idle-flipped instance exactly as on a `'syncing'` one, and an app that
94
+ * shows "waiting for the other member" on `'syncing'` merely sees "idle" instead after
95
+ * five minutes. If an app ever needs the gated LABEL to survive the idle flip, make the
96
+ * hibernation manager's idle transition preserve `'syncing'` rather than adding a flag.
97
+ */
98
+ export function isAwaitingFirstSync(instance: StrandInstance): boolean {
99
+ return instance.libp2pNode !== undefined && instance.database === undefined;
100
+ }
101
+
102
+ /**
103
+ * The status a LIVE instance (runtime rebuilt or woken) should report: `'active'` once its
104
+ * database is published, `'syncing'` while the first-sync gate still withholds it. Every
105
+ * site that would otherwise write `'active'` after a build or a wake goes through this,
106
+ * so a gated joiner never reads as writable.
107
+ */
108
+ export function liveStrandStatus(instance: StrandInstance): 'active' | 'syncing' {
109
+ return instance.database ? 'active' : 'syncing';
110
+ }
111
+
112
+ /**
113
+ * Configuration for starting a strand instance
114
+ */
115
+ export interface StartStrandConfig {
116
+ strandRow: StrandRow;
117
+ /**
118
+ * sApp configuration provided by the hosting application. Absent for a **storage
119
+ * replica** — a strand this node stores and serves without the app installed
120
+ * (`CadreNodeConfig.hostUnclaimedStrands`): no schema signature check, no `App`
121
+ * schema, no `sAppInfo` on the instance, and the default latency hint. A replica is
122
+ * always a joiner; `StrandDatabase` refuses to found one. An app that later claims the
123
+ * strand upgrades the running replica in place ({@link StrandInstanceManager.attachSApp}).
124
+ */
125
+ sAppConfig?: SAppConfig;
126
+ storage?: StorageConfig;
127
+ network?: NetworkConfig;
128
+ profile: NodeProfile;
129
+ defaultLatencyHint: LatencyHint;
130
+ privateKey?: PrivateKey;
131
+ /** Cohort-derived discovery seed (multiaddr strings). Defaults to [] when omitted. */
132
+ bootstrapNodes?: string[];
133
+ /**
134
+ * Require a valid author signature on the sApp schema before bring-up.
135
+ * Defaults to true (fail closed) when omitted; set false only for dev/test
136
+ * with unsigned demo schemas. Mirrors {@link CadreNodeConfig.requireSignedSchemas}.
137
+ */
138
+ requireSignedSchemas?: boolean;
139
+ /**
140
+ * Whether this node founds the strand (vs. joins it). Forwarded to the
141
+ * StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
142
+ * runs once at bring-up. Joiners leave this unset and write nothing. Callers
143
+ * resolve it BEFORE calling `startStrand` (`CadreNode.launchStrand` derives it
144
+ * from the row's `FounderPeerId` when no explicit flag is given — see
145
+ * {@link StrandConfig.founder}); a founder request that arrives while the
146
+ * instance is already tracked goes through {@link StrandInstanceManager.foundExistingStrand},
147
+ * never through a repeat `startStrand`.
148
+ */
149
+ founder?: boolean;
150
+ /**
151
+ * THIS party's own strand membership private key (base64 protobuf) for a closed
152
+ * strand — the identity the founder bootstrap derives `Member.Key` /
153
+ * `Manager.MemberKey` from. Resolved by the caller (`CadreNode.launchStrand`: an
154
+ * explicit attach-time key, else the control-layer `StrandPartyKey` row, healed by
155
+ * mint-on-launch for the founding machine) and retained with the launch config, so a
156
+ * hibernation wake rebuilds under the same identity. Deliberately NOT the strand
157
+ * row's shared `MemberPrivateKey` (see `StrandDatabaseConfig.partyMemberPrivateKey`).
158
+ * Absent for open strands, and for joiners with no persisted party key.
159
+ */
160
+ partyMemberPrivateKey?: string;
161
+ /**
162
+ * Number of nodes Optimystic is told this strand's replication cluster should
163
+ * have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
164
+ * forwards here: every node on the strand should use the same value, and it is
165
+ * frozen when the strand's libp2p node is created. Defaults to
166
+ * `DEFAULT_STRAND_CLUSTER_SIZE` (4); values below `MIN_CLUSTER_SIZE` (2) are
167
+ * rejected by `resolveStrandClusterSize`. Strand-only — the
168
+ * control network's breadth is the fixed `CONTROL_REPLICATION_BREADTH`.
169
+ */
170
+ clusterSize?: number;
171
+ /**
172
+ * Machines that SERVE THIS STRAND (this node included), for the strand node's
173
+ * block-repair corroboration yardstick — see `resolveRepairYardstick`. It must be
174
+ * an authenticated per-strand count: the machines that actually run a node for this
175
+ * strand, not the machines that exist.
176
+ *
177
+ * **The party's enrolled-machine count is NOT this number and must never be passed
178
+ * here.** A strand launches only on machines whose embedding app registered its sApp
179
+ * config (`CadreNode.addStrand`), so a closed strand shared by two machines of a
180
+ * three-machine party is served by two. Passing three over-declares, and
181
+ * over-declaring is the unsafe direction: at a declaration of 3 or more Optimystic
182
+ * pins the repair corroboration floor at two corroborating peers, which a cohort
183
+ * that can only ever field one peer can never reach — so the strand can never repair
184
+ * a block (`cluster-fetch:no-quorum`, surfacing as reads failing with `Missing
185
+ * block`). That regression is why this field was renamed off "enrolledMachines":
186
+ * `bug-strand-yardstick-counts-party-machines`.
187
+ *
188
+ * **Nothing feeds it in production today.** No authenticated per-strand serving count
189
+ * exists yet, so `CadreNode` passes nothing and the strand node runs the frozen
190
+ * `STRAND_CLUSTER_POLICY` — declaring no yardstick, which leaves the known,
191
+ * upstream-tracked single-voter exposure
192
+ * (`backlog/debt-read-repair-single-voter-corroboration`). Building the count is
193
+ * `backlog/feat-strand-yardstick-from-serving-machines`; this field and its
194
+ * threading are the seam it plugs into.
195
+ *
196
+ * Volatile when a source does exist: re-resolve it on every resume beside the cohort
197
+ * seed, since machines join and leave a strand while it hibernates. Omitting it means
198
+ * "this node does not know", which declares nothing.
199
+ */
200
+ servingMachines?: number;
201
+ /**
202
+ * Tuning for the strand peer-join block catch-up ({@link PeerJoinBackfill}),
203
+ * forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
204
+ * libp2p node connects (or reconnects) to a peer, every block in the strand's
205
+ * own raw store that the peer is not known to hold is pushed to it. Runs only on strands
206
+ * with per-strand storage; `{ enabled: false }` restores the pre-existing
207
+ * no-backfill behaviour.
208
+ */
209
+ backfill?: PeerJoinBackfillConfig;
210
+ /**
211
+ * Whether this strand's libp2p node is built with Optimystic's change notifications,
212
+ * forwarded from {@link CadreNodeConfig.strandReactivity} and resolved per strand by
213
+ * `strandCohortTopicOption`. Retained with the launch config, so a hibernation wake
214
+ * rebuilds the node with the same setting.
215
+ */
216
+ reactivity?: StrandReactivityConfig;
217
+ /**
218
+ * Tuning for the CLOSED-strand revoked-peer gate
219
+ * ({@link StrandRevocationEnforcer}), forwarded from
220
+ * {@link CadreNodeConfig.strandRevocationEnforcement}. Armed only for
221
+ * `strandRow.Type === 'c'` (an open strand has no membership rows to derive a
222
+ * deny set from); `{ enabled: false }` restores the pre-existing behaviour
223
+ * (a removed party's peers keep being served).
224
+ */
225
+ revocationEnforcement?: StrandRevocationEnforcementConfig;
226
+
227
+ /**
228
+ * The staged formation membership invitation for THIS strand — the seam the
229
+ * bring-up membership reconciler (`strand-membership-reconciler.ts`) reads to
230
+ * redeem the party's `Strand.Member` seat, and clears once the invitation is
231
+ * spent, burned, or dead. `CadreNode.launchStrand` wires it over its in-memory
232
+ * `pendingMembershipInvites` map; read lazily per pass, so a re-formation that
233
+ * replaces the entry between passes is picked up. Only consulted for closed
234
+ * strands launched with a {@link partyMemberPrivateKey}; harmless otherwise.
235
+ */
236
+ pendingMembershipInvite?: PendingMembershipInviteSource;
237
+
238
+ /**
239
+ * Tuning for the CLOSED-strand membership reconciler, forwarded from
240
+ * {@link CadreNodeConfig.strandMembershipReconciliation}. Armed only for
241
+ * closed strands launched with a {@link partyMemberPrivateKey};
242
+ * `{ enabled: false }` disarms the loop (test fixtures that hand-drive the
243
+ * membership writers). Its `pollIntervalMs` is the reconciler's IDLE cadence
244
+ * and the cap of its unfinished-join retry ladder, not a flat retry interval;
245
+ * default `DEFAULT_MEMBERSHIP_RECONCILE_INTERVAL_MS`.
246
+ */
247
+ membershipReconciliation?: StrandMembershipReconciliationConfig;
248
+
249
+ /**
250
+ * Called when THIS node's own peer id turns up in the strand's revoked set —
251
+ * this node's party was removed from the closed strand, or left it. Fires once
252
+ * per removal (re-armed if the party is re-admitted; a resume also rebuilds the
253
+ * enforcer and may legitimately re-fire), and only for closed strands with the
254
+ * gate armed. `CadreNode`
255
+ * wires it to its `strand:revoked` event; nothing is stopped or torn down on
256
+ * this node's behalf — what to do about it is the app's call.
257
+ */
258
+ onSelfRevoked?: (strandId: string) => void;
259
+
260
+ /**
261
+ * Called when the membership reconciler reports a blocked re-join — this node holds a
262
+ * staged membership invitation for the strand that its writes cannot redeem, which is
263
+ * what a REMOVED party handed a fresh invitation hits (see "Reporting a blocked
264
+ * re-join" in `strand-membership-reconciler.ts` for the two triggers, one of them a
265
+ * suspicion rather than a verdict). At most once per re-arm of the loop — a further
266
+ * invitation staged while the loop is still running does not reset the report — and
267
+ * only for closed strands with a party key. `CadreNode` wires it to its `strand:rejoin-blocked`
268
+ * event; nothing is stopped or torn down — a remaining manager admitting this party's
269
+ * key directly is the remedy, and the loop finishes the join by itself once that lands.
270
+ */
271
+ onRejoinBlocked?: (strandId: string) => void;
272
+
273
+ /**
274
+ * Tuning for the JOINING machine's first-sync write gate (`strand-first-sync-gate.ts`),
275
+ * forwarded from {@link CadreNodeConfig.strandFirstSync}: the probe backstop while a
276
+ * non-founder launch is `'syncing'`, and the default budget {@link whenWritable} waits
277
+ * before rejecting. Founders and machines that already hold the Header are never gated.
278
+ */
279
+ firstSync?: StrandFirstSyncConfig;
280
+
281
+ /**
282
+ * Called when a launch that came up `'syncing'` (see {@link firstSync}) becomes
283
+ * writable: the strand's `Strand.Header` arrived from a peer — or a founder request
284
+ * ran the bootstrap against the gated database — and `instance.database` is now set.
285
+ * Never called for a launch that was writable when `startStrand`/`resumeStrand`
286
+ * resolved. `CadreNode` wires it to its `strand:writable` event. Retained with the
287
+ * launch config, so a hibernation resume that comes up gated announces too.
288
+ */
289
+ onWritable?: (strandId: string) => void;
290
+
291
+ /**
292
+ * Called synchronously inside every commit on the strand's database that wrote something,
293
+ * with who holds it — the app's commits and cadre-core's own membership writes alike.
294
+ * `CadreNode` wires it to its `strand:commit-durability` event. Retained with the launch
295
+ * config and passed to every database built for the launch, so a hibernation wake's rebuilt
296
+ * database reports too. A throw is logged and does not affect the commit.
297
+ */
298
+ onCommitDurability?: (commit: StrandCommitDurability) => void;
299
+
300
+ /**
301
+ * Called when a commit reported `followed` below full replication becomes held by every member
302
+ * of its cohort, while the database that made it is still open. `CadreNode` wires it to its
303
+ * `strand:commit-held` event. Retained and isolated like {@link onCommitDurability}.
304
+ */
305
+ onCommitHeld?: (event: StrandCommitHeld) => void;
306
+
307
+ /**
308
+ * Re-announce this strand's delegate peer id (`delegatePeerId` — the strand
309
+ * node's own transport peer id) to ONE relay (`relayAddr`, a direct dial addr
310
+ * with a trailing `/p2p/<relayPeerId>`), unthrottled. Awaited before every
311
+ * relay-reservation RE-drive of that relay — never before the first attempt,
312
+ * which the launch/resume seed pass has already announced for.
313
+ *
314
+ * Why: a party control node running the relay server admits the strand node on
315
+ * an in-memory delegate grant that a relay restart drops, so a re-drive that did
316
+ * not re-announce first would be denied at the relay's gate. `CadreNode` wires
317
+ * its own control-mesh announce here; a throw or rejection is logged by the
318
+ * supervisor and the re-drive still runs. Retained with the launch config, so a
319
+ * hibernation wake's rebuilt supervisors carry it too.
320
+ */
321
+ announceDelegateToRelay?: (strandId: string, relayAddr: string, delegatePeerId: string) => Promise<void>;
322
+
323
+ /**
324
+ * The node-local strand network state (`strand-network-state.ts`): this strand's
325
+ * slice of it is handed to db-p2p as `persistence`, so the strand node saves its
326
+ * FRET routing table (with each peer's signed address record) on every connection
327
+ * and re-imports it when it is next built. Absent ⇒ nothing is saved and the node
328
+ * starts with an empty table. `CadreNode` passes its store. Retained with the launch
329
+ * config, so a hibernation wake rebuilds the node over the same state.
330
+ */
331
+ networkState?: StrandNetworkStateStore;
332
+ }
333
+
334
+ /**
335
+ * Volatile inputs re-resolved when resuming a quiesced strand — the ones that can
336
+ * have moved since it last ran, and that the rebuilt libp2p node freezes again. Each
337
+ * one omitted keeps the value the retained launch config already holds, so a resume
338
+ * that passes nothing rebuilds the strand exactly as it last ran.
339
+ */
340
+ export interface ResumeStrandOverrides {
341
+ /**
342
+ * Freshly-resolved cohort discovery seed (multiaddr strings). Grows as peers are
343
+ * learned since the strand first launched.
344
+ */
345
+ bootstrapNodes?: string[];
346
+ /**
347
+ * Freshly-read count of the machines serving this strand, for the repair yardstick
348
+ * (see {@link StartStrandConfig.servingMachines} — including why the party's
349
+ * enrolled-machine count is not it). Moves whenever a machine starts or stops
350
+ * serving the strand, which a hibernating strand does not otherwise notice. Nothing
351
+ * passes it today. Omitting it here does NOT mean "unknown" the way omitting it from
352
+ * {@link StartStrandConfig} does — it retains the last value; see `resumeStrand`.
353
+ */
354
+ servingMachines?: number;
355
+ }
356
+
357
+ /** A {@link StrandInstanceManager.resumeStrand} or {@link StrandInstanceManager.quiesceStrand} of one strand. */
358
+ type RuntimeTransition =
359
+ | { kind: 'resume'; operation: Promise<StrandInstance> }
360
+ | { kind: 'quiesce'; operation: Promise<void> };
361
+
362
+ /**
363
+ * Get the isolated storage path for a specific strand.
364
+ *
365
+ * @deprecated This helper is Node-only and throws in React Native (it assumes a
366
+ * filesystem layout). Use a storage provider factory function instead, which
367
+ * receives the strandId and can create strand-specific storage paths using
368
+ * platform-appropriate methods.
369
+ *
370
+ * @example
371
+ * // Instead of using getStrandStoragePath, use a storage provider factory:
372
+ * const storage = {
373
+ * provider: (strandId: string) => new FileRawStorage(`./data/strands/${strandId}`)
374
+ * };
375
+ */
376
+ export function getStrandStoragePath(basePath: string, strandId: string): string {
377
+ // Check if we're in a Node.js environment
378
+ if (typeof process === 'undefined' || !process.versions?.node) {
379
+ throw new Error(
380
+ 'getStrandStoragePath is not available in React Native. ' +
381
+ 'Use a storage provider factory function instead.'
382
+ );
383
+ }
384
+
385
+ // Sanitize strandId for filesystem safety (UUIDs should be safe, but just in case)
386
+ const safeId = strandId.replace(/[^a-zA-Z0-9-]/g, '_');
387
+
388
+ // Build the path with plain string joins rather than the Node `path` module.
389
+ // A static `require('path')` forces RN bundlers (e.g. Metro) to *resolve* the
390
+ // module at bundle time even though this Node-only helper throws above before
391
+ // ever reaching here — joining by hand keeps the module free of any Node
392
+ // built-in reference, so RN bundles need no `path` shim.
393
+ const trimmedBase = basePath.replace(/[\\/]+$/, '');
394
+ return `${trimmedBase}/strands/${safeId}`;
395
+ }
396
+
397
+ /**
398
+ * Resolve a storage provider for a specific strand.
399
+ * If the provider is a factory function, call it with the strandId.
400
+ *
401
+ * Called only from {@link StrandInstanceManager.startStrand}, which owns the result
402
+ * for the instance's lifetime — see `strandStorages`, and which has already asserted
403
+ * the id is usable as a scope key. Do not move that assertion here: this function
404
+ * returns early when no provider is configured, and the id becomes a libp2p protocol
405
+ * prefix on that path too.
406
+ *
407
+ * @param provider - Storage provider (instance or factory)
408
+ * @param strandId - The strand ID to create storage for
409
+ * @returns The resolved IRawStorage instance, or undefined if no provider
410
+ */
411
+ function resolveStrandStorage(
412
+ provider: RawStorageProvider | undefined,
413
+ strandId: string
414
+ ): IRawStorage | undefined {
415
+ if (!provider) {
416
+ return undefined;
417
+ }
418
+ const storage = typeof provider === 'function' ? provider(strandId) : provider;
419
+ // Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's cached-storage.ts).
420
+ // Called ONCE per strand launch — `startStrand` keeps the result for the instance's
421
+ // lifetime — so the wrap survives every runtime rebuild the instance goes through.
422
+ return wrapStorageWithCache(storage, strandId);
423
+ }
424
+
425
+ /**
426
+ * Verify an sApp's schema signature (fail-closed unless `requireSignedSchemas` is
427
+ * false) and project the config onto the instance's `sAppInfo`.
428
+ */
429
+ function verifiedSAppInfo(strandId: string, sAppConfig: SAppConfig, requireSignedSchemas: boolean | undefined): SAppInfo {
430
+ assertSchemaSignature(sAppConfig, { requireSignature: requireSignedSchemas ?? true });
431
+ log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
432
+ return {
433
+ id: sAppConfig.id,
434
+ version: sAppConfig.version,
435
+ schema: sAppConfig.schema,
436
+ signature: sAppConfig.signature
437
+ };
438
+ }
439
+
440
+ /** Log wording for what a launch runs: the sApp and its version, or a storage replica. */
441
+ function describeSApp(sAppConfig: SAppConfig | undefined): string {
442
+ return sAppConfig ? `sApp: ${sAppConfig.id} v${sAppConfig.version}` : 'storage replica';
443
+ }
444
+
445
+ /**
446
+ * Manages individual strand instances - creates and destroys isolated libp2p nodes
447
+ * for each strand the cadre participates in.
448
+ */
449
+ export class StrandInstanceManager {
450
+ private instances: Map<string, StrandInstance> = new Map();
451
+ /**
452
+ * Retained launch config per strand, captured in `startStrand` and cleared in
453
+ * `stopStrand`. `resumeStrand` reuses it to rebuild a quiesced strand's runtime
454
+ * without the caller re-threading storage/network/profile/key/sApp config.
455
+ */
456
+ private launchConfigs: Map<string, StartStrandConfig> = new Map();
457
+ /**
458
+ * The per-strand peer-join block catch-up, keyed by strand id. Private (not on
459
+ * the public {@link StrandInstance}) because it is runtime plumbing with the
460
+ * same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
461
+ * stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
462
+ * with a fresh caught-up-peer memo, which is intended (a resumed node may have
463
+ * missed writes).
464
+ */
465
+ private backfills: Map<string, PeerJoinBackfill> = new Map();
466
+ /**
467
+ * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
468
+ * id. Same lifecycle rationale as {@link backfills}: created in
469
+ * `buildStrandRuntime`, started in {@link publishDatabase}, stopped and dropped
470
+ * in `releaseRuntime`, so quiesce → resume rebuilds it with a fresh (initially
471
+ * empty, fail-open) snapshot.
472
+ */
473
+ private revocationEnforcers: Map<string, StrandRevocationEnforcer> = new Map();
474
+ /**
475
+ * The per-strand membership reconciler (closed strands launched with a party
476
+ * key only), keyed by strand id. Same lifecycle rationale as {@link backfills}:
477
+ * created in `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so
478
+ * quiesce → resume rebuilds it and re-runs the idempotent join ladder (redeem a
479
+ * staged invitation, write this machine's own `MemberPeer` binding) from scratch.
480
+ */
481
+ private membershipReconcilers: Map<string, StrandMembershipReconciler> = new Map();
482
+ /**
483
+ * The per-strand first-sync gate (`strand-first-sync-gate.ts`), keyed by strand id:
484
+ * holds a launch's `StrandDatabase` while it is NOT yet published on the instance.
485
+ * Present from the moment the database object exists in `buildStrandRuntime` until
486
+ * {@link publishDatabase} hands it to the app — so it doubles as the rollback handle
487
+ * for a failed `initialize()` — and, for a joiner that holds no `Strand.Header` yet,
488
+ * for as long as the launch stays `'syncing'`. Same lifecycle as {@link backfills}:
489
+ * stopped (its database closed) and dropped in `releaseRuntime`, so a quiesce →
490
+ * resume rebuild re-probes over the same store. An entry and `instance.database`
491
+ * are mutually exclusive.
492
+ */
493
+ private firstSyncGates: Map<string, StrandFirstSyncGate> = new Map();
494
+ /**
495
+ * Callers blocked in {@link whenWritable}, keyed by strand id. Resolved by
496
+ * {@link publishDatabase}, rejected by {@link stopStrand}; each waiter also carries
497
+ * its own timeout. Deliberately NOT cleared by `releaseRuntime`: a waiter outlives a
498
+ * quiesce → resume cycle, since the rebuild is what may finally publish the database.
499
+ */
500
+ private writableWaiters: Map<string, Set<WritableWaiter>> = new Map();
501
+ /**
502
+ * The per-strand relay-reservation supervisors — ONE PER CONFIGURED RELAY, each
503
+ * over the node's own bare `/p2p-circuit` listener for that relay
504
+ * (`strand-network-config.ts`) — keyed by strand id. Same lifecycle rationale as
505
+ * {@link backfills}: started in `buildStrandRuntime` right after the libp2p node
506
+ * exists, stopped FIRST and dropped in `releaseRuntime` (before the database
507
+ * closes and the node stops, so no re-drive dials a node being torn down), so
508
+ * quiesce → resume rebuilds them over the new node. What makes a strand node's
509
+ * lost relay slot come back on its own — the control node has the same loop in
510
+ * `CadreNode.reserveRelays`.
511
+ */
512
+ private relaySupervisors: Map<string, RelayReservationSupervisor[]> = new Map();
513
+ /**
514
+ * The per-strand redial of the strand's other members (`strand-peer-redial.ts`), keyed by
515
+ * strand id. Same lifecycle as {@link relaySupervisors}: started in `buildStrandRuntime` once
516
+ * the libp2p node exists, stopped first in `releaseRuntime`, so a hibernating strand dials
517
+ * nobody and a resume redials over the new node.
518
+ */
519
+ private peerRedials: Map<string, StrandPeerRedial> = new Map();
520
+ /**
521
+ * The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
522
+ * store, resolved once in `startStrand` and held until `stopStrand` disposes it.
523
+ * Private for the same reason `backfills` is: runtime plumbing, not part of the
524
+ * public {@link StrandInstance}.
525
+ *
526
+ * Deliberately NOT touched by `releaseRuntime`: that is what lets a quiesce →
527
+ * resume cycle rebuild the libp2p node over the SAME store, keeping its
528
+ * write-through cache warm (and, on an in-memory backend, keeping the strand's
529
+ * blocks at all). An entry exists iff `instances` does AND the launch config
530
+ * supplied a storage provider.
531
+ *
532
+ * NOTE: a hibernating strand therefore keeps its store — and its share of the
533
+ * process-wide cache pool — resident for as long as the instance is tracked. That
534
+ * is the point (a warm wake), and the pool evicts under pressure, so the cost is
535
+ * one map entry per hibernating strand today. If a device ever hibernates strands
536
+ * by the hundred, revisit: dropping the store at quiesce and paying for a cold
537
+ * wake becomes the better trade.
538
+ */
539
+ private strandStorages: Map<string, IRawStorage> = new Map();
540
+ /**
541
+ * The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
542
+ * `resumeStrand`, or a deferred join's database retry), present until it settles. The
543
+ * instance is tracked for the whole build, with its database not yet constructed or not yet
544
+ * initialized, so a caller that must act on the FINISHED runtime ({@link attachSApp},
545
+ * {@link whenRuntimeBuilt}) waits on this instead. A failed build's rejection belongs to the
546
+ * call that started it.
547
+ */
548
+ private runtimeBuilds: Map<string, Promise<unknown>> = new Map();
549
+ /**
550
+ * The latest resume or quiesce requested per strand id, present until it settles. Each one
551
+ * starts only once the one requested before it has settled, so a wake that lands during a
552
+ * quiesce rebuilds after it, and a quiesce requested during a rebuild releases what that
553
+ * rebuild finished. The WHOLE operation, not its entry in {@link runtimeBuilds}: that settles
554
+ * before a resume's own catch records `'error'`, so a joiner awaiting it could read a stale
555
+ * status.
556
+ */
557
+ private transitions: Map<string, RuntimeTransition> = new Map();
558
+ /**
559
+ * The pending database retry per strand id of a launch whose database could not be built
560
+ * because no seed peer had answered yet ({@link deferUnreachableJoin}). Cleared by
561
+ * {@link releaseRuntime}, so a stop or a quiesce cancels it.
562
+ */
563
+ private unreachableJoinRetries: Map<string, unknown> = new Map();
564
+ private stopping = false;
565
+
566
+ constructor() {
567
+ log('StrandInstanceManager created');
568
+ }
569
+
570
+ /**
571
+ * Get all current strand instances
572
+ */
573
+ getInstances(): Map<string, StrandInstance> {
574
+ return new Map(this.instances);
575
+ }
576
+
577
+ /**
578
+ * Get a specific strand instance
579
+ */
580
+ getInstance(strandId: string): StrandInstance | undefined {
581
+ return this.instances.get(strandId);
582
+ }
583
+
584
+ /**
585
+ * Check if a strand is currently running
586
+ */
587
+ hasStrand(strandId: string): boolean {
588
+ return this.instances.has(strandId);
589
+ }
590
+
591
+ /**
592
+ * Start a new strand instance.
593
+ *
594
+ * A failed launch leaves NOTHING tracked: the instance and its retained launch
595
+ * config are both dropped before the error is rethrown, so the strand id is
596
+ * free for a genuine retry. This matches the pre-registration failure path
597
+ * (a rejected schema signature, which throws before anything is recorded) —
598
+ * both failure modes of this call leave the same residue: none. Callers learn
599
+ * of the failure from the rejected promise (and, on the control-discovered
600
+ * path, from CadreNode's `strand:error` event), not from an error record left
601
+ * behind in `instances`.
602
+ *
603
+ * The strand's raw storage is resolved HERE, once, and held for the instance's
604
+ * lifetime (see `strandStorages`) — `buildStrandRuntime` only reads it, so a
605
+ * hibernation wake never re-enters the embedder's provider.
606
+ */
607
+ async startStrand(config: StartStrandConfig): Promise<StrandInstance> {
608
+ const { strandRow, sAppConfig } = config;
609
+ const strandId = strandRow.Id;
610
+
611
+ if (this.stopping) {
612
+ throw new Error('StrandInstanceManager is stopping');
613
+ }
614
+
615
+ if (this.instances.has(strandId)) {
616
+ // Callers resolve founder-ness BEFORE reaching here (CadreNode.launchStrand) and
617
+ // honor a founder request on a tracked instance via foundExistingStrand — so a
618
+ // founder flag arriving at this early return against a non-founder retained
619
+ // config is a dropped bootstrap, and must never again be silent.
620
+ if (config.founder === true && this.launchConfigs.get(strandId)?.founder !== true) {
621
+ log('startStrand: strand %s is already running but was NOT launched as founder — ' +
622
+ 'this early return DROPS the founder request; use foundExistingStrand', strandId);
623
+ }
624
+ log('Strand %s already running', strandId);
625
+ return this.instances.get(strandId)!;
626
+ }
627
+
628
+ log('Starting strand instance: %s (%s)', strandId, describeSApp(sAppConfig));
629
+ const tTotal = performance.now();
630
+
631
+ // The id becomes two names below: the storage scope key the embedder's provider
632
+ // turns into a directory or database name, and `networkName` in buildStrandRuntime,
633
+ // from which the libp2p protocol prefix `/optimystic/strand-<id>` is built. A
634
+ // replicated strand row carries whatever id the founding node wrote, so neither is
635
+ // safe unchecked. Asserted HERE rather than in `resolveStrandStorage` because that
636
+ // function returns early when no provider is configured — a node with no storage
637
+ // still reaches buildStrandRuntime and still mints the protocol prefix.
638
+ assertStrandScopeKey(strandId);
639
+
640
+ const sAppInfo = sAppConfig ? verifiedSAppInfo(strandId, sAppConfig, config.requireSignedSchemas) : undefined;
641
+
642
+ // Resolve this strand's storage ONCE, before anything is recorded. If a factory
643
+ // function is provided, it is called with the strandId to create strand-specific
644
+ // storage (e.g. strand-isolated directories). A provider that throws therefore
645
+ // fails the launch alongside the schema-signature check, leaving nothing tracked.
646
+ const strandStorage = resolveStrandStorage(config.storage?.provider, strandId);
647
+ if (strandStorage) {
648
+ log('Strand %s using provided storage provider', strandId);
649
+ }
650
+
651
+ // Determine latency hint: sApp config > default
652
+ const latencyHint = sAppConfig?.latencyHint ?? config.defaultLatencyHint;
653
+
654
+ const instance: StrandInstance = {
655
+ strandId,
656
+ status: 'starting',
657
+ sAppInfo,
658
+ memberPrivateKey: strandRow.MemberPrivateKey ?? undefined,
659
+ connectedPeers: 0,
660
+ lastActivity: new Date(),
661
+ latencyHint
662
+ };
663
+
664
+ this.instances.set(strandId, instance);
665
+ this.launchConfigs.set(strandId, config);
666
+ if (strandStorage) {
667
+ this.strandStorages.set(strandId, strandStorage);
668
+ }
669
+
670
+ try {
671
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, config));
672
+ timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
673
+ log('Strand %s started successfully (%s)', strandId, describeSApp(sAppConfig));
674
+ return instance;
675
+ } catch (error) {
676
+ // Status/error first — the (now discarded) record is still what `log` reports on.
677
+ instance.status = 'error';
678
+ instance.error = error instanceof Error ? error.message : String(error);
679
+ log('Failed to start strand %s: %s', strandId, instance.error);
680
+ // Drop the dead record so this strand id can be launched again. Keep the
681
+ // `launchConfigs` has an entry iff `instances` does invariant — resumeStrand
682
+ // reads both, and a config without an instance would strand the config.
683
+ this.instances.delete(strandId);
684
+ this.launchConfigs.delete(strandId);
685
+ // The store this launch resolved goes with it: nothing owns it any more, and a
686
+ // retained cache wrapper would be handed back (already retired) on a retry.
687
+ await this.disposeStrandStorage(strandId);
688
+ throw error;
689
+ }
690
+ }
691
+
692
+ /** Hold `build` in {@link runtimeBuilds} until it settles, and settle as it does. */
693
+ private async trackRuntimeBuild<T>(strandId: string, build: Promise<T>): Promise<T> {
694
+ this.runtimeBuilds.set(strandId, build);
695
+ try {
696
+ return await build;
697
+ } finally {
698
+ if (this.runtimeBuilds.get(strandId) === build) {
699
+ this.runtimeBuilds.delete(strandId);
700
+ }
701
+ }
702
+ }
703
+
704
+ /**
705
+ * Wait until no runtime build of `strandId` is in flight. Never rejects: a failed build
706
+ * is reported by the call that started it, and the caller re-reads the instance after.
707
+ *
708
+ * NOTE: after a failed launch the re-read sees the instance gone only because
709
+ * `startStrand`'s catch deletes it synchronously, and its continuation is queued before
710
+ * this waiter's. If that cleanup ever awaits before `instances.delete`, a waiter can act
711
+ * on the dying record; move the cleanup inside the tracked promise then.
712
+ */
713
+ private async settleRuntimeBuilds(strandId: string): Promise<void> {
714
+ for (let build = this.runtimeBuilds.get(strandId); build; build = this.runtimeBuilds.get(strandId)) {
715
+ await build.catch((error: unknown) => {
716
+ log('Strand %s: the runtime build waited on failed (reported by its launcher): %o', strandId, error);
717
+ });
718
+ }
719
+ }
720
+
721
+ /**
722
+ * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
723
+ * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
724
+ * launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
725
+ * volatile inputs (bootstrapNodes, servingMachines, network, profile,
726
+ * privateKey, sApp config) from `config`, so the caller controls the
727
+ * cohort-derived values. Storage is the
728
+ * one input it does NOT re-read from `config`: that belongs to the instance and
729
+ * comes from `strandStorages`.
730
+ *
731
+ * A database that cannot be built because no seed peer has answered yet does not fail the
732
+ * build: the runtime comes up `'syncing'` without it and the database phase alone is
733
+ * retried on the same node ({@link deferUnreachableJoin}).
734
+ */
735
+ private async buildStrandRuntime(instance: StrandInstance, config: StartStrandConfig): Promise<void> {
736
+ const strandId = instance.strandId;
737
+
738
+ // Refuse a bad `linkRoundTripMs` before any strand bring-up, for the reason
739
+ // `CadreNode.start()` does: every budget derived from it here is behind a condition
740
+ // (backfill disabled, no relay addrs), so a zero or NaN declaration would otherwise
741
+ // surface later inside a path that logs and carries on. `link-budget.ts`.
742
+ resolveLinkRoundTripMs(config.network?.linkRoundTripMs);
743
+
744
+ // The store the instance OWNS (resolved once in `startStrand`), not a fresh
745
+ // resolution: a rebuild must reach the same backend through the same warm cache.
746
+ const strandStorage = this.strandStorages.get(strandId);
747
+ // Whether this runtime arms the peer-join block catch-up (below, after the database
748
+ // phase). Decided once because the node options read it too: they turn Optimystic's
749
+ // rebalance reaction off exactly where the catch-up runs.
750
+ const backfillArms = strandStorage !== undefined && config.backfill?.enabled !== false;
751
+
752
+ // db-p2p namespaces every one of the node's protocol ids by network name
753
+ // (`/optimystic/<networkName>/...`), so anything dialing this node's own
754
+ // services must derive its prefix from the SAME string the node was built
755
+ // with — hence one binding for both, not two literals that can drift.
756
+ const networkName = `strand-${strandId}`;
757
+ const protocolPrefix = `/optimystic/${networkName}`;
758
+
759
+ // Whether this node runs the circuit-relay server, and its init — the same
760
+ // resolution the control node takes from the same config (`relay-server.ts`).
761
+ const relayServer = resolveRelayServer(config.network, config.profile);
762
+ // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
763
+ // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
764
+ // bind one port), the announce config is dropped (it names the control node's
765
+ // address), and every configured relay becomes one bare `/p2p-circuit` SEARCH
766
+ // listener plus a dial addr in `relayAddrs` for the per-relay supervisor started
767
+ // below — the same route the control node takes, because the configured
768
+ // `<relay>/p2p-circuit` shape loses its address on a relay restart, a hangup, or
769
+ // libp2p's own reservation refresh and recovers from none of them. See
770
+ // `strand-network-config.ts` for all three, and `relay-reservation.ts`.
771
+ //
772
+ // `relayAddrs` is destructured OFF: it is not a `createLibp2pNode` option, and
773
+ // spreading it would hand db-p2p a key it does not know.
774
+ const { relayAddrs: relayDialAddrs = [], ...addrOptions } = strandNodeAddrs(config.network);
775
+
776
+ // Bound once, before anything is built — so, like the round-trip check above, a
777
+ // rejected clusterSize fails the build with nothing to release. The breadth is also
778
+ // the ceiling on the repair yardstick and the revocation enforcer's wide-strand
779
+ // test below, and all three must be derived from the same resolution.
780
+ const strandClusterSize = resolveStrandClusterSize(config.clusterSize);
781
+
782
+ // The CLOSED-strand revoked-peer gate (see strand-revocation-enforcer.ts).
783
+ // Constructed BEFORE the libp2p node because the node's options embed its
784
+ // predicates; its deny set starts empty (admit everything — fail-open, same
785
+ // posture as bring-up everywhere here) and is first populated once
786
+ // `publishDatabase` starts it. Registered in the map immediately so the
787
+ // failure rollback (releaseRuntime) tears it down like every other runtime
788
+ // component, and so every publish path finds it. Open strands skip it
789
+ // entirely: their Member/MemberPeer tables are empty by schema
790
+ // (`OnlyClosed`), so there is nothing to derive a deny set from.
791
+ const liveDatabase = (): StrandDatabase => {
792
+ if (!instance.database) {
793
+ throw new Error(`strand ${strandId} has no live database`);
794
+ }
795
+ return instance.database;
796
+ };
797
+ const revocationEnforcer =
798
+ config.strandRow.Type === 'c' && config.revocationEnforcement?.enabled !== false
799
+ ? new StrandRevocationEnforcer({
800
+ label: strandId,
801
+ readRows: () => readStrandRevocationRows(liveDatabase().getDatabase()),
802
+ // The tables `readStrandRevocationRows` reads. Called by `start()`, which
803
+ // `publishDatabase` runs right after setting `instance.database`.
804
+ watchChanges: (onChange, options) => liveDatabase().watchTables(['Strand.Member', 'Strand.MemberPeer'], onChange, options),
805
+ cohortBreadth: strandClusterSize,
806
+ // Read per sweep, never captured: this closure is built BEFORE the
807
+ // libp2p node exists (its options embed the enforcer's predicates),
808
+ // and `releaseRuntime` clears the field again on quiesce. Undefined
809
+ // therefore means "no transport right now", which the enforcer
810
+ // treats as "nothing to tear down".
811
+ getNetwork: () => instance.libp2pNode,
812
+ onSelfRevoked: () => config.onSelfRevoked?.(strandId)
813
+ }, config.revocationEnforcement)
814
+ : undefined;
815
+ if (revocationEnforcer) {
816
+ this.revocationEnforcers.set(strandId, revocationEnforcer);
817
+ }
818
+
819
+ // Closed strands compose revoked-peer denial onto the caller's gater and arm
820
+ // the fail-closed per-stream gate; open strands keep the raw configured gater
821
+ // (their peers are legitimately cross-party and nothing is ever revoked).
822
+ const revocationGateOptions = revocationEnforcer
823
+ ? {
824
+ connectionGater: createRevocationConnectionGater(revocationEnforcer, config.network?.connectionGater),
825
+ authorizeInboundStream: (remotePeerId: string, protocol: string) =>
826
+ revocationEnforcer.authorizeStream(remotePeerId, protocol)
827
+ }
828
+ : config.network?.connectionGater
829
+ ? { connectionGater: config.network.connectionGater }
830
+ : {};
831
+
832
+ const reactivityOption = strandCohortTopicOption(config.reactivity, strandId);
833
+ if (reactivityOption.cohortTopic) {
834
+ log('strand %s: change notifications on', strandId);
835
+ }
836
+
837
+ try {
838
+ let t0 = performance.now();
839
+ const node = await createLibp2pNode({
840
+ port: 0, // Random port
841
+ bootstrapNodes: config.bootstrapNodes ?? [],
842
+ networkName,
843
+ storage: strandStorage,
844
+ fretProfile: config.profile === 'storage' ? 'core' : 'edge',
845
+ relay: relayServer.enabled,
846
+ // NOTE: unlike the control node, a strand node has no unauthorized-reservation budget,
847
+ // so with the per-connection cap off any peer that reaches this node can hold one of
848
+ // `maxReservations` slots and forward uncapped through it. If strand relays are ever
849
+ // abused for bandwidth, give strand nodes a budget or a capped init of their own.
850
+ ...(relayServer.enabled && { relayServerInit: relayServer.init }),
851
+ clusterSize: strandClusterSize,
852
+ // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
853
+ // breadth reasoning, and the shape match with the control policy is a coincidence.
854
+ //
855
+ // The builder declares this node's block-repair corroboration yardstick from the
856
+ // count of machines SERVING this strand, capped at the breadth above (a block
857
+ // never lives on more machines than the cohort is wide). Given no count — the
858
+ // production path today, since no per-strand serving count exists yet — it
859
+ // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
860
+ // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
861
+ // serving set that changed while the strand slept.
862
+ // The read deadline is settled by the same helper the control node uses, because the
863
+ // two networks ride one link: the host's own, else derived from its declared
864
+ // `linkRoundTripMs`, else nothing, and the base policy's COHORT_READ_DEADLINE_MS stands.
865
+ clusterPolicy: strandClusterPolicy(strandClusterSize, {
866
+ servingMachines: config.servingMachines,
867
+ cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(config.network)
868
+ }),
869
+ arachnode: {
870
+ enableRingZulu: config.profile === 'storage'
871
+ },
872
+ // One join catch-up per node. On a storage-profile node Optimystic's rebalance growth
873
+ // arm pushes a joiner every block whose cohort it entered, one request per block, and
874
+ // it never hears of the peer-join catch-up's pushes, so each block would go twice. The
875
+ // catch-up pushes every block to every peer it serves, the growth arm's targets
876
+ // included. With the catch-up disabled, the reaction stays on as the only one.
877
+ //
878
+ // A peer that leaves the cohort and comes back is still re-pushed what it missed while
879
+ // away: the catch-up re-runs on each reconnect (`peer-join-backfill.ts`).
880
+ //
881
+ // NOTE: this also leaves the node at its bootstrap Arachnode ring (no ring shifts) and
882
+ // never marks a shed block reclaimable; neither is used today. If storage nodes ever
883
+ // need ring shifts or storage reclamation, drop this: the catch-up already reports its
884
+ // accepted pushes to the rebalance monitor (`reportBlockHeld`); measure whether the
885
+ // growth arm still re-sends.
886
+ ...(backfillArms && { rebalance: { enabled: false } }),
887
+ ...(config.privateKey && { privateKey: config.privateKey }),
888
+ // Built per runtime, not per launch: the adapter stops saving once the strand's
889
+ // state is forgotten, and a rebuilt runtime is what starts it saving again.
890
+ //
891
+ // NOTE: the saved state also restores which peers db-p2p saw serving the strand,
892
+ // and it counts them in the cohort until FRET finds them unreachable. A node built
893
+ // while every other member is offline therefore fails its first writes ("Failed to
894
+ // get super-majority") — for 4-6 s on loopback, the window a running node already
895
+ // sees when a member drops. If apps report failed sends after relaunching alone on
896
+ // a slow relayed link, retry the write in the app or raise it upstream (db-p2p's
897
+ // remembered-serving fallback); dropping `persistence` brings back the split
898
+ // restart.
899
+ ...(config.networkState && { persistence: strandNetworkStatePersistence(config.networkState, strandId) }),
900
+ // With relays configured, built with libp2p's relay discovery off (given a transport
901
+ // list to wrap — `node-transports.ts`), so the supervisors below are the only thing
902
+ // that fills the bare `/p2p-circuit` slots: otherwise a cohort seed running the relay
903
+ // server takes one as soon as it connects (`relay-reservation.ts`, "WHY LIBP2P'S
904
+ // RELAY DISCOVERY IS SWITCHED OFF").
905
+ ...nodeTransportsOption(config.network, addrOptions, relayDialAddrs.length > 0),
906
+ ...(config.network?.noiseCrypto && { noiseCrypto: config.network.noiseCrypto }),
907
+ // Unconditional, and the same default the control node takes: every node of
908
+ // every party runs the monitor over its connection to a slow phone, so the
909
+ // widened ping deadline has to reach the strand nodes too (see
910
+ // DEFAULT_CONNECTION_MONITOR).
911
+ connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
912
+ // The same declared link the control node states, and for the same reason: a strand
913
+ // node is also the listener for every other member's strand node, and Optimystic
914
+ // derives that listener's limit from this (`link-budget.ts`).
915
+ linkRoundTripMs: resolveLinkRoundTripMs(config.network?.linkRoundTripMs),
916
+ // The same dial limits as the control node: a strand node dials the same machines,
917
+ // through the same relays and gates.
918
+ ...optimysticDialLimits(config.network?.linkRoundTripMs),
919
+ // Listen entries plus the WebSocket transport switch they imply — a strand node
920
+ // announces nothing the operator configured (`strand-network-config.ts`), and
921
+ // spreads AFTER `transports` above because the switch is a no-op whenever a
922
+ // transport list was passed. The bare `/p2p-circuit` entries (one per relay) open
923
+ // no connection here; each registers a pending reservation that the matching
924
+ // supervisor below fills. That is what gives a NAT'd strand node a reachable relay
925
+ // slot — and works against a party-run relay because the launch path announced
926
+ // this strand's derived peerId to it first (delegate admission; see cadre-node.ts).
927
+ ...addrOptions,
928
+ // The raw configured gater (open strands), or the revocation-composed
929
+ // gater plus the fail-closed per-stream revoked-peer gate (closed
930
+ // strands) — resolved above, before the try.
931
+ ...revocationGateOptions,
932
+ // `{}` unless the operator enabled change notifications for this strand (`strand-reactivity.ts`).
933
+ ...reactivityOption
934
+ }) as Libp2pNodeWithRepo;
935
+ timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
936
+
937
+ instance.libp2pNode = node;
938
+
939
+ // One reservation supervisor PER RELAY, started now so the first attempts
940
+ // overlap the database bring-up below (the relay is never in a strand's
941
+ // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
942
+ // — so an open relay connection cannot disturb that bring-up). Their first
943
+ // attempts are awaited before the strand goes `active`, below. Registered in the
944
+ // map immediately so the failure rollback (releaseRuntime) stops them like every
945
+ // other runtime component.
946
+ this.relaySupervisors.set(strandId, this.startRelaySupervisors(strandId, node, relayDialAddrs, config));
947
+ // Redials the strand's other members whenever none is connected, on the budget the
948
+ // declared link gives a relayed dial — what reconnects two restarted phones on a slow
949
+ // link, where FRET's own redial cannot (`strand-peer-redial.ts`). The relays are
950
+ // excluded by peer id: the node is connected to them throughout, and they are no member.
951
+ this.peerRedials.set(strandId, superviseStrandPeerRedial({
952
+ label: strandId,
953
+ node,
954
+ seed: config.bootstrapNodes ?? [],
955
+ excluded: new Set(groupAddrsByPeerId([...relayDialAddrs]).keys()),
956
+ ...(revocationEnforcer && { isRevoked: (peerId: string) => revocationEnforcer.isRevoked(peerId) })
957
+ }, strandRedialTimings(config.network?.linkRoundTripMs)));
958
+
959
+ const databaseBuild = await this.buildStrandDatabase(instance, config, node);
960
+ if (!this.isCurrentRuntime(instance, node)) {
961
+ // A stop released this runtime during the database phase (`stopStrand` does not
962
+ // wait for builds); arming the loops below would restart them on a stopped node.
963
+ log('Strand %s was stopped while its runtime was being built', strandId);
964
+ return;
965
+ }
966
+
967
+ // The membership reconciler: finish this party's join on every machine —
968
+ // redeem a staged formation invitation if one is pending, then write the
969
+ // durable machine→party binding (Strand.MemberPeer). Armed here, the one
970
+ // seam where the strand's transport peer id exists and its database has been
971
+ // attempted, for launch AND hibernation resume, for closed strands that carry the
972
+ // party's own membership key. start() kicks an immediate pass WITHOUT
973
+ // awaiting it — a joiner at this instant has not synced the founder's rows
974
+ // and may lack write quorum, so bring-up is never blocked; an unfinished join
975
+ // then retries on a short doubling ladder capped at the configured
976
+ // pollIntervalMs (which is also the flat cadence while nobody has admitted this
977
+ // party yet), and the loop stops once the member row and this machine's own
978
+ // binding are both in place. On a founder the founding transaction already wrote both
979
+ // (`bindFounderPeer`), so the first pass finds them and stops. Armed AFTER the database
980
+ // phase above, so a gated or deferred launch always has a reconciler registered by the
981
+ // time it can publish.
982
+ if (config.strandRow.Type === 'c' && config.partyMemberPrivateKey
983
+ && config.membershipReconciliation?.enabled !== false) {
984
+ const reconciler = new StrandMembershipReconciler({
985
+ label: strandId,
986
+ partyMemberPrivateKey: config.partyMemberPrivateKey,
987
+ // Read per pass, never captured — same lifecycle argument as the
988
+ // enforcer's getNetwork: quiesce drops both handles.
989
+ getDatabase: () => instance.database?.getDatabase(),
990
+ getOwnPeerId: () => instance.libp2pNode?.peerId.toString(),
991
+ pendingInvite: config.pendingMembershipInvite,
992
+ // The enforcer's CURRENT snapshot view of this node's own peer id — the
993
+ // reconciler stops rather than fight a self-revocation. Absent when the
994
+ // gate is disarmed (fail-open: the loop just runs).
995
+ isSelfRevoked: revocationEnforcer
996
+ ? () => {
997
+ const ownPeerId = instance.libp2pNode?.peerId.toString();
998
+ return ownPeerId !== undefined && revocationEnforcer.isRevoked(ownPeerId);
999
+ }
1000
+ : undefined,
1001
+ onRejoinBlocked: () => config.onRejoinBlocked?.(strandId)
1002
+ }, config.membershipReconciliation);
1003
+ this.membershipReconcilers.set(strandId, reconciler);
1004
+ reconciler.start();
1005
+ }
1006
+
1007
+ // Peer-join block catch-up: push this strand's own blocks to each newly
1008
+ // connected peer, so a machine that joined after blocks were committed
1009
+ // still ends up physically holding them (without per-strand storage there
1010
+ // is nothing to copy). Armed for EVERY stored strand: `PeerJoinBackfill`
1011
+ // only does work when the strand's libp2p node reports a peer connection,
1012
+ // so on a device that is genuinely alone it is inert, and arming it at
1013
+ // launch is what closes the "founded alone, never replicates" hole — a
1014
+ // peer that joins later gets the founder's blocks without any relaunch.
1015
+ // No `authorizePeer` gate, deliberately — see the module comment in
1016
+ // peer-join-backfill.ts for the strand-side argument (and why the control
1017
+ // network, which DOES gate, is different).
1018
+ //
1019
+ // NOTE: cost is one PeerJoinBackfill object + one `peer:identify` listener
1020
+ // per running strand — linear in strand count, negligible at the handful a
1021
+ // device or host runs today. If a node ever hosts strands by the hundred,
1022
+ // move to one shared listener that dispatches by strand id.
1023
+ if (backfillArms) {
1024
+ if (node.keyNetwork) {
1025
+ const backfill = new PeerJoinBackfill({
1026
+ label: strandId,
1027
+ libp2p: node,
1028
+ peerNetwork: node.keyNetwork,
1029
+ storage: strandStorage,
1030
+ // The same prefix the receiver registered its block-transfer handler
1031
+ // under — derived from networkName above, never re-spelled here.
1032
+ protocolPrefix,
1033
+ reportBlockHeld: node.reportBlockHeld
1034
+ }, {
1035
+ // Dial and response deadlines counted in link round trips, not fixed milliseconds:
1036
+ // a relayed dial costs a fixed number of exchanges, so a fixed budget has a link
1037
+ // speed above which this catch-up can never reach the peer at all. See
1038
+ // `link-budget.ts`. Spread BEFORE the host's own config so an explicit
1039
+ // `strandBackfill.dialTimeoutMs` still wins.
1040
+ ...peerJoinPushBudget(config.network?.linkRoundTripMs),
1041
+ ...config.backfill
1042
+ });
1043
+ backfill.start();
1044
+ this.backfills.set(strandId, backfill);
1045
+ } else {
1046
+ log('Strand %s: libp2p node exposes no keyNetwork; peer-join block catch-up is inert', strandId);
1047
+ }
1048
+ }
1049
+
1050
+ // The relay supervisors' FIRST attempts, all of them, before `active`: the happy
1051
+ // path still publishes its circuit addr before `addStrand` resolves (the
1052
+ // strand-addr RPC answers read it, and the same-party circuit scenario asserts
1053
+ // it). Fail-SOFT — a first attempt that lands nothing does not fail the launch.
1054
+ // The strand's database is up, and the supervisor keeps trying on its backoff;
1055
+ // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
1056
+ //
1057
+ // NOTE: a relay that is down costs this launch one full drive — the reservation budget
1058
+ // counted from the declared link round trip, 18 s at its default (`link-budget.ts`) — and
1059
+ // `StrandWatcher` launches strands one at a time, so N strands cost N of those in
1060
+ // bring-up during a relay outage, and MORE on a host that declared a slower link. If that
1061
+ // ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
1062
+ // than shortening the drive.
1063
+ t0 = performance.now();
1064
+ await this.awaitFirstRelayAttempts(strandId);
1065
+ timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
1066
+
1067
+ // No `lastActivity` stamp: bringing a runtime up is not activity. A check-in marks
1068
+ // activity before its resume and must see only what landed while the strand was up.
1069
+ instance.status = liveStrandStatus(instance);
1070
+ if (databaseBuild === 'awaiting-seed') {
1071
+ // Armed only now that the build is over: a retry firing mid-build would find the
1072
+ // build in flight.
1073
+ this.deferUnreachableJoin(instance);
1074
+ } else if (instance.status === 'syncing') {
1075
+ log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
1076
+ 'a member of the strand is reached', strandId);
1077
+ }
1078
+ } catch (error) {
1079
+ // Roll back any partially-attached runtime so the instance is left with
1080
+ // NEITHER handle. Otherwise the `libp2pNode || database` "already live"
1081
+ // guard in resumeStrand/handleStrandWake would treat a half-built strand
1082
+ // as healthy — leaking the libp2p node and never retrying the rebuild.
1083
+ await this.releaseRuntime(instance).catch((cleanupErr) => {
1084
+ log('buildStrandRuntime cleanup for strand %s also failed: %o', strandId, cleanupErr);
1085
+ });
1086
+ throw error;
1087
+ }
1088
+ }
1089
+
1090
+ /**
1091
+ * The database phase of a runtime build, on the libp2p node already attached: create the
1092
+ * StrandDatabase, initialize it, then publish it or arm its first-sync gate. Run by
1093
+ * {@link buildStrandRuntime} and again by each {@link retryUnreachableJoin}, with a new
1094
+ * StrandDatabase per attempt.
1095
+ *
1096
+ * Returns `'awaiting-seed'` when initialize failed only because no seed peer has answered
1097
+ * (`cohort-unreachable`): that attempt's database is closed and the node keeps running. Any
1098
+ * other failure throws, with the attempt's database left with its gate for
1099
+ * {@link releaseRuntime} to close.
1100
+ */
1101
+ private async buildStrandDatabase(
1102
+ instance: StrandInstance,
1103
+ config: StartStrandConfig,
1104
+ node: Libp2pNodeWithRepo
1105
+ ): Promise<StrandDatabaseBuild> {
1106
+ const strandId = instance.strandId;
1107
+ // Held by the first-sync gate (registered before initialize) rather than attached
1108
+ // to the instance: a failed init is then cleaned up through the gate (close() is safe on
1109
+ // a partially-initialized db), and the app only ever sees `instance.database` once this
1110
+ // machine is allowed to write to the strand.
1111
+ let t0 = performance.now();
1112
+ const strandDb = new StrandDatabase({
1113
+ strandId,
1114
+ sAppConfig: config.sAppConfig,
1115
+ libp2pNode: node,
1116
+ coordinatedRepo: node.coordinatedRepo,
1117
+ // Founder bootstrap inputs: the strand's type drives which membership rows
1118
+ // are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
1119
+ // caller from the control-layer StrandPartyKey row) derives the founding
1120
+ // Member/Manager key. The row's shared MemberPrivateKey rides along as the
1121
+ // read-gating secret only — it derives nobody's identity.
1122
+ strandType: config.strandRow.Type,
1123
+ memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
1124
+ partyMemberPrivateKey: config.partyMemberPrivateKey,
1125
+ founder: config.founder,
1126
+ // The founding transaction carries this machine's MemberPeer binding unless the
1127
+ // reconciler that would otherwise write it is disarmed (see buildStrandRuntime).
1128
+ bindFounderPeer: config.membershipReconciliation?.enabled !== false,
1129
+ onCommitDurability: config.onCommitDurability,
1130
+ onCommitHeld: config.onCommitHeld
1131
+ });
1132
+ const storageRepo = node.storageRepo;
1133
+ const gate = new StrandFirstSyncGate({
1134
+ label: strandId,
1135
+ database: strandDb,
1136
+ onHeaderHeld: () => this.publishDatabase(instance, strandDb),
1137
+ subscribeStorageChanges: storageRepo && ((onChange) => storageRepo.onAnyCollectionChange(onChange))
1138
+ }, config.firstSync);
1139
+ this.firstSyncGates.set(strandId, gate);
1140
+ // Judged before initialize, not at the first probe: initialize itself fetches and keeps the
1141
+ // schema catalog's blocks (three on a first join of the message-cost strand), so the store is
1142
+ // never empty by then. A first join listens from here, so a member's push landing while the
1143
+ // database initializes is heard (initialize's own catalog fetch raises no change event). A
1144
+ // node with no change feed could not hear that push, so it probes at once as before.
1145
+ const firstJoin = config.founder !== true && storageRepo !== undefined && await this.strandStoreEmpty(strandId);
1146
+ if (firstJoin) {
1147
+ gate.listen();
1148
+ }
1149
+ try {
1150
+ await strandDb.initialize();
1151
+ } catch (error) {
1152
+ if (!isCohortUnreachableRead(error)) {
1153
+ throw error;
1154
+ }
1155
+ await this.discardDatabaseAttempt(strandId, gate);
1156
+ log('Strand %s: no seed peer has answered yet, so this joiner cannot build its database (%s); ' +
1157
+ 'its runtime stays up and the build is retried every %dms',
1158
+ strandId, error instanceof Error ? error.message : String(error), UNREACHABLE_JOIN_RETRY_MS);
1159
+ return 'awaiting-seed';
1160
+ }
1161
+ timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
1162
+ if (this.firstSyncGates.get(strandId) !== gate) {
1163
+ // A stop released the runtime while initialize ran, closing this database through its
1164
+ // gate part-way; close what initialize finished after that.
1165
+ await strandDb.close();
1166
+ return 'released';
1167
+ }
1168
+
1169
+ // The first-sync write gate. A founder just wrote the Header in its bootstrap; any
1170
+ // other machine may commit only once it holds the Header — which it can only have
1171
+ // received from a peer (now, or on an earlier run over the same store) — and has read
1172
+ // each other Strand table and each App table once, so their collections are fetched
1173
+ // too. Publishing the database is what makes the strand writable to the app; until
1174
+ // then the instance stays `'syncing'` and the gate re-probes when data lands. This first
1175
+ // probe goes through the gate, so what it finds held is not read again. Everything armed
1176
+ // after the database phase reads `instance.database` lazily, so the reconciler and
1177
+ // enforcer wait with it — and `publishDatabase` kicks the reconciler when the wait ends,
1178
+ // so a joiner's membership rows land with the strand rather than one retry later.
1179
+ //
1180
+ // A first join (the store held nothing when this build began) on which nothing has landed yet
1181
+ // skips this probe: a member's peer-join catch-up is pushing it the same blocks, and the probe
1182
+ // would fetch them one read at a time meanwhile. The gate probes when that push lands, or at
1183
+ // its backstop. A first join the push already reached, and a store that held anything (a
1184
+ // restart, a hibernation wake), probe at once, so re-attaching offline is not delayed.
1185
+ t0 = performance.now();
1186
+ const awaitPush = firstJoin && !gate.heardDataSinceProbe;
1187
+ if (awaitPush) {
1188
+ log('Strand %s: first join over an empty store; waiting for a member\'s push before the first sync probe', strandId);
1189
+ }
1190
+ const synced = config.founder === true || (!awaitPush && await gate.probeOnce());
1191
+ timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
1192
+ if (this.firstSyncGates.get(strandId) !== gate) {
1193
+ // A stop released the runtime during the probe and closed this database through its gate.
1194
+ return 'released';
1195
+ }
1196
+ if (synced) {
1197
+ this.publishDatabase(instance, strandDb);
1198
+ } else {
1199
+ gate.start();
1200
+ }
1201
+ return 'built';
1202
+ }
1203
+
1204
+ /**
1205
+ * Whether this strand's raw store holds no block. False for a store that cannot list its
1206
+ * blocks, or whose listing fails, so such a launch probes at once.
1207
+ */
1208
+ private async strandStoreEmpty(strandId: string): Promise<boolean> {
1209
+ const storage = this.strandStorages.get(strandId);
1210
+ if (!storage?.listBlockIds) return false;
1211
+ try {
1212
+ for await (const _blockId of storage.listBlockIds()) {
1213
+ return false;
1214
+ }
1215
+ return true;
1216
+ } catch (error) {
1217
+ log('Strand %s: listing the raw store failed; probing first sync at once: %o', strandId, error);
1218
+ return false;
1219
+ }
1220
+ }
1221
+
1222
+ /** Close a database attempt that failed to initialize, and drop its gate if it is still the registered one. */
1223
+ private async discardDatabaseAttempt(strandId: string, gate: StrandFirstSyncGate): Promise<void> {
1224
+ gate.stop();
1225
+ if (this.firstSyncGates.get(strandId) === gate) {
1226
+ this.firstSyncGates.delete(strandId);
1227
+ }
1228
+ await gate.database.close();
1229
+ }
1230
+
1231
+ /**
1232
+ * Keep a launch `'syncing'` when its database could not be built because no seed peer has
1233
+ * answered yet, and retry the database build every {@link UNREACHABLE_JOIN_RETRY_MS} on the
1234
+ * runtime already up.
1235
+ *
1236
+ * Optimystic (since 1.11.0) will not rule a block never-created while any of a node's
1237
+ * `bootstrapNodes` is unheard, so a joiner alone with an unreachable seed cannot build even
1238
+ * the schema catalog: `strandDb.initialize()` fails with `cohort-unreachable`. A founder
1239
+ * given a seed is refused the same way (normally it has none: no sibling runs the strand yet). Refusing is
1240
+ * right (building it alone would fork the catalog), but failing the launch would drop the
1241
+ * strand until the watcher relaunched it, and `addStrand` would reject with a raw Optimystic
1242
+ * error. Deferring gives a joiner the same contract the first-sync gate gives one that did
1243
+ * build: the strand stays tracked, {@link whenWritable} waits and rejects with the retryable
1244
+ * `StrandAwaitingFirstSyncError`, and the launch proceeds once a seed peer answers.
1245
+ *
1246
+ * The libp2p node, its connections and the background loops stay up meanwhile. A seed peer
1247
+ * can answer just after the schema read gave up on it — on a slow link it connects and its
1248
+ * peer-join backfill delivers the whole store — and the retry then builds the database, and
1249
+ * runs the first-sync probe, while that peer is still connected.
1250
+ */
1251
+ private deferUnreachableJoin(instance: StrandInstance): void {
1252
+ const strandId = instance.strandId;
1253
+ const timer = setTimeout(() => {
1254
+ this.unreachableJoinRetries.delete(strandId);
1255
+ void this.retryUnreachableJoin(instance);
1256
+ }, UNREACHABLE_JOIN_RETRY_MS);
1257
+ (timer as { unref?: () => void }).unref?.();
1258
+ this.unreachableJoinRetries.set(strandId, timer);
1259
+ }
1260
+
1261
+ /**
1262
+ * One retry scheduled by {@link deferUnreachableJoin}: the database phase again, on the
1263
+ * runtime still attached. Skipped when the strand was stopped or released since — a quiesce
1264
+ * releases it and the resume after rebuilds the whole runtime — or already has a database.
1265
+ * A failure other than an unanswered seed releases the runtime and marks the strand
1266
+ * `'error'`, as a failed resume does.
1267
+ */
1268
+ private async retryUnreachableJoin(instance: StrandInstance): Promise<void> {
1269
+ const strandId = instance.strandId;
1270
+ const config = this.launchConfigs.get(strandId);
1271
+ // Every strand node is built by `buildStrandRuntime`, as a node with its coordinated repo.
1272
+ const node = instance.libp2pNode as Libp2pNodeWithRepo | undefined;
1273
+ if (!config || !node || !this.isCurrentRuntime(instance, node) || instance.database
1274
+ || this.firstSyncGates.has(strandId) || this.isQuiescing(strandId)) {
1275
+ return;
1276
+ }
1277
+ let build: StrandDatabaseBuild;
1278
+ try {
1279
+ // Tracked like any runtime build, so a wake, a quiesce or an sApp attach waits for it
1280
+ // rather than acting on a database that is still initializing.
1281
+ build = await this.trackRuntimeBuild(strandId, this.buildStrandDatabase(instance, config, node));
1282
+ } catch (error) {
1283
+ if (!this.isCurrentRuntime(instance, node)) {
1284
+ log('Strand %s: a database retry failed after its runtime was released: %o', strandId, error);
1285
+ return;
1286
+ }
1287
+ await this.releaseRuntime(instance).catch((cleanupErr) => {
1288
+ log('Strand %s: releasing the runtime after a failed database retry also failed: %o', strandId, cleanupErr);
1289
+ });
1290
+ instance.status = 'error';
1291
+ instance.error = error instanceof Error ? error.message : String(error);
1292
+ log('Strand %s: building the database of a joiner that was waiting for a seed peer failed: %s', strandId, instance.error);
1293
+ return;
1294
+ }
1295
+ if (build === 'awaiting-seed' && this.isCurrentRuntime(instance, node) && !this.isQuiescing(strandId)) {
1296
+ this.deferUnreachableJoin(instance);
1297
+ }
1298
+ }
1299
+
1300
+ /** Whether `instance` is still tracked, with `node` still its runtime's libp2p node. */
1301
+ private isCurrentRuntime(instance: StrandInstance, node: Libp2p): boolean {
1302
+ return !this.stopping && this.instances.get(instance.strandId) === instance && instance.libp2pNode === node;
1303
+ }
1304
+
1305
+ /**
1306
+ * Release an instance's strand-network runtime: stop the relay supervisors and
1307
+ * the other background loops, close the StrandDatabase, then stop the libp2p
1308
+ * node (construction order in reverse), clearing both fields and zeroing
1309
+ * connectedPeers. Tolerant of partially-built state — either handle may be
1310
+ * absent — so it doubles as rollback for a failed `buildStrandRuntime`. Shared
1311
+ * by `quiesceStrand`, `stopStrand`, and that rollback path.
1312
+ *
1313
+ * Leaves `strandStorages` untouched by design — the store outlives the runtime it
1314
+ * was built into, which is what makes a resume warm. Only `stopStrand` disposes it.
1315
+ */
1316
+ private async releaseRuntime(instance: StrandInstance): Promise<void> {
1317
+ // Relay supervisors FIRST of all — before anything below is torn down — so no
1318
+ // re-drive is scheduled against a node being stopped. `stop()` is synchronous
1319
+ // and never awaits a drive, so a relay that is down cannot delay this teardown.
1320
+ // `stop()` also cancels the drive already in flight, so an attempt started
1321
+ // moments before this does not keep dialing and polling the node the lines
1322
+ // below are stopping; its result is discarded either way.
1323
+ const unreachableJoinRetry = this.unreachableJoinRetries.get(instance.strandId);
1324
+ if (unreachableJoinRetry !== undefined) {
1325
+ clearTimeout(unreachableJoinRetry as ReturnType<typeof setTimeout>);
1326
+ this.unreachableJoinRetries.delete(instance.strandId);
1327
+ }
1328
+ const relaySupervisors = this.relaySupervisors.get(instance.strandId);
1329
+ if (relaySupervisors) {
1330
+ relaySupervisors.forEach((supervisor) => supervisor.stop());
1331
+ this.relaySupervisors.delete(instance.strandId);
1332
+ }
1333
+ this.peerRedials.get(instance.strandId)?.stop();
1334
+ this.peerRedials.delete(instance.strandId);
1335
+ // Backfill next — before the database closes and the libp2p node stops — so
1336
+ // no NEW catch-up push is issued against a torn-down transport. A push already
1337
+ // in flight is not awaited; it fails into the module's own per-chunk catch.
1338
+ const backfill = this.backfills.get(instance.strandId);
1339
+ if (backfill) {
1340
+ backfill.stop();
1341
+ this.backfills.delete(instance.strandId);
1342
+ }
1343
+ // The revoked-peer enforcer goes with the runtime it gated: a resume
1344
+ // rebuilds it with a fresh (initially empty, fail-open) snapshot.
1345
+ const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
1346
+ if (revocationEnforcer) {
1347
+ revocationEnforcer.stop();
1348
+ this.revocationEnforcers.delete(instance.strandId);
1349
+ }
1350
+ // The membership reconciler likewise: a resume rebuilds it and re-runs the
1351
+ // idempotent ladder, which is what makes resume-after-partial-join heal.
1352
+ const membershipReconciler = this.membershipReconcilers.get(instance.strandId);
1353
+ if (membershipReconciler) {
1354
+ membershipReconciler.stop();
1355
+ this.membershipReconcilers.delete(instance.strandId);
1356
+ }
1357
+ // A still-gated database (a joiner that never received the Header, or a launch whose
1358
+ // initialize() failed) is closed through its gate; a published one through the
1359
+ // instance. Never both — publishing moves the handle from the gate to the instance.
1360
+ const gate = this.firstSyncGates.get(instance.strandId);
1361
+ if (gate) {
1362
+ gate.stop();
1363
+ this.firstSyncGates.delete(instance.strandId);
1364
+ await gate.database.close();
1365
+ }
1366
+ if (instance.database) {
1367
+ await instance.database.close();
1368
+ instance.database = undefined;
1369
+ }
1370
+ if (instance.libp2pNode) {
1371
+ await instance.libp2pNode.stop();
1372
+ instance.libp2pNode = undefined;
1373
+ }
1374
+ instance.connectedPeers = 0;
1375
+ }
1376
+
1377
+ /**
1378
+ * One {@link superviseRelayReservation} per relay dial addr, each over exactly
1379
+ * that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
1380
+ * one relay re-drives only that one. They are the only thing that fills the node's
1381
+ * search slots once the node is built with libp2p's relay discovery off
1382
+ * (`nodeTransportsOption`). Default retry timings — the control node's (2 s
1383
+ * doubling to 60 s between failed attempts, a 5 s liveness check while held). Each DRIVE's
1384
+ * own deadline is counted in link round trips at this host's declared
1385
+ * `network.linkRoundTripMs` (`link-budget.ts`), so the same declaration that lengthens the
1386
+ * control node's drive lengthens these.
1387
+ *
1388
+ * Each supervisor's `beforeRedrive` is the caller's
1389
+ * {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
1390
+ * node's peer id, so a re-drive after a relay restart is preceded by a fresh
1391
+ * delegate grant on the relay about to be dialed. Returns `[]` when no relay is
1392
+ * configured — the strand then simply has no circuit listener to fill.
1393
+ */
1394
+ private startRelaySupervisors(
1395
+ strandId: string,
1396
+ node: Libp2pNodeWithRepo,
1397
+ relayDialAddrs: readonly string[],
1398
+ config: StartStrandConfig
1399
+ ): RelayReservationSupervisor[] {
1400
+ if (relayDialAddrs.length === 0) {
1401
+ return [];
1402
+ }
1403
+ const announce = config.announceDelegateToRelay;
1404
+ const delegatePeerId = node.peerId.toString();
1405
+ const timeoutMs = relayReservationBudgetMs(config.network?.linkRoundTripMs);
1406
+ return relayDialAddrs.map((relayAddr) => superviseRelayReservation(node, [relayAddr], {
1407
+ timeoutMs,
1408
+ ...(announce && { beforeRedrive: () => announce(strandId, relayAddr, delegatePeerId) })
1409
+ }));
1410
+ }
1411
+
1412
+ /**
1413
+ * Wait for every relay supervisor's first attempt to settle, concurrently, and log
1414
+ * the ones that landed nothing. Never throws — a strand whose relay is unreachable
1415
+ * at launch still comes up, with its supervisors retrying in the background.
1416
+ */
1417
+ private async awaitFirstRelayAttempts(strandId: string): Promise<void> {
1418
+ const supervisors = this.relaySupervisors.get(strandId) ?? [];
1419
+ if (supervisors.length === 0) {
1420
+ return;
1421
+ }
1422
+ await Promise.all(supervisors.map((supervisor) => supervisor.firstAttempt));
1423
+ supervisors.forEach((supervisor) => {
1424
+ if (supervisor.lastError !== null) {
1425
+ log('Strand %s: relay reservation not held after the first attempt (retrying in the background): %s',
1426
+ strandId, supervisor.lastError);
1427
+ }
1428
+ });
1429
+ }
1430
+
1431
+ /**
1432
+ * Drop the strand's owned store and release this strand's claim on its cache. The
1433
+ * wrapper counts holders, so the cache is emptied and unregistered from the shared
1434
+ * pool only if no other scope still holds it. Called only where the strand's whole
1435
+ * lifetime ends (`stopStrand`, and the failed-launch rollback) — never on a quiesce.
1436
+ *
1437
+ * A dispose failure is logged, not thrown: the store is already unreferenced here,
1438
+ * and failing the stop over a cache-bookkeeping error would leave the caller unable
1439
+ * to tear the strand down. `disposeStorageCache` no-ops for an unwrapped store
1440
+ * (e.g. `MemoryRawStorage`), so no instanceof test is needed.
1441
+ */
1442
+ private async disposeStrandStorage(strandId: string): Promise<void> {
1443
+ const storage = this.strandStorages.get(strandId);
1444
+ if (!storage) {
1445
+ return;
1446
+ }
1447
+ this.strandStorages.delete(strandId);
1448
+ try {
1449
+ await disposeStorageCache(storage);
1450
+ } catch (error) {
1451
+ log('Failed to dispose storage cache for strand %s: %o', strandId, error);
1452
+ }
1453
+ }
1454
+
1455
+ /**
1456
+ * Refresh a strand's revoked-peer deny set NOW — and, with it, run the
1457
+ * teardown sweep that hangs up every connected revoked peer.
1458
+ *
1459
+ * The gate already refreshes on its own when `Strand.Member` or
1460
+ * `Strand.MemberPeer` changes, a local `revokeMember`/`leaveStrand` included
1461
+ * (`strand-revocation-enforcer.ts` → "When the deny set is refreshed"). This
1462
+ * call is for a caller that wants to await the cut: it resolves once the sweep
1463
+ * has run.
1464
+ *
1465
+ * Quiet no-op — matching `quiesceStrand`'s posture — when the strand is not
1466
+ * tracked, is quiesced, is open, or has the gate disabled: in every one of
1467
+ * those cases there is no enforcer to refresh. Never rejects (the enforcer's
1468
+ * refresh contains its own failures), and resolves only once the sweep has
1469
+ * finished.
1470
+ */
1471
+ async refreshRevocationEnforcement(strandId: string): Promise<void> {
1472
+ const enforcer = this.revocationEnforcers.get(strandId);
1473
+ if (!enforcer) {
1474
+ log('refreshRevocationEnforcement: strand %s has no armed revocation enforcer', strandId);
1475
+ return;
1476
+ }
1477
+ await enforcer.refresh();
1478
+ }
1479
+
1480
+ /**
1481
+ * A fresh membership invitation was staged for `strandId` (`CadreNode`'s
1482
+ * `adoptFormationMembershipInvite`): re-arm the strand's membership reconciler so the
1483
+ * invitation is attempted now rather than never — the loop finished during the first
1484
+ * join and would otherwise stay stopped until a relaunch. Quiet no-op when no reconciler
1485
+ * is armed: a first formation stages before the strand is added, and bring-up then arms
1486
+ * a loop that finds the invitation by itself. Returns at once; the pass runs on the
1487
+ * loop's own serialized chain and never rejects.
1488
+ */
1489
+ notifyMembershipInviteStaged(strandId: string): void {
1490
+ const reconciler = this.membershipReconcilers.get(strandId);
1491
+ if (!reconciler) {
1492
+ log('notifyMembershipInviteStaged: strand %s has no armed membership reconciler — bring-up will find the invitation', strandId);
1493
+ return;
1494
+ }
1495
+ void reconciler.rearm();
1496
+ }
1497
+
1498
+ /**
1499
+ * Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
1500
+ * self arm of `removeMemberPeer`, signed with the retained launch config's party
1501
+ * key. Called by `CadreNode.unpublishStrand` BEFORE the local stop, while the
1502
+ * strand database and transport are still live: after the unpublish commits, the
1503
+ * party's `StrandPartyKey` control row is gone, so no restart could ever sign
1504
+ * this removal again — the in-memory retained key is the last chance.
1505
+ *
1506
+ * Never throws. Quiet no-op for a strand that is untracked, open, or launched
1507
+ * without a party key; a quiesced (hibernating) strand, or a strand whose write
1508
+ * quorum is already unreachable, leaves the stale binding behind with a log —
1509
+ * it grants nothing today (admission is not allowlist-gated) and only
1510
+ * mis-credits diversity. The strand's own membership reconciler is stopped —
1511
+ * and its in-flight pass awaited — FIRST, so a pass that was already past its
1512
+ * own stopped check cannot re-register the binding this is about to clear.
1513
+ *
1514
+ * NOTE: deliberately NOT handled here (or anywhere yet): clearing the bindings
1515
+ * of a machine removed from the CADRE at the control layer, or of a party's
1516
+ * OTHER machines when the party unpublishes — the party still holds the key on
1517
+ * every surviving machine and can clear from any of them, and the stale rows
1518
+ * grant nothing while admission is deny-list only. Revisit when
1519
+ * `feat-strand-member-allowlist-admission` lands and a stale binding starts
1520
+ * granting admission rather than merely mis-crediting diversity.
1521
+ */
1522
+ async clearOwnMemberPeerBinding(strandId: string): Promise<void> {
1523
+ const instance = this.instances.get(strandId);
1524
+ const config = this.launchConfigs.get(strandId);
1525
+ if (!instance || !config || config.strandRow.Type !== 'c' || !config.partyMemberPrivateKey) {
1526
+ return;
1527
+ }
1528
+ const reconciler = this.membershipReconcilers.get(strandId);
1529
+ if (reconciler) {
1530
+ reconciler.stop();
1531
+ await reconciler.settle();
1532
+ }
1533
+ const db = instance.database?.getDatabase();
1534
+ const peerId = instance.libp2pNode?.peerId.toString();
1535
+ if (!db || !peerId) {
1536
+ log('clearOwnMemberPeerBinding: strand %s has no live runtime — own MemberPeer binding left in place', strandId);
1537
+ return;
1538
+ }
1539
+ try {
1540
+ // Background write on the app's database: never join a transaction the app has open.
1541
+ await removeMemberPeer(db, {
1542
+ memberKeyPair: strandMemberKeyPair(config.partyMemberPrivateKey),
1543
+ peerId
1544
+ }, { joinOpenTransaction: false });
1545
+ log('clearOwnMemberPeerBinding: strand %s — own binding for peer %s cleared', strandId, peerId);
1546
+ } catch (error) {
1547
+ log('clearOwnMemberPeerBinding: strand %s — best-effort clear failed (strand may already be unreachable): %o',
1548
+ strandId, error);
1549
+ }
1550
+ }
1551
+
1552
+ /**
1553
+ * Quiesce a strand: release its strand-network resources (stop the libp2p node,
1554
+ * close the StrandDatabase) while RETAINING the instance record — identity,
1555
+ * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
1556
+ * resumed later, then mark it `hibernating`. Mechanically this is `stopStrand` minus
1557
+ * the instance/config deletion. A tracked strand with no runtime left (a failed
1558
+ * rebuild already released it) is only marked; an untracked one is left alone.
1559
+ *
1560
+ * Ordered with {@link resumeStrand} (see {@link transitions}): it starts once the resume
1561
+ * or launch build before it has settled, so it releases a finished runtime, and the
1562
+ * status write is its last step, so a resume requested meanwhile writes after it. A call
1563
+ * made while another quiesce is the latest request joins it. A failed release leaves the
1564
+ * status as it was and rejects.
1565
+ */
1566
+ async quiesceStrand(strandId: string): Promise<void> {
1567
+ const previous = this.transitions.get(strandId);
1568
+ if (previous?.kind === 'quiesce') {
1569
+ log('quiesceStrand: strand %s — joining the quiesce already requested', strandId);
1570
+ return previous.operation;
1571
+ }
1572
+ const operation = this.runQuiesce(strandId, previous);
1573
+ this.holdTransition(strandId, { kind: 'quiesce', operation });
1574
+ return operation;
1575
+ }
1576
+
1577
+ /** Body of {@link quiesceStrand}. */
1578
+ private async runQuiesce(strandId: string, previous: RuntimeTransition | undefined): Promise<void> {
1579
+ await this.settleTransition(strandId, previous);
1580
+ await this.settleRuntimeBuilds(strandId);
1581
+ const instance = this.instances.get(strandId);
1582
+ if (!instance) {
1583
+ log('quiesceStrand: strand %s not found', strandId);
1584
+ return;
1585
+ }
1586
+ if (instance.libp2pNode || instance.database) {
1587
+ log('Quiescing strand instance: %s', strandId);
1588
+ await this.releaseRuntime(instance);
1589
+ log('Strand %s quiesced (resources released, instance retained)', strandId);
1590
+ } else {
1591
+ log('quiesceStrand: strand %s already quiesced', strandId);
1592
+ }
1593
+ instance.status = 'hibernating';
1594
+ }
1595
+
1596
+ /**
1597
+ * Whether the latest runtime transition requested for `strandId` is a quiesce that has
1598
+ * not finished. Its handles may still be attached and its status still live, but it is on
1599
+ * its way to `hibernating`, so a caller that needs it up must {@link resumeStrand} it — the
1600
+ * resume runs after the quiesce — rather than treat it as live.
1601
+ */
1602
+ isQuiescing(strandId: string): boolean {
1603
+ return this.transitions.get(strandId)?.kind === 'quiesce';
1604
+ }
1605
+
1606
+ /**
1607
+ * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
1608
+ * from the retained launch config and re-attach them, transitioning it back to
1609
+ * `active`. `overrides` re-applies volatile inputs that may have changed since
1610
+ * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
1611
+ * count) and updates the retained config so a later resume reuses the latest
1612
+ * values. Returns the live instance unchanged if it is already running.
1613
+ *
1614
+ * Overlapping calls share one rebuild: a call made while a resume is the latest request
1615
+ * for the strand — a wake landing during a check-in — joins it and settles exactly as it
1616
+ * does, `'error'` status included. The joiner's own `overrides` are ignored; the first
1617
+ * resume's seed wins, and both callers resolved the cohort seed moments apart. A call made
1618
+ * while a quiesce is the latest request runs after that quiesce and rebuilds (see
1619
+ * {@link transitions}), and one made while `startStrand` is still building waits for that
1620
+ * build rather than starting a second.
1621
+ */
1622
+ async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
1623
+ if (this.stopping) {
1624
+ throw new Error('StrandInstanceManager is stopping');
1625
+ }
1626
+ // No `await` between this read and `holdTransition`, or a second caller slips through
1627
+ // and builds.
1628
+ const previous = this.transitions.get(strandId);
1629
+ if (previous?.kind === 'resume') {
1630
+ log('resumeStrand: strand %s — joining the resume already in flight', strandId);
1631
+ return previous.operation;
1632
+ }
1633
+ const operation = this.runResume(strandId, previous, overrides);
1634
+ this.holdTransition(strandId, { kind: 'resume', operation });
1635
+ return operation;
1636
+ }
1637
+
1638
+ /** Record `transition` as the latest for `strandId` until it settles. */
1639
+ private holdTransition(strandId: string, transition: RuntimeTransition): void {
1640
+ this.transitions.set(strandId, transition);
1641
+ const release = (): void => {
1642
+ if (this.transitions.get(strandId) === transition) {
1643
+ this.transitions.delete(strandId);
1644
+ }
1645
+ };
1646
+ // The rejection itself reaches the caller through the operation returned to it.
1647
+ void transition.operation.then(release, release);
1648
+ }
1649
+
1650
+ /** Wait for the transition requested before this one. Never rejects: its failure belongs to its caller. */
1651
+ private async settleTransition(strandId: string, previous: RuntimeTransition | undefined): Promise<void> {
1652
+ if (!previous) {
1653
+ return;
1654
+ }
1655
+ await (previous.operation as Promise<unknown>).catch((error: unknown) => {
1656
+ log('Strand %s: the %s queued ahead failed (reported by its caller): %o', strandId, previous.kind, error);
1657
+ });
1658
+ }
1659
+
1660
+ /** Body of {@link resumeStrand}; at most one runs per strand at a time. */
1661
+ private async runResume(
1662
+ strandId: string,
1663
+ previous: RuntimeTransition | undefined,
1664
+ overrides?: ResumeStrandOverrides
1665
+ ): Promise<StrandInstance> {
1666
+ // Before any "already live" test: a quiesce ahead still has the handles attached, and
1667
+ // mid-build (a launch still building) the libp2p node is attached while the database is
1668
+ // not. A failed launch has already dropped the instance, and the check below reports it
1669
+ // untracked.
1670
+ await this.settleTransition(strandId, previous);
1671
+ await this.settleRuntimeBuilds(strandId);
1672
+ const instance = this.instances.get(strandId);
1673
+ if (!instance) {
1674
+ throw new Error(`Cannot resume strand ${strandId}: not tracked`);
1675
+ }
1676
+ const launchConfig = this.launchConfigs.get(strandId);
1677
+ if (!launchConfig) {
1678
+ throw new Error(`Cannot resume strand ${strandId}: no retained launch config`);
1679
+ }
1680
+ if (instance.libp2pNode || instance.database) {
1681
+ log('resumeStrand: strand %s already live', strandId);
1682
+ return instance;
1683
+ }
1684
+
1685
+ log('Resuming strand instance: %s', strandId);
1686
+ const tTotal = performance.now();
1687
+
1688
+ // Re-apply volatile inputs and persist them so a subsequent resume reuses them.
1689
+ // Each `??` matters: a resume that passes no override must keep the retained value,
1690
+ // and a resume that passes one must leave it retained for the next resume — otherwise
1691
+ // a later no-override wake silently reverts to whatever launch time saw.
1692
+ const resumeConfig: StartStrandConfig = {
1693
+ ...launchConfig,
1694
+ bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes,
1695
+ // NOTE: `??` retains, so an override can raise or lower the count but cannot CLEAR it
1696
+ // back to "this node no longer knows" — the direction that would declare nothing. Moot
1697
+ // while nothing feeds `servingMachines` at all; if a source lands
1698
+ // (`backlog/feat-strand-yardstick-from-serving-machines`) that can legitimately lose the
1699
+ // count — a strand whose member rows became unreadable — this merge must gain an explicit
1700
+ // clear rather than silently declaring a stale number over a serving set it can no longer
1701
+ // see. Same applies to `bootstrapNodes` above, where a stale seed is harmless.
1702
+ servingMachines: overrides?.servingMachines ?? launchConfig.servingMachines
1703
+ };
1704
+ this.launchConfigs.set(strandId, resumeConfig);
1705
+
1706
+ instance.status = 'starting';
1707
+ try {
1708
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, resumeConfig));
1709
+ timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
1710
+ log('Strand %s resumed successfully', strandId);
1711
+ return instance;
1712
+ } catch (error) {
1713
+ instance.status = 'error';
1714
+ instance.error = error instanceof Error ? error.message : String(error);
1715
+ log('Failed to resume strand %s: %s', strandId, instance.error);
1716
+ throw error;
1717
+ }
1718
+ }
1719
+
1720
+ /**
1721
+ * Honor a founder request against an ALREADY-TRACKED strand — the seam that
1722
+ * closes the "whoever launches first decides whether the bootstrap runs" gap:
1723
+ * an instance first launched as a joiner (an app's own attach, or a watcher
1724
+ * poll winning the launch race) used to swallow a later founder request
1725
+ * silently, leaving the strand active with no `Strand.Header`.
1726
+ *
1727
+ * Flips the RETAINED launch config's `founder` to true, so every later
1728
+ * quiesce → resume rebuild founds as well (the bootstrap is insert-if-absent —
1729
+ * {@link StrandDatabase.ensureFounderBootstrap} — so re-running it per rebuild
1730
+ * writes nothing twice), and runs the bootstrap against the live database now.
1731
+ *
1732
+ * @returns how the request resolved:
1733
+ * - `'already-founder'` — the retained config already founds; nothing to do.
1734
+ * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1735
+ * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1736
+ * database) or being quiesced (its database is closing), so the bootstrap could
1737
+ * not run here: the CALLER must wake the
1738
+ * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1739
+ * manager does not) so the rebuild — which now founds — runs it.
1740
+ * @param resolvePartyKey - Asked for the party's own membership key when (and only
1741
+ * when) a CLOSED strand's retained config carries none — the instance was launched
1742
+ * as a joiner before its `StrandPartyKey` row existed or replicated. The resolved
1743
+ * key is retained alongside the founder flip so the bootstrap (now, or on the
1744
+ * caller's wake for `'needs-resume'`) can seat the founding Member/Manager.
1745
+ * @throws when the strand is not tracked — this seam exists only for the
1746
+ * tracked-instance launch path; an untracked id is a caller bug.
1747
+ * @throws whatever the live bootstrap throws (e.g. `PreSplitStrandIdentityError`),
1748
+ * after {@link withdrawFounderRequest} — so a retry re-attempts the founding rather
1749
+ * than resolving `'already-founder'` over an instance that never founded. A caller
1750
+ * whose `'needs-resume'` wake fails owes the same withdrawal.
1751
+ */
1752
+ async foundExistingStrand(
1753
+ strandId: string,
1754
+ resolvePartyKey?: () => Promise<string | undefined>
1755
+ ): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'> {
1756
+ const instance = this.instances.get(strandId);
1757
+ const config = this.launchConfigs.get(strandId);
1758
+ if (!instance || !config) {
1759
+ throw new Error(`Cannot found strand ${strandId}: not tracked`);
1760
+ }
1761
+ if (config.founder === true) {
1762
+ return 'already-founder';
1763
+ }
1764
+ const partyMemberPrivateKey =
1765
+ config.partyMemberPrivateKey
1766
+ ?? (config.strandRow.Type === 'c' ? await resolvePartyKey?.() : undefined);
1767
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1768
+ // config object, which is not ours to rewrite.
1769
+ this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1770
+ if (this.isQuiescing(strandId) || (!instance.database && !this.firstSyncGates.has(strandId))) {
1771
+ return 'needs-resume';
1772
+ }
1773
+ try {
1774
+ await this.ensureFounderBootstrap(strandId);
1775
+ } catch (error) {
1776
+ this.withdrawFounderRequest(strandId);
1777
+ throw error;
1778
+ }
1779
+ return 'bootstrapped';
1780
+ }
1781
+
1782
+ /**
1783
+ * Give a tracked storage replica ({@link StartStrandConfig.sAppConfig} absent) the app's
1784
+ * schema in place — the seam for an app on this machine claiming a strand this node was
1785
+ * already hosting. No runtime rebuild: the libp2p node, its peer id and connections, the
1786
+ * store and its warm cache all stay, and the new `App` tables read the blocks the replica
1787
+ * already holds. Callers attach BEFORE {@link foundExistingStrand}: the founder bootstrap
1788
+ * writes the sApp into `Strand.Header` and refuses a database that has none.
1789
+ *
1790
+ * A runtime build in flight (the replica's launch, or a hibernation wake) is waited out
1791
+ * first, so the attach acts on the database that build produced. Then, in order: the
1792
+ * schema signature is checked before anything changes; the retained launch config takes
1793
+ * the sApp, so every later rebuild applies it through `composeStrand`; the live database —
1794
+ * published, or still held by the first-sync gate — gets the schema; and only then does
1795
+ * the instance record `sAppInfo` and the sApp's latency hint. An instance with no live
1796
+ * database gets the config alone: a quiesced one applies the schema at its next resume, a
1797
+ * launch waiting for a seed peer at its next database retry.
1798
+ *
1799
+ * @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
1800
+ * is logged and left as it is, as a second claim of a claimed strand always has been —
1801
+ * else `'attached'`.
1802
+ * @throws when the strand is not tracked (a caller bug, or a launch that failed while
1803
+ * this waited on it); when the schema signature is refused, having changed nothing; and
1804
+ * whatever the live apply throws (e.g. a quiesce closed the database mid-apply) — the
1805
+ * retained config then still carries the sApp and `sAppInfo` stays unset, so the next
1806
+ * claim retries the apply.
1807
+ *
1808
+ * NOTE: two claims of one replica racing each other both pass the `sAppInfo` check and
1809
+ * apply concurrently; the loser of the declarative diff can reject, and its `addStrand`
1810
+ * retry then resolves `'already-attached'`. Needs an app calling `addStrand` twice at once
1811
+ * (or beside a watcher retry); if it is ever seen, chain attaches per strand id.
1812
+ */
1813
+ async attachSApp(
1814
+ strandId: string,
1815
+ sAppConfig: SAppConfig,
1816
+ options: { requireSignedSchemas?: boolean } = {}
1817
+ ): Promise<'attached' | 'already-attached'> {
1818
+ await this.settleRuntimeBuilds(strandId);
1819
+ const instance = this.instances.get(strandId);
1820
+ const config = this.launchConfigs.get(strandId);
1821
+ if (!instance || !config) {
1822
+ throw new Error(`Cannot attach an sApp to strand ${strandId}: not tracked`);
1823
+ }
1824
+ if (instance.sAppInfo) {
1825
+ if (instance.sAppInfo.id !== sAppConfig.id) {
1826
+ log('attachSApp: strand %s already runs sApp %s — the claim for sApp %s is ignored',
1827
+ strandId, instance.sAppInfo.id, sAppConfig.id);
1828
+ }
1829
+ return 'already-attached';
1830
+ }
1831
+ const sAppInfo = verifiedSAppInfo(strandId, sAppConfig, options.requireSignedSchemas);
1832
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1833
+ // config object, which is not ours to rewrite.
1834
+ this.launchConfigs.set(strandId, { ...config, sAppConfig });
1835
+ const database = instance.database ?? this.firstSyncGates.get(strandId)?.database;
1836
+ if (database) {
1837
+ await database.attachAppSchema(sAppConfig);
1838
+ } else {
1839
+ log('attachSApp: strand %s has no live database — its next build applies the sApp schema', strandId);
1840
+ }
1841
+ instance.sAppInfo = sAppInfo;
1842
+ if (sAppConfig.latencyHint) {
1843
+ // HibernationManager reads the hint whenever it arms a timer, so the app's hint governs
1844
+ // from the next one: a timer already armed runs once at the old duration, and a replica
1845
+ // launched under a realtime default was never tracked, so it stays up until relaunched.
1846
+ instance.latencyHint = sAppConfig.latencyHint;
1847
+ }
1848
+ log('Strand %s upgraded from storage replica to %s', strandId, describeSApp(sAppConfig));
1849
+ return 'attached';
1850
+ }
1851
+
1852
+ /**
1853
+ * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1854
+ * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
1855
+ * the hibernation manager's idle timer fired on it), including one whose database build
1856
+ * waits for a seed peer to answer ({@link deferUnreachableJoin}). `false` for a writable,
1857
+ * quiesced, or untracked strand. The predicate {@link whenWritable}'s callers gate on.
1858
+ */
1859
+ isAwaitingFirstSync(strandId: string): boolean {
1860
+ const instance = this.instances.get(strandId);
1861
+ return instance !== undefined && isAwaitingFirstSync(instance);
1862
+ }
1863
+
1864
+ /**
1865
+ * Resolve once the strand's database is published to the app — immediately for a
1866
+ * writable strand — or reject with {@link StrandAwaitingFirstSyncError} after
1867
+ * `timeoutMs` (default: the retained launch config's `firstSync.timeoutMs`, else
1868
+ * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}). The rejection is RETRYABLE: nothing is
1869
+ * torn down, the gate keeps probing, and a later call waits afresh. A quiesced
1870
+ * (hibernating) strand is not woken here — the wait spans a resume, so a caller that
1871
+ * wants one wakes it (`CadreNode.wakeStrand`) and waits. Rejects immediately for an
1872
+ * untracked strand, and whenever the strand is stopped while a wait is pending.
1873
+ */
1874
+ whenWritable(strandId: string, options?: { timeoutMs?: number }): Promise<StrandInstance> {
1875
+ const instance = this.instances.get(strandId);
1876
+ if (!instance) {
1877
+ return Promise.reject(new Error(`Cannot wait for strand ${strandId} to become writable: not tracked`));
1878
+ }
1879
+ if (instance.database) {
1880
+ return Promise.resolve(instance);
1881
+ }
1882
+ const timeoutMs = options?.timeoutMs
1883
+ ?? this.launchConfigs.get(strandId)?.firstSync?.timeoutMs
1884
+ ?? DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS;
1885
+ const startedAt = Date.now();
1886
+ return new Promise<StrandInstance>((resolve, reject) => {
1887
+ const waiters = this.writableWaiters.get(strandId) ?? new Set<WritableWaiter>();
1888
+ this.writableWaiters.set(strandId, waiters);
1889
+ const settle = (): void => {
1890
+ clearTimeout(timer);
1891
+ waiters.delete(waiter);
1892
+ if (waiters.size === 0) {
1893
+ this.writableWaiters.delete(strandId);
1894
+ }
1895
+ };
1896
+ const waiter: WritableWaiter = {
1897
+ resolve: () => { settle(); resolve(instance); },
1898
+ reject: (error) => { settle(); reject(error); }
1899
+ };
1900
+ const timer = setTimeout(
1901
+ () => waiter.reject(new StrandAwaitingFirstSyncError(strandId, Date.now() - startedAt)),
1902
+ timeoutMs);
1903
+ (timer as { unref?: () => void }).unref?.();
1904
+ waiters.add(waiter);
1905
+ });
1906
+ }
1907
+
1908
+ /**
1909
+ * The tracked instance once no runtime build of it is in flight, or `undefined` when none
1910
+ * is tracked by then — never launched, or the launch waited on failed and dropped its
1911
+ * record. For a caller about to hand a tracked instance to an app: until its build settles
1912
+ * the instance is `'starting'` with no database, and a failed launch leaves no instance.
1913
+ * Never rejects; a failed build is reported by the call that started it.
1914
+ */
1915
+ async whenRuntimeBuilt(strandId: string): Promise<StrandInstance | undefined> {
1916
+ await this.settleRuntimeBuilds(strandId);
1917
+ return this.instances.get(strandId);
1918
+ }
1919
+
1920
+ /**
1921
+ * Hand a launch's database to the app: set `instance.database`, retire the gate that
1922
+ * held it, release every {@link whenWritable} waiter, and — when the launch had already
1923
+ * been reported gated — flip it `'active'` and announce it through the retained config's
1924
+ * `onWritable`. "Reported gated" is any status but `'starting'`: `'syncing'`, or the
1925
+ * `'idle'` the hibernation manager's idle timer flips a long-gated joiner to (see
1926
+ * {@link isAwaitingFirstSync}) — the Header arriving after that flip must announce too,
1927
+ * or an app hanging on `strand:writable` never learns the strand opened. A publish that
1928
+ * lands while the instance is still `'starting'` (the Header arrived during the rest of
1929
+ * bring-up) announces nothing: `startStrand` / `resumeStrand` are about to report the
1930
+ * strand `'active'`, and `strand:writable` is defined as the follow-up to a gated
1931
+ * launch, never a duplicate of `strand:started`.
1932
+ *
1933
+ * It is also the one seam where a WITHHELD database becomes available, so it starts the
1934
+ * revoked-peer enforcer and kicks the membership reconciler. The enforcer is registered
1935
+ * before the database phase, so every publish path — a launch never gated, the gate's
1936
+ * `onHeaderHeld`, {@link ensureFounderBootstrap}, a deferred join's retry, a hibernation
1937
+ * wake's rebuild — starts it here, and its change subscription needs the published
1938
+ * database. The reconciler's own immediate pass, on a gated launch, ran while
1939
+ * `instance.database` was still unset, found no database, and would otherwise not try
1940
+ * again until its retry timer fired. A launch that was never gated publishes here BEFORE
1941
+ * the reconciler is constructed, so there is nothing registered to kick and the loop's own
1942
+ * immediate pass already sees the database — which is why the kick needs no `wasGated`
1943
+ * test of its own.
1944
+ */
1945
+ private publishDatabase(instance: StrandInstance, database: StrandDatabase): void {
1946
+ const strandId = instance.strandId;
1947
+ const gate = this.firstSyncGates.get(strandId);
1948
+ if (gate) {
1949
+ gate.open();
1950
+ this.firstSyncGates.delete(strandId);
1951
+ }
1952
+ instance.database = database;
1953
+ const wasGated = instance.status !== 'starting';
1954
+ if (wasGated) {
1955
+ // A Header delivered by a peer is strand activity; a publish during bring-up is not
1956
+ // (see the build's end in `buildStrandRuntime`).
1957
+ instance.lastActivity = new Date();
1958
+ instance.status = 'active';
1959
+ log('Strand %s is now writable: Strand.Header held', strandId);
1960
+ }
1961
+ const waiters = this.writableWaiters.get(strandId);
1962
+ if (waiters) {
1963
+ this.writableWaiters.delete(strandId);
1964
+ waiters.forEach((waiter) => waiter.resolve());
1965
+ }
1966
+ if (wasGated) {
1967
+ this.launchConfigs.get(strandId)?.onWritable?.(strandId);
1968
+ }
1969
+ // Neither throws nor awaits: publishing must stay synchronous. `start()` is a no-op on an
1970
+ // enforcer already started or stopped, and the reconciler pass chains on the loop's own
1971
+ // tail, so it cannot overlap a pass already in flight.
1972
+ this.revocationEnforcers.get(strandId)?.start();
1973
+ void this.membershipReconcilers.get(strandId)?.reconcile();
1974
+ }
1975
+
1976
+ /** Reject every {@link whenWritable} waiter for a strand that is going away. */
1977
+ private rejectWritableWaiters(strandId: string, reason: string): void {
1978
+ const waiters = this.writableWaiters.get(strandId);
1979
+ if (!waiters) {
1980
+ return;
1981
+ }
1982
+ this.writableWaiters.delete(strandId);
1983
+ waiters.forEach((waiter) => waiter.reject(new Error(`Strand ${strandId} ${reason} before becoming writable`)));
1984
+ }
1985
+
1986
+ /**
1987
+ * Undo {@link foundExistingStrand}'s founder flip after the founding it promised
1988
+ * failed, so the retained config again says what this instance actually runs as (a
1989
+ * joiner) and the next founder request re-attempts the bootstrap. The resolved party
1990
+ * key stays retained — it is this party's identity for the strand either way. No-op
1991
+ * when the strand is untracked or its config does not found.
1992
+ */
1993
+ withdrawFounderRequest(strandId: string): void {
1994
+ const config = this.launchConfigs.get(strandId);
1995
+ if (config?.founder === true) {
1996
+ this.launchConfigs.set(strandId, { ...config, founder: false });
1997
+ }
1998
+ }
1999
+
2000
+ /**
2001
+ * Run the (idempotent) founder bootstrap against a tracked strand's LIVE
2002
+ * database, independently of what the retained launch config says.
2003
+ *
2004
+ * Separate from {@link foundExistingStrand} because a caller that resolved
2005
+ * `'needs-resume'` and woke the strand must not assume the wake's own rebuild
2006
+ * founded it: `HibernationManager` COALESCES wakes, so a wake already in flight
2007
+ * when the config flipped had already read the PRE-flip config and rebuilt as a
2008
+ * joiner. Re-running the bootstrap costs one insert-if-absent probe per table
2009
+ * and is the only thing that makes "founding resolves once the Header is
2010
+ * written" true on that path.
2011
+ *
2012
+ * A strand still `'syncing'` (its database withheld behind the first-sync gate) is
2013
+ * founded through the gate: the bootstrap writes the Header this machine was waiting
2014
+ * to receive, so a successful run publishes the database and the strand goes
2015
+ * `'active'` — the one legitimate way a joiner launch becomes writable without a peer.
2016
+ *
2017
+ * @throws when the strand is not tracked, or is still quiesced (no live
2018
+ * database) — both mean the bootstrap did NOT run, which a founder request
2019
+ * must never swallow.
2020
+ */
2021
+ async ensureFounderBootstrap(strandId: string): Promise<void> {
2022
+ const instance = this.instances.get(strandId);
2023
+ if (!instance) {
2024
+ throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: not tracked`);
2025
+ }
2026
+ const gate = this.firstSyncGates.get(strandId);
2027
+ const database = instance.database ?? gate?.database;
2028
+ if (!database) {
2029
+ throw new Error(
2030
+ `Cannot run the founder bootstrap for strand ${strandId}: it has no live database to ` +
2031
+ 'write to — it is quiesced (resume it first), or its database build is waiting for a ' +
2032
+ 'seed peer to answer.'
2033
+ );
2034
+ }
2035
+ // Forward the RETAINED config's party key: a foundExistingStrand that flipped a
2036
+ // joiner launch to founder may have resolved a key the live database's captured
2037
+ // config (built at the original launch) never saw.
2038
+ await database.ensureFounderBootstrap(this.launchConfigs.get(strandId)?.partyMemberPrivateKey);
2039
+ if (!instance.database) {
2040
+ this.publishDatabase(instance, database);
2041
+ }
2042
+ }
2043
+
2044
+ /**
2045
+ * Stop a strand instance
2046
+ */
2047
+ async stopStrand(strandId: string): Promise<void> {
2048
+ const instance = this.instances.get(strandId);
2049
+ if (!instance) {
2050
+ log('Strand %s not found', strandId);
2051
+ return;
2052
+ }
2053
+
2054
+ log('Stopping strand instance: %s', strandId);
2055
+ instance.status = 'stopping';
2056
+ this.rejectWritableWaiters(strandId, 'was stopped');
2057
+
2058
+ try {
2059
+ await this.releaseRuntime(instance);
2060
+ instance.status = 'stopped';
2061
+ this.instances.delete(strandId);
2062
+ this.launchConfigs.delete(strandId);
2063
+ // Storage is released only here — NOT in releaseRuntime, which a quiesce shares.
2064
+ await this.disposeStrandStorage(strandId);
2065
+ log('Strand %s stopped successfully', strandId);
2066
+ } catch (error) {
2067
+ instance.status = 'error';
2068
+ instance.error = error instanceof Error ? error.message : String(error);
2069
+ log('Error stopping strand %s: %s', strandId, instance.error);
2070
+ throw error;
2071
+ }
2072
+ }
2073
+
2074
+ /**
2075
+ * Stop all strand instances
2076
+ */
2077
+ async stopAll(): Promise<void> {
2078
+ this.stopping = true;
2079
+ log('Stopping all strand instances (%d)', this.instances.size);
2080
+
2081
+ const stopPromises = Array.from(this.instances.keys()).map(id =>
2082
+ this.stopStrand(id).catch(err => {
2083
+ log('Error stopping strand %s during shutdown: %s', id, err);
2084
+ })
2085
+ );
2086
+
2087
+ await Promise.all(stopPromises);
2088
+ this.stopping = false;
2089
+ log('All strand instances stopped');
2090
+ }
2091
+ }
2092
+