@serfab/cadre-core 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +5 -3
  2. package/dist/cadre-node.d.ts +70 -34
  3. package/dist/cadre-node.js +197 -60
  4. package/dist/cadre-node.js.map +1 -1
  5. package/dist/control-database.d.ts +24 -10
  6. package/dist/control-database.js +48 -16
  7. package/dist/control-database.js.map +1 -1
  8. package/dist/control-formation-recorder.d.ts +4 -4
  9. package/dist/control-formation-recorder.js +6 -6
  10. package/dist/control-formation-recorder.js.map +1 -1
  11. package/dist/control-read-retry.d.ts +12 -0
  12. package/dist/control-read-retry.js +12 -0
  13. package/dist/control-read-retry.js.map +1 -1
  14. package/dist/control-write-retry.d.ts +5 -4
  15. package/dist/control-write-retry.js +24 -7
  16. package/dist/control-write-retry.js.map +1 -1
  17. package/dist/index.d.ts +5 -4
  18. package/dist/index.js +10 -5
  19. package/dist/index.js.map +1 -1
  20. package/dist/link-budget.d.ts +182 -0
  21. package/dist/link-budget.js +195 -0
  22. package/dist/link-budget.js.map +1 -0
  23. package/dist/membership-connection-gater.d.ts +18 -5
  24. package/dist/membership-connection-gater.js +50 -10
  25. package/dist/membership-connection-gater.js.map +1 -1
  26. package/dist/peer-dial.d.ts +36 -17
  27. package/dist/peer-dial.js +37 -17
  28. package/dist/peer-dial.js.map +1 -1
  29. package/dist/peer-join-backfill.d.ts +58 -4
  30. package/dist/peer-join-backfill.js +165 -20
  31. package/dist/peer-join-backfill.js.map +1 -1
  32. package/dist/relay-reservation.d.ts +59 -11
  33. package/dist/relay-reservation.js +169 -43
  34. package/dist/relay-reservation.js.map +1 -1
  35. package/dist/seed-bootstrap.js +2 -1
  36. package/dist/seed-bootstrap.js.map +1 -1
  37. package/dist/storage-scope.d.ts +50 -12
  38. package/dist/storage-scope.js +85 -12
  39. package/dist/storage-scope.js.map +1 -1
  40. package/dist/strand-first-sync-gate.d.ts +43 -5
  41. package/dist/strand-first-sync-gate.js +55 -7
  42. package/dist/strand-first-sync-gate.js.map +1 -1
  43. package/dist/strand-formation-manager.js +2 -1
  44. package/dist/strand-formation-manager.js.map +1 -1
  45. package/dist/strand-id.d.ts +35 -0
  46. package/dist/strand-id.js +46 -0
  47. package/dist/strand-id.js.map +1 -0
  48. package/dist/strand-instance-manager.d.ts +27 -2
  49. package/dist/strand-instance-manager.js +75 -15
  50. package/dist/strand-instance-manager.js.map +1 -1
  51. package/dist/strand-membership-reconciler.d.ts +200 -45
  52. package/dist/strand-membership-reconciler.js +302 -86
  53. package/dist/strand-membership-reconciler.js.map +1 -1
  54. package/dist/strand-membership-writer.d.ts +3 -2
  55. package/dist/strand-membership-writer.js +67 -90
  56. package/dist/strand-membership-writer.js.map +1 -1
  57. package/dist/strand-network-config.d.ts +3 -3
  58. package/dist/strand-network-config.js +3 -3
  59. package/dist/strand-solicitation.js +2 -1
  60. package/dist/strand-solicitation.js.map +1 -1
  61. package/dist/strand-watcher.d.ts +45 -0
  62. package/dist/strand-watcher.js +74 -13
  63. package/dist/strand-watcher.js.map +1 -1
  64. package/dist/types.d.ts +190 -24
  65. package/dist/types.js +73 -4
  66. package/dist/types.js.map +1 -1
  67. package/package.json +8 -8
  68. package/src/cadre-node.ts +7445 -7302
  69. package/src/control-database.ts +48 -16
  70. package/src/control-formation-recorder.ts +6 -6
  71. package/src/control-read-retry.ts +214 -202
  72. package/src/control-write-retry.ts +24 -7
  73. package/src/index.ts +29 -2
  74. package/src/link-budget.ts +217 -0
  75. package/src/membership-connection-gater.ts +56 -11
  76. package/src/peer-dial.ts +39 -17
  77. package/src/peer-join-backfill.ts +192 -23
  78. package/src/relay-reservation.ts +1119 -951
  79. package/src/seed-bootstrap.ts +2 -1
  80. package/src/storage-scope.ts +163 -78
  81. package/src/strand-first-sync-gate.ts +55 -7
  82. package/src/strand-formation-manager.ts +647 -646
  83. package/src/strand-id.ts +48 -0
  84. package/src/strand-instance-manager.ts +1516 -1440
  85. package/src/strand-membership-reconciler.ts +334 -92
  86. package/src/strand-membership-writer.ts +1962 -1987
  87. package/src/strand-network-config.ts +250 -250
  88. package/src/strand-solicitation.ts +500 -499
  89. package/src/strand-watcher.ts +388 -325
  90. package/src/types.ts +192 -23
@@ -1,1440 +1,1516 @@
1
- import debug from 'debug';
2
- import type { PrivateKey } from '@libp2p/interface';
3
- import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
4
- import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
5
- import { StrandDatabase } from './strand-database.js';
6
- import { PeerJoinBackfill, type PeerJoinBackfillConfig } from './peer-join-backfill.js';
7
- import {
8
- StrandRevocationEnforcer,
9
- createRevocationConnectionGater,
10
- readStrandRevocationRows,
11
- type StrandRevocationEnforcementConfig
12
- } from './strand-revocation-enforcer.js';
13
- import {
14
- StrandMembershipReconciler,
15
- type PendingMembershipInviteSource,
16
- type StrandMembershipReconciliationConfig
17
- } from './strand-membership-reconciler.js';
18
- import { removeMemberPeer } from './strand-membership-writer.js';
19
- import { strandMemberKeyPair } from './strand-member-key.js';
20
- import { assertSchemaSignature } from './schema-verification.js';
21
- import {
22
- StrandFirstSyncGate,
23
- StrandAwaitingFirstSyncError,
24
- strandFirstSyncComplete,
25
- DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS,
26
- type StrandFirstSyncConfig
27
- } from './strand-first-sync-gate.js';
28
- import type {
29
- StrandInstance,
30
- StrandRow,
31
- StorageConfig,
32
- NetworkConfig,
33
- LatencyHint,
34
- NodeProfile,
35
- SAppConfig,
36
- SAppInfo,
37
- RawStorageProvider,
38
- Libp2pNodeWithRepo
39
- } from './types.js';
40
- import { resolveStrandClusterSize, strandClusterPolicy } from './types.js';
41
- import { strandNodeAddrs } from './strand-network-config.js';
42
- import { superviseRelayReservation, type RelayReservationSupervisor } from './relay-reservation.js';
43
-
44
- const log = debug('sereus:cadre:strand-manager');
45
- const timing = debug('sereus:cadre:timing');
46
-
47
- /** One caller blocked in {@link StrandInstanceManager.whenWritable}. */
48
- interface WritableWaiter {
49
- resolve: () => void;
50
- reject: (error: Error) => void;
51
- }
52
-
53
- /**
54
- * Whether a tracked instance is launched but waiting for its first sync: the runtime is
55
- * up (`libp2pNode` set) while the database is still withheld (`database` unset). That
56
- * is the first-sync gate's state regardless of what `status` says — the hibernation
57
- * manager's idle timer can flip a long-gated joiner to `'idle'` without changing it.
58
- *
59
- * NOTE: `status` is therefore not the source of truth for "gated": a joiner that reaches
60
- * nobody for `idleTimeout` (5 min at the interactive hint) reads `'idle'`, then
61
- * `'hibernating'`, while still holding no database. Harmless today — every decision the
62
- * runtime makes goes through this predicate, {@link StrandInstanceManager.publishDatabase}
63
- * announces on the idle-flipped instance exactly as on a `'syncing'` one, and an app that
64
- * shows "waiting for the other member" on `'syncing'` merely sees "idle" instead after
65
- * five minutes. If an app ever needs the gated LABEL to survive the idle flip, make the
66
- * hibernation manager's idle transition preserve `'syncing'` rather than adding a flag.
67
- */
68
- export function isAwaitingFirstSync(instance: StrandInstance): boolean {
69
- return instance.libp2pNode !== undefined && instance.database === undefined;
70
- }
71
-
72
- /**
73
- * The status a LIVE instance (runtime rebuilt or woken) should report: `'active'` once its
74
- * database is published, `'syncing'` while the first-sync gate still withholds it. Every
75
- * site that would otherwise write `'active'` after a build or a wake goes through this,
76
- * so a gated joiner never reads as writable.
77
- */
78
- export function liveStrandStatus(instance: StrandInstance): 'active' | 'syncing' {
79
- return instance.database ? 'active' : 'syncing';
80
- }
81
-
82
- /**
83
- * Configuration for starting a strand instance
84
- */
85
- export interface StartStrandConfig {
86
- strandRow: StrandRow;
87
- /** sApp configuration provided by the hosting application */
88
- sAppConfig: SAppConfig;
89
- storage?: StorageConfig;
90
- network?: NetworkConfig;
91
- profile: NodeProfile;
92
- defaultLatencyHint: LatencyHint;
93
- privateKey?: PrivateKey;
94
- /** Cohort-derived discovery seed (multiaddr strings). Defaults to [] when omitted. */
95
- bootstrapNodes?: string[];
96
- /**
97
- * Require a valid author signature on the sApp schema before bring-up.
98
- * Defaults to true (fail closed) when omitted; set false only for dev/test
99
- * with unsigned demo schemas. Mirrors {@link CadreNodeConfig.requireSignedSchemas}.
100
- */
101
- requireSignedSchemas?: boolean;
102
- /**
103
- * Whether this node founds the strand (vs. joins it). Forwarded to the
104
- * StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
105
- * runs once at bring-up. Joiners leave this unset and write nothing. Callers
106
- * resolve it BEFORE calling `startStrand` (`CadreNode.launchStrand` derives it
107
- * from the row's `FounderOwnerKey` when no explicit flag is given — see
108
- * {@link StrandConfig.founder}); a founder request that arrives while the
109
- * instance is already tracked goes through {@link StrandInstanceManager.foundExistingStrand},
110
- * never through a repeat `startStrand`.
111
- */
112
- founder?: boolean;
113
- /**
114
- * THIS party's own strand membership private key (base64 protobuf) for a closed
115
- * strand — the identity the founder bootstrap derives `Member.Key` /
116
- * `Manager.MemberKey` from. Resolved by the caller (`CadreNode.launchStrand`: an
117
- * explicit attach-time key, else the control-layer `StrandPartyKey` row, healed by
118
- * mint-on-launch for the founding machine) and retained with the launch config, so a
119
- * hibernation wake rebuilds under the same identity. Deliberately NOT the strand
120
- * row's shared `MemberPrivateKey` (see `StrandDatabaseConfig.partyMemberPrivateKey`).
121
- * Absent for open strands, and for joiners with no persisted party key.
122
- */
123
- partyMemberPrivateKey?: string;
124
- /**
125
- * Number of nodes Optimystic is told this strand's replication cluster should
126
- * have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
127
- * forwards here: every node on the strand should use the same value, and it is
128
- * frozen when the strand's libp2p node is created. Defaults to
129
- * `DEFAULT_STRAND_CLUSTER_SIZE` (4); values below `MIN_CLUSTER_SIZE` (2) are
130
- * rejected by `resolveStrandClusterSize`. Strand-only — the
131
- * control network's breadth is the fixed `CONTROL_REPLICATION_BREADTH`.
132
- */
133
- clusterSize?: number;
134
- /**
135
- * Machines that SERVE THIS STRAND (this node included), for the strand node's
136
- * block-repair corroboration yardstick — see `resolveRepairYardstick`. It must be
137
- * an authenticated per-strand count: the machines that actually run a node for this
138
- * strand, not the machines that exist.
139
- *
140
- * **The party's enrolled-machine count is NOT this number and must never be passed
141
- * here.** A strand launches only on machines whose embedding app registered its sApp
142
- * config (`CadreNode.addStrand`), so a closed strand shared by two machines of a
143
- * three-machine party is served by two. Passing three over-declares, and
144
- * over-declaring is the unsafe direction: at a declaration of 3 or more Optimystic
145
- * pins the repair corroboration floor at two corroborating peers, which a cohort
146
- * that can only ever field one peer can never reach — so the strand can never repair
147
- * a block (`cluster-fetch:no-quorum`, surfacing as reads failing with `Missing
148
- * block`). That regression is why this field was renamed off "enrolledMachines":
149
- * `bug-strand-yardstick-counts-party-machines`.
150
- *
151
- * **Nothing feeds it in production today.** No authenticated per-strand serving count
152
- * exists yet, so `CadreNode` passes nothing and the strand node runs the frozen
153
- * `STRAND_CLUSTER_POLICY` — declaring no yardstick, which leaves the known,
154
- * upstream-tracked single-voter exposure
155
- * (`backlog/debt-read-repair-single-voter-corroboration`). Building the count is
156
- * `backlog/feat-strand-yardstick-from-serving-machines`; this field and its
157
- * threading are the seam it plugs into.
158
- *
159
- * Volatile when a source does exist: re-resolve it on every resume beside the cohort
160
- * seed, since machines join and leave a strand while it hibernates. Omitting it means
161
- * "this node does not know", which declares nothing.
162
- */
163
- servingMachines?: number;
164
- /**
165
- * Tuning for the strand peer-join block catch-up ({@link PeerJoinBackfill}),
166
- * forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
167
- * libp2p node connects to a peer this runtime has not yet caught up, every
168
- * block in the strand's own raw store is pushed to it. Runs only on strands
169
- * with per-strand storage; `{ enabled: false }` restores the pre-existing
170
- * no-backfill behaviour.
171
- */
172
- backfill?: PeerJoinBackfillConfig;
173
- /**
174
- * Tuning for the CLOSED-strand revoked-peer gate
175
- * ({@link StrandRevocationEnforcer}), forwarded from
176
- * {@link CadreNodeConfig.strandRevocationEnforcement}. Armed only for
177
- * `strandRow.Type === 'c'` (an open strand has no membership rows to derive a
178
- * deny set from); `{ enabled: false }` restores the pre-existing behaviour
179
- * (a removed party's peers keep being served).
180
- */
181
- revocationEnforcement?: StrandRevocationEnforcementConfig;
182
-
183
- /**
184
- * The staged formation membership invitation for THIS strand — the seam the
185
- * bring-up membership reconciler (`strand-membership-reconciler.ts`) reads to
186
- * redeem the party's `Strand.Member` seat, and clears once the invitation is
187
- * spent, burned, or dead. `CadreNode.launchStrand` wires it over its in-memory
188
- * `pendingMembershipInvites` map; read lazily per pass, so a re-formation that
189
- * replaces the entry between passes is picked up. Only consulted for closed
190
- * strands launched with a {@link partyMemberPrivateKey}; harmless otherwise.
191
- */
192
- pendingMembershipInvite?: PendingMembershipInviteSource;
193
-
194
- /**
195
- * Tuning for the CLOSED-strand membership reconciler, forwarded from
196
- * {@link CadreNodeConfig.strandMembershipReconciliation}. Armed only for
197
- * closed strands launched with a {@link partyMemberPrivateKey};
198
- * `{ enabled: false }` disarms the loop (test fixtures that hand-drive the
199
- * membership writers). When it names no `pollIntervalMs` the reconciler
200
- * mirrors {@link revocationEnforcement}'s cadence — as its IDLE cadence and
201
- * the cap of its unfinished-join retry ladder, not as a flat retry interval.
202
- */
203
- membershipReconciliation?: StrandMembershipReconciliationConfig;
204
-
205
- /**
206
- * Called when THIS node's own peer id turns up in the strand's revoked set —
207
- * this node's party was removed from the closed strand, or left it. Fires once
208
- * per removal (re-armed if the party is re-admitted; a resume also rebuilds the
209
- * enforcer and may legitimately re-fire), and only for closed strands with the
210
- * gate armed. `CadreNode`
211
- * wires it to its `strand:revoked` event; nothing is stopped or torn down on
212
- * this node's behalf — what to do about it is the app's call.
213
- */
214
- onSelfRevoked?: (strandId: string) => void;
215
-
216
- /**
217
- * Tuning for the JOINING machine's first-sync write gate (`strand-first-sync-gate.ts`),
218
- * forwarded from {@link CadreNodeConfig.strandFirstSync}: the Header probe cadence while
219
- * a non-founder launch is `'syncing'`, and the default budget {@link whenWritable} waits
220
- * before rejecting. Founders and machines that already hold the Header are never gated.
221
- */
222
- firstSync?: StrandFirstSyncConfig;
223
-
224
- /**
225
- * Called when a launch that came up `'syncing'` (see {@link firstSync}) becomes
226
- * writable: the strand's `Strand.Header` arrived from a peer — or a founder request
227
- * ran the bootstrap against the gated database — and `instance.database` is now set.
228
- * Never called for a launch that was writable when `startStrand`/`resumeStrand`
229
- * resolved. `CadreNode` wires it to its `strand:writable` event. Retained with the
230
- * launch config, so a hibernation resume that comes up gated announces too.
231
- */
232
- onWritable?: (strandId: string) => void;
233
-
234
- /**
235
- * Re-announce this strand's delegate peer id (`delegatePeerId` — the strand
236
- * node's own transport peer id) to ONE relay (`relayAddr`, a direct dial addr
237
- * with a trailing `/p2p/<relayPeerId>`), unthrottled. Awaited before every
238
- * relay-reservation RE-drive of that relay — never before the first attempt,
239
- * which the launch/resume seed pass has already announced for.
240
- *
241
- * Why: a party control node running the relay server admits the strand node on
242
- * an in-memory delegate grant that a relay restart drops, so a re-drive that did
243
- * not re-announce first would be denied at the relay's gate. `CadreNode` wires
244
- * its own control-mesh announce here; a throw or rejection is logged by the
245
- * supervisor and the re-drive still runs. Retained with the launch config, so a
246
- * hibernation wake's rebuilt supervisors carry it too.
247
- */
248
- announceDelegateToRelay?: (strandId: string, relayAddr: string, delegatePeerId: string) => Promise<void>;
249
- }
250
-
251
- /**
252
- * Volatile inputs re-resolved when resuming a quiesced strand — the ones that can
253
- * have moved since it last ran, and that the rebuilt libp2p node freezes again. Each
254
- * one omitted keeps the value the retained launch config already holds, so a resume
255
- * that passes nothing rebuilds the strand exactly as it last ran.
256
- */
257
- export interface ResumeStrandOverrides {
258
- /**
259
- * Freshly-resolved cohort discovery seed (multiaddr strings). Grows as peers are
260
- * learned since the strand first launched.
261
- */
262
- bootstrapNodes?: string[];
263
- /**
264
- * Freshly-read count of the machines serving this strand, for the repair yardstick
265
- * (see {@link StartStrandConfig.servingMachines} — including why the party's
266
- * enrolled-machine count is not it). Moves whenever a machine starts or stops
267
- * serving the strand, which a hibernating strand does not otherwise notice. Nothing
268
- * passes it today. Omitting it here does NOT mean "unknown" the way omitting it from
269
- * {@link StartStrandConfig} does — it retains the last value; see `resumeStrand`.
270
- */
271
- servingMachines?: number;
272
- }
273
-
274
- /**
275
- * Get the isolated storage path for a specific strand.
276
- *
277
- * @deprecated This helper is Node-only and throws in React Native (it assumes a
278
- * filesystem layout). Use a storage provider factory function instead, which
279
- * receives the strandId and can create strand-specific storage paths using
280
- * platform-appropriate methods.
281
- *
282
- * @example
283
- * // Instead of using getStrandStoragePath, use a storage provider factory:
284
- * const storage = {
285
- * provider: (strandId: string) => new FileRawStorage(`./data/strands/${strandId}`)
286
- * };
287
- */
288
- export function getStrandStoragePath(basePath: string, strandId: string): string {
289
- // Check if we're in a Node.js environment
290
- if (typeof process === 'undefined' || !process.versions?.node) {
291
- throw new Error(
292
- 'getStrandStoragePath is not available in React Native. ' +
293
- 'Use a storage provider factory function instead.'
294
- );
295
- }
296
-
297
- // Sanitize strandId for filesystem safety (UUIDs should be safe, but just in case)
298
- const safeId = strandId.replace(/[^a-zA-Z0-9-]/g, '_');
299
-
300
- // Build the path with plain string joins rather than the Node `path` module.
301
- // A static `require('path')` forces RN bundlers (e.g. Metro) to *resolve* the
302
- // module at bundle time even though this Node-only helper throws above before
303
- // ever reaching here — joining by hand keeps the module free of any Node
304
- // built-in reference, so RN bundles need no `path` shim.
305
- const trimmedBase = basePath.replace(/[\\/]+$/, '');
306
- return `${trimmedBase}/strands/${safeId}`;
307
- }
308
-
309
- /**
310
- * Resolve a storage provider for a specific strand.
311
- * If the provider is a factory function, call it with the strandId.
312
- *
313
- * Called only from {@link StrandInstanceManager.startStrand}, which owns the result
314
- * for the instance's lifetime — see `strandStorages`.
315
- *
316
- * @param provider - Storage provider (instance or factory)
317
- * @param strandId - The strand ID to create storage for
318
- * @returns The resolved IRawStorage instance, or undefined if no provider
319
- */
320
- function resolveStrandStorage(
321
- provider: RawStorageProvider | undefined,
322
- strandId: string
323
- ): IRawStorage | undefined {
324
- if (!provider) {
325
- return undefined;
326
- }
327
- const storage = typeof provider === 'function' ? provider(strandId) : provider;
328
- // Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's cached-storage.ts).
329
- // Called ONCE per strand launch — `startStrand` keeps the result for the instance's
330
- // lifetime — so the wrap survives every runtime rebuild the instance goes through.
331
- return wrapStorageWithCache(storage, strandId);
332
- }
333
-
334
- /**
335
- * Manages individual strand instances - creates and destroys isolated libp2p nodes
336
- * for each strand the cadre participates in.
337
- */
338
- export class StrandInstanceManager {
339
- private instances: Map<string, StrandInstance> = new Map();
340
- /**
341
- * Retained launch config per strand, captured in `startStrand` and cleared in
342
- * `stopStrand`. `resumeStrand` reuses it to rebuild a quiesced strand's runtime
343
- * without the caller re-threading storage/network/profile/key/sApp config.
344
- */
345
- private launchConfigs: Map<string, StartStrandConfig> = new Map();
346
- /**
347
- * The per-strand peer-join block catch-up, keyed by strand id. Private (not on
348
- * the public {@link StrandInstance}) because it is runtime plumbing with the
349
- * same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
350
- * stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
351
- * with a fresh caught-up-peer memo, which is intended (a resumed node may have
352
- * missed writes).
353
- */
354
- private backfills: Map<string, PeerJoinBackfill> = new Map();
355
- /**
356
- * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
357
- * id. Same lifecycle rationale as {@link backfills}: created in
358
- * `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so quiesce →
359
- * resume rebuilds it with a fresh (initially empty, fail-open) snapshot.
360
- */
361
- private revocationEnforcers: Map<string, StrandRevocationEnforcer> = new Map();
362
- /**
363
- * The per-strand membership reconciler (closed strands launched with a party
364
- * key only), keyed by strand id. Same lifecycle rationale as {@link backfills}:
365
- * created in `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so
366
- * quiesce → resume rebuilds it and re-runs the idempotent join ladder (redeem a
367
- * staged invitation, write this machine's own `MemberPeer` binding) from scratch.
368
- */
369
- private membershipReconcilers: Map<string, StrandMembershipReconciler> = new Map();
370
- /**
371
- * The per-strand first-sync gate (`strand-first-sync-gate.ts`), keyed by strand id:
372
- * holds a launch's `StrandDatabase` while it is NOT yet published on the instance.
373
- * Present from the moment the database object exists in `buildStrandRuntime` until
374
- * {@link publishDatabase} hands it to the app — so it doubles as the rollback handle
375
- * for a failed `initialize()` — and, for a joiner that holds no `Strand.Header` yet,
376
- * for as long as the launch stays `'syncing'`. Same lifecycle as {@link backfills}:
377
- * stopped (its database closed) and dropped in `releaseRuntime`, so a quiesce →
378
- * resume rebuild re-probes over the same store. An entry and `instance.database`
379
- * are mutually exclusive.
380
- */
381
- private firstSyncGates: Map<string, StrandFirstSyncGate> = new Map();
382
- /**
383
- * Callers blocked in {@link whenWritable}, keyed by strand id. Resolved by
384
- * {@link publishDatabase}, rejected by {@link stopStrand}; each waiter also carries
385
- * its own timeout. Deliberately NOT cleared by `releaseRuntime`: a waiter outlives a
386
- * quiesce → resume cycle, since the rebuild is what may finally publish the database.
387
- */
388
- private writableWaiters: Map<string, Set<WritableWaiter>> = new Map();
389
- /**
390
- * The per-strand relay-reservation supervisors — ONE PER CONFIGURED RELAY, each
391
- * over the node's own bare `/p2p-circuit` listener for that relay
392
- * (`strand-network-config.ts`) — keyed by strand id. Same lifecycle rationale as
393
- * {@link backfills}: started in `buildStrandRuntime` right after the libp2p node
394
- * exists, stopped FIRST and dropped in `releaseRuntime` (before the database
395
- * closes and the node stops, so no re-drive dials a node being torn down), so
396
- * quiesce → resume rebuilds them over the new node. What makes a strand node's
397
- * lost relay slot come back on its own — the control node has the same loop in
398
- * `CadreNode.reserveRelays`.
399
- */
400
- private relaySupervisors: Map<string, RelayReservationSupervisor[]> = new Map();
401
- /**
402
- * The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
403
- * store, resolved once in `startStrand` and held until `stopStrand` disposes it.
404
- * Private for the same reason `backfills` is: runtime plumbing, not part of the
405
- * public {@link StrandInstance}.
406
- *
407
- * Deliberately NOT touched by `releaseRuntime`: that is what lets a quiesce →
408
- * resume cycle rebuild the libp2p node over the SAME store, keeping its
409
- * write-through cache warm (and, on an in-memory backend, keeping the strand's
410
- * blocks at all). An entry exists iff `instances` does AND the launch config
411
- * supplied a storage provider.
412
- *
413
- * NOTE: a hibernating strand therefore keeps its store — and its share of the
414
- * process-wide cache pool — resident for as long as the instance is tracked. That
415
- * is the point (a warm wake), and the pool evicts under pressure, so the cost is
416
- * one map entry per hibernating strand today. If a device ever hibernates strands
417
- * by the hundred, revisit: dropping the store at quiesce and paying for a cold
418
- * wake becomes the better trade.
419
- */
420
- private strandStorages: Map<string, IRawStorage> = new Map();
421
- private stopping = false;
422
-
423
- constructor() {
424
- log('StrandInstanceManager created');
425
- }
426
-
427
- /**
428
- * Get all current strand instances
429
- */
430
- getInstances(): Map<string, StrandInstance> {
431
- return new Map(this.instances);
432
- }
433
-
434
- /**
435
- * Get a specific strand instance
436
- */
437
- getInstance(strandId: string): StrandInstance | undefined {
438
- return this.instances.get(strandId);
439
- }
440
-
441
- /**
442
- * Check if a strand is currently running
443
- */
444
- hasStrand(strandId: string): boolean {
445
- return this.instances.has(strandId);
446
- }
447
-
448
- /**
449
- * Start a new strand instance.
450
- *
451
- * A failed launch leaves NOTHING tracked: the instance and its retained launch
452
- * config are both dropped before the error is rethrown, so the strand id is
453
- * free for a genuine retry. This matches the pre-registration failure path
454
- * (a rejected schema signature, which throws before anything is recorded) —
455
- * both failure modes of this call leave the same residue: none. Callers learn
456
- * of the failure from the rejected promise (and, on the control-discovered
457
- * path, from CadreNode's `strand:error` event), not from an error record left
458
- * behind in `instances`.
459
- *
460
- * The strand's raw storage is resolved HERE, once, and held for the instance's
461
- * lifetime (see `strandStorages`) — `buildStrandRuntime` only reads it, so a
462
- * hibernation wake never re-enters the embedder's provider.
463
- */
464
- async startStrand(config: StartStrandConfig): Promise<StrandInstance> {
465
- const { strandRow, sAppConfig } = config;
466
- const strandId = strandRow.Id;
467
-
468
- if (this.stopping) {
469
- throw new Error('StrandInstanceManager is stopping');
470
- }
471
-
472
- if (this.instances.has(strandId)) {
473
- // Callers resolve founder-ness BEFORE reaching here (CadreNode.launchStrand) and
474
- // honor a founder request on a tracked instance via foundExistingStrand — so a
475
- // founder flag arriving at this early return against a non-founder retained
476
- // config is a dropped bootstrap, and must never again be silent.
477
- if (config.founder === true && this.launchConfigs.get(strandId)?.founder !== true) {
478
- log('startStrand: strand %s is already running but was NOT launched as founder — ' +
479
- 'this early return DROPS the founder request; use foundExistingStrand', strandId);
480
- }
481
- log('Strand %s already running', strandId);
482
- return this.instances.get(strandId)!;
483
- }
484
-
485
- log('Starting strand instance: %s (sApp: %s v%s)', strandId, sAppConfig.id, sAppConfig.version);
486
- const tTotal = performance.now();
487
-
488
- // Verify schema signature before proceeding (fail-closed by default)
489
- const requireSignature = config.requireSignedSchemas ?? true;
490
- assertSchemaSignature(sAppConfig, { requireSignature });
491
- log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
492
-
493
- // Convert SAppConfig to SAppInfo for the instance
494
- const sAppInfo: SAppInfo = {
495
- id: sAppConfig.id,
496
- version: sAppConfig.version,
497
- schema: sAppConfig.schema,
498
- signature: sAppConfig.signature
499
- };
500
-
501
- // Resolve this strand's storage ONCE, before anything is recorded. If a factory
502
- // function is provided, it is called with the strandId to create strand-specific
503
- // storage (e.g. strand-isolated directories). A provider that throws therefore
504
- // fails the launch alongside the schema-signature check, leaving nothing tracked.
505
- const strandStorage = resolveStrandStorage(config.storage?.provider, strandId);
506
- if (strandStorage) {
507
- log('Strand %s using provided storage provider', strandId);
508
- }
509
-
510
- // Determine latency hint: sApp config > default
511
- const latencyHint = sAppConfig.latencyHint ?? config.defaultLatencyHint;
512
-
513
- const instance: StrandInstance = {
514
- strandId,
515
- status: 'starting',
516
- sAppInfo,
517
- memberPrivateKey: strandRow.MemberPrivateKey ?? undefined,
518
- connectedPeers: 0,
519
- lastActivity: new Date(),
520
- latencyHint
521
- };
522
-
523
- this.instances.set(strandId, instance);
524
- this.launchConfigs.set(strandId, config);
525
- if (strandStorage) {
526
- this.strandStorages.set(strandId, strandStorage);
527
- }
528
-
529
- try {
530
- await this.buildStrandRuntime(instance, config);
531
- timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
532
- log('Strand %s started successfully with sApp %s', strandId, sAppConfig.id);
533
- return instance;
534
- } catch (error) {
535
- // Status/error first — the (now discarded) record is still what `log` reports on.
536
- instance.status = 'error';
537
- instance.error = error instanceof Error ? error.message : String(error);
538
- log('Failed to start strand %s: %s', strandId, instance.error);
539
- // Drop the dead record so this strand id can be launched again. Keep the
540
- // `launchConfigs` has an entry iff `instances` does invariant — resumeStrand
541
- // reads both, and a config without an instance would strand the config.
542
- this.instances.delete(strandId);
543
- this.launchConfigs.delete(strandId);
544
- // The store this launch resolved goes with it: nothing owns it any more, and a
545
- // retained cache wrapper would be handed back (already retired) on a retry.
546
- await this.disposeStrandStorage(strandId);
547
- throw error;
548
- }
549
- }
550
-
551
- /**
552
- * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
553
- * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
554
- * launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
555
- * volatile inputs (bootstrapNodes, servingMachines, network, profile,
556
- * privateKey, sApp config) from `config`, so the caller controls the
557
- * cohort-derived values. Storage is the
558
- * one input it does NOT re-read from `config`: that belongs to the instance and
559
- * comes from `strandStorages`.
560
- */
561
- private async buildStrandRuntime(instance: StrandInstance, config: StartStrandConfig): Promise<void> {
562
- const strandId = instance.strandId;
563
- const { sAppConfig } = config;
564
-
565
- // The store the instance OWNS (resolved once in `startStrand`), not a fresh
566
- // resolution: a rebuild must reach the same backend through the same warm cache.
567
- const strandStorage = this.strandStorages.get(strandId);
568
-
569
- // db-p2p namespaces every one of the node's protocol ids by network name
570
- // (`/optimystic/<networkName>/...`), so anything dialing this node's own
571
- // services must derive its prefix from the SAME string the node was built
572
- // with — hence one binding for both, not two literals that can drift.
573
- const networkName = `strand-${strandId}`;
574
- const protocolPrefix = `/optimystic/${networkName}`;
575
-
576
- // Determine relay mode: if explicitly set in config, use that;
577
- // otherwise default to true for storage profile nodes.
578
- const enableRelay = config.network?.enableRelay ?? (config.profile === 'storage');
579
- // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
580
- // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
581
- // bind one port), the announce config is dropped (it names the control node's
582
- // address), and every configured relay becomes one bare `/p2p-circuit` SEARCH
583
- // listener plus a dial addr in `relayAddrs` for the per-relay supervisor started
584
- // below — the same route the control node takes, because the configured
585
- // `<relay>/p2p-circuit` shape loses its address on a relay restart, a hangup, or
586
- // libp2p's own reservation refresh and recovers from none of them. See
587
- // `strand-network-config.ts` for all three, and `relay-reservation.ts`.
588
- //
589
- // `relayAddrs` is destructured OFF: it is not a `createLibp2pNode` option, and
590
- // spreading it would hand db-p2p a key it does not know.
591
- const { relayAddrs: relayDialAddrs = [], ...addrOptions } = strandNodeAddrs(config.network);
592
-
593
- // The CLOSED-strand revoked-peer gate (see strand-revocation-enforcer.ts).
594
- // Constructed BEFORE the libp2p node because the node's options embed its
595
- // predicates; its deny set starts empty (admit everything — fail-open, same
596
- // posture as bring-up everywhere here) and is first populated after the
597
- // strand database initializes below. Registered in the map immediately so
598
- // the failure rollback (releaseRuntime) tears it down like every other
599
- // runtime component. Open strands skip it entirely: their Member/MemberPeer
600
- // tables are empty by schema (`OnlyClosed`), so there is nothing to derive
601
- // a deny set from — arming would be pointless polling.
602
- const revocationEnforcer =
603
- config.strandRow.Type === 'c' && config.revocationEnforcement?.enabled !== false
604
- ? new StrandRevocationEnforcer({
605
- label: strandId,
606
- readRows: () => {
607
- const database = instance.database;
608
- if (!database) {
609
- throw new Error(`strand ${strandId} has no live database`);
610
- }
611
- return readStrandRevocationRows(database.getDatabase());
612
- },
613
- // Read per sweep, never captured: this closure is built BEFORE the
614
- // libp2p node exists (its options embed the enforcer's predicates),
615
- // and `releaseRuntime` clears the field again on quiesce. Undefined
616
- // therefore means "no transport right now", which the enforcer
617
- // treats as "nothing to tear down".
618
- getNetwork: () => instance.libp2pNode,
619
- onSelfRevoked: () => config.onSelfRevoked?.(strandId)
620
- }, config.revocationEnforcement)
621
- : undefined;
622
- if (revocationEnforcer) {
623
- this.revocationEnforcers.set(strandId, revocationEnforcer);
624
- }
625
-
626
- // Closed strands compose revoked-peer denial onto the caller's gater and arm
627
- // the fail-closed per-stream gate; open strands keep the raw configured gater
628
- // (their peers are legitimately cross-party and nothing is ever revoked).
629
- const revocationGateOptions = revocationEnforcer
630
- ? {
631
- connectionGater: createRevocationConnectionGater(revocationEnforcer, config.network?.connectionGater),
632
- authorizeInboundStream: (remotePeerId: string, protocol: string) =>
633
- revocationEnforcer.authorizeStream(remotePeerId, protocol)
634
- }
635
- : config.network?.connectionGater
636
- ? { connectionGater: config.network.connectionGater }
637
- : {};
638
-
639
- try {
640
- // Bound once: the breadth is also the ceiling on the repair yardstick below, and the
641
- // two must be derived from the same resolution. Inside the `try` deliberately — a
642
- // rejected clusterSize is a build failure that runs the same cleanup as any other.
643
- const strandClusterSize = resolveStrandClusterSize(config.clusterSize);
644
-
645
- let t0 = performance.now();
646
- const node = await createLibp2pNode({
647
- port: 0, // Random port
648
- bootstrapNodes: config.bootstrapNodes ?? [],
649
- networkName,
650
- storage: strandStorage,
651
- fretProfile: config.profile === 'storage' ? 'core' : 'edge',
652
- relay: enableRelay,
653
- clusterSize: strandClusterSize,
654
- // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
655
- // breadth reasoning, and the shape match with the control policy is a coincidence.
656
- //
657
- // The builder declares this node's block-repair corroboration yardstick from the
658
- // count of machines SERVING this strand, capped at the breadth above (a block
659
- // never lives on more machines than the cohort is wide). Given no count — the
660
- // production path today, since no per-strand serving count exists yet — it
661
- // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
662
- // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
663
- // serving set that changed while the strand slept.
664
- clusterPolicy: strandClusterPolicy(strandClusterSize, config.servingMachines),
665
- arachnode: {
666
- enableRingZulu: config.profile === 'storage'
667
- },
668
- ...(config.privateKey && { privateKey: config.privateKey }),
669
- ...(config.network?.transports && { transports: config.network.transports }),
670
- ...(config.network?.noiseCrypto && { noiseCrypto: config.network.noiseCrypto }),
671
- // Listen entries plus the WebSocket transport switch they imply — a strand node
672
- // announces nothing the operator configured (`strand-network-config.ts`), and
673
- // spreads AFTER `transports` above because the switch is a no-op whenever the
674
- // embedder supplied its own factories. The bare `/p2p-circuit` entries (one per
675
- // relay) open no connection here; each registers a pending reservation that the
676
- // matching supervisor below fills, which is what gives a NAT'd strand node a
677
- // reachable relay slot — and works against a party-run relay because the launch
678
- // path announced this strand's derived peerId to it first (delegate admission;
679
- // see cadre-node.ts).
680
- ...addrOptions,
681
- // The raw configured gater (open strands), or the revocation-composed
682
- // gater plus the fail-closed per-stream revoked-peer gate (closed
683
- // strands) — resolved above, before the try.
684
- ...revocationGateOptions
685
- }) as Libp2pNodeWithRepo;
686
- timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
687
-
688
- instance.libp2pNode = node;
689
-
690
- // One reservation supervisor PER RELAY, started now so the first attempts
691
- // overlap the database bring-up below (the relay is never in a strand's
692
- // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
693
- // — so an open relay connection cannot disturb that bring-up). Their first
694
- // attempts are awaited before the strand goes `active`, below. Registered in the
695
- // map immediately so the failure rollback (releaseRuntime) stops them like every
696
- // other runtime component.
697
- this.relaySupervisors.set(strandId, this.startRelaySupervisors(strandId, node, relayDialAddrs, config));
698
-
699
- // Create and initialize the StrandDatabase.
700
- //
701
- // Held by the first-sync gate (registered before initialize) rather than attached
702
- // to the instance: a failed init is then cleaned up by releaseRuntime below through
703
- // the gate (close() is safe on a partially-initialized db), and the app only ever
704
- // sees `instance.database` once this machine is allowed to write to the strand.
705
- t0 = performance.now();
706
- const strandDb = new StrandDatabase({
707
- strandId,
708
- sAppConfig,
709
- libp2pNode: node,
710
- coordinatedRepo: node.coordinatedRepo,
711
- // Founder bootstrap inputs: the strand's type drives which membership rows
712
- // are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
713
- // caller from the control-layer StrandPartyKey row) derives the founding
714
- // Member/Manager key. The row's shared MemberPrivateKey rides along as the
715
- // read-gating secret only — it derives nobody's identity.
716
- strandType: config.strandRow.Type,
717
- memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
718
- partyMemberPrivateKey: config.partyMemberPrivateKey,
719
- founder: config.founder
720
- });
721
- const gate = new StrandFirstSyncGate({
722
- label: strandId,
723
- database: strandDb,
724
- onHeaderHeld: () => this.publishDatabase(instance, strandDb)
725
- }, config.firstSync);
726
- this.firstSyncGates.set(strandId, gate);
727
- await strandDb.initialize();
728
- timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
729
-
730
- // The first-sync write gate. A founder just wrote the Header in its bootstrap; any
731
- // other machine may commit only once it holds the Header — which it can only have
732
- // received from a peer (now, or on an earlier run over the same store) — and has read
733
- // each App table once, so their collections are fetched too. Publishing the database
734
- // is what makes the strand writable to the app; until then the instance stays
735
- // `'syncing'` and the gate re-probes on its cadence. Everything armed below reads
736
- // `instance.database` lazily, so the reconciler and enforcer wait with it — and
737
- // `publishDatabase` kicks the reconciler when the wait ends, so a joiner's membership
738
- // rows land with the strand rather than one retry later.
739
- t0 = performance.now();
740
- if (config.founder === true || await strandFirstSyncComplete(strandDb.getDatabase(), strandId)) {
741
- this.publishDatabase(instance, strandDb);
742
- } else {
743
- gate.start();
744
- }
745
- timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
746
-
747
- // The membership reconciler: finish this party's join on every machine —
748
- // redeem a staged formation invitation if one is pending, then write the
749
- // durable machine→party binding (Strand.MemberPeer). Armed here, the one
750
- // seam where the strand's transport peer id and a live Database both exist
751
- // for launch AND hibernation resume, for closed strands that carry the
752
- // party's own membership key. start() kicks an immediate pass WITHOUT
753
- // awaiting it — a joiner at this instant has not synced the founder's rows
754
- // and may lack write quorum, so bring-up is never blocked; an unfinished join
755
- // then retries on a short doubling ladder capped at the configured
756
- // pollIntervalMs (which is also the flat cadence while nobody has admitted this
757
- // party yet), and the loop stops once the member row and this machine's own
758
- // binding are both in place. Armed AFTER the gate above, so a gated launch
759
- // always has a reconciler registered by the time the gate can publish.
760
- if (config.strandRow.Type === 'c' && config.partyMemberPrivateKey
761
- && config.membershipReconciliation?.enabled !== false) {
762
- const reconciler = new StrandMembershipReconciler({
763
- label: strandId,
764
- partyMemberPrivateKey: config.partyMemberPrivateKey,
765
- // Read per pass, never captured — same lifecycle argument as the
766
- // enforcer's getNetwork: quiesce drops both handles.
767
- getDatabase: () => instance.database?.getDatabase(),
768
- getOwnPeerId: () => instance.libp2pNode?.peerId.toString(),
769
- pendingInvite: config.pendingMembershipInvite,
770
- // The enforcer's CURRENT snapshot view of this node's own peer id — the
771
- // reconciler stops rather than fight a self-revocation. Absent when the
772
- // gate is disarmed (fail-open: the loop just runs).
773
- isSelfRevoked: revocationEnforcer
774
- ? () => {
775
- const ownPeerId = instance.libp2pNode?.peerId.toString();
776
- return ownPeerId !== undefined && revocationEnforcer.isRevoked(ownPeerId);
777
- }
778
- : undefined
779
- }, {
780
- pollIntervalMs: config.membershipReconciliation?.pollIntervalMs
781
- ?? config.revocationEnforcement?.pollIntervalMs
782
- });
783
- this.membershipReconcilers.set(strandId, reconciler);
784
- reconciler.start();
785
- }
786
-
787
- // Arm the revoked-peer deny-set poll now that the database it reads
788
- // exists. start() kicks an immediate refresh WITHOUT awaiting it —
789
- // bring-up and resume are never blocked on a membership read; until the
790
- // first read lands the empty snapshot admits everyone (fail-open).
791
- revocationEnforcer?.start();
792
-
793
- // Peer-join block catch-up: push this strand's own blocks to each newly
794
- // connected peer, so a machine that joined after blocks were committed
795
- // still ends up physically holding them (without per-strand storage there
796
- // is nothing to copy). Armed for EVERY stored strand: `PeerJoinBackfill`
797
- // only does work when the strand's libp2p node reports a peer connection,
798
- // so on a device that is genuinely alone it is inert, and arming it at
799
- // launch is what closes the "founded alone, never replicates" hole — a
800
- // peer that joins later gets the founder's blocks without any relaunch.
801
- // No `authorizePeer` gate, deliberately — see the module comment in
802
- // peer-join-backfill.ts for the strand-side argument (and why the control
803
- // network, which DOES gate, is different).
804
- //
805
- // NOTE: cost is one PeerJoinBackfill object + one `connection:open` listener
806
- // per running strand — linear in strand count, negligible at the handful a
807
- // device or host runs today. If a node ever hosts strands by the hundred,
808
- // move to one shared listener that dispatches by strand id.
809
- if (strandStorage && config.backfill?.enabled !== false) {
810
- if (node.keyNetwork) {
811
- const backfill = new PeerJoinBackfill({
812
- label: strandId,
813
- libp2p: node,
814
- peerNetwork: node.keyNetwork,
815
- storage: strandStorage,
816
- // The same prefix the receiver registered its block-transfer handler
817
- // under — derived from networkName above, never re-spelled here.
818
- protocolPrefix
819
- }, config.backfill);
820
- backfill.start();
821
- this.backfills.set(strandId, backfill);
822
- } else {
823
- log('Strand %s: libp2p node exposes no keyNetwork; peer-join block catch-up is inert', strandId);
824
- }
825
- }
826
-
827
- // The relay supervisors' FIRST attempts, all of them, before `active`: the happy
828
- // path still publishes its circuit addr before `addStrand` resolves (the
829
- // strand-addr RPC answers read it, and the same-party circuit scenario asserts
830
- // it). Fail-SOFT — a first attempt that lands nothing does not fail the launch.
831
- // The strand's database is up, and the supervisor keeps trying on its backoff;
832
- // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
833
- //
834
- // NOTE: a relay that is down costs this launch one full drive (10 s), and
835
- // `StrandWatcher` launches strands one at a time — so N strands cost N × 10 s
836
- // of bring-up during a relay outage. If that ever matters, stop awaiting here
837
- // (the circuit addr then lands after `active`) rather than shortening the drive.
838
- t0 = performance.now();
839
- await this.awaitFirstRelayAttempts(strandId);
840
- timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
841
-
842
- instance.status = liveStrandStatus(instance);
843
- instance.lastActivity = new Date();
844
- if (instance.status === 'syncing') {
845
- log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
846
- 'a member of the strand is reached', strandId);
847
- }
848
- } catch (error) {
849
- // Roll back any partially-attached runtime so the instance is left with
850
- // NEITHER handle. Otherwise the `libp2pNode || database` "already live"
851
- // guard in resumeStrand/handleStrandWake would treat a half-built strand
852
- // as healthy — leaking the libp2p node and never retrying the rebuild.
853
- await this.releaseRuntime(instance).catch((cleanupErr) => {
854
- log('buildStrandRuntime cleanup for strand %s also failed: %o', strandId, cleanupErr);
855
- });
856
- throw error;
857
- }
858
- }
859
-
860
- /**
861
- * Release an instance's strand-network runtime: stop the relay supervisors and
862
- * the other background loops, close the StrandDatabase, then stop the libp2p
863
- * node (construction order in reverse), clearing both fields and zeroing
864
- * connectedPeers. Tolerant of partially-built state — either handle may be
865
- * absent — so it doubles as rollback for a failed `buildStrandRuntime`. Shared
866
- * by `quiesceStrand`, `stopStrand`, and that rollback path.
867
- *
868
- * Leaves `strandStorages` untouched by design — the store outlives the runtime it
869
- * was built into, which is what makes a resume warm. Only `stopStrand` disposes it.
870
- */
871
- private async releaseRuntime(instance: StrandInstance): Promise<void> {
872
- // Relay supervisors FIRST of all — before anything below is torn down — so no
873
- // re-drive is scheduled against a node being stopped. `stop()` is synchronous
874
- // and never awaits a drive, so a relay that is down cannot delay this teardown.
875
- // NOTE: a drive ALREADY in flight cannot be aborted (the drive takes no
876
- // AbortSignal — `backlog/bug-relay-drive-not-cancellable`); it fails soft
877
- // against the stopped node within its own 10 s deadline and its result is
878
- // discarded.
879
- const relaySupervisors = this.relaySupervisors.get(instance.strandId);
880
- if (relaySupervisors) {
881
- relaySupervisors.forEach((supervisor) => supervisor.stop());
882
- this.relaySupervisors.delete(instance.strandId);
883
- }
884
- // Backfill next — before the database closes and the libp2p node stops — so
885
- // no NEW catch-up push is issued against a torn-down transport. A push already
886
- // in flight is not awaited; it fails into the module's own per-chunk catch.
887
- const backfill = this.backfills.get(instance.strandId);
888
- if (backfill) {
889
- backfill.stop();
890
- this.backfills.delete(instance.strandId);
891
- }
892
- // The revoked-peer enforcer goes with the runtime it gated: a resume
893
- // rebuilds it with a fresh (initially empty, fail-open) snapshot.
894
- const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
895
- if (revocationEnforcer) {
896
- revocationEnforcer.stop();
897
- this.revocationEnforcers.delete(instance.strandId);
898
- }
899
- // The membership reconciler likewise: a resume rebuilds it and re-runs the
900
- // idempotent ladder, which is what makes resume-after-partial-join heal.
901
- const membershipReconciler = this.membershipReconcilers.get(instance.strandId);
902
- if (membershipReconciler) {
903
- membershipReconciler.stop();
904
- this.membershipReconcilers.delete(instance.strandId);
905
- }
906
- // A still-gated database (a joiner that never received the Header, or a launch whose
907
- // initialize() failed) is closed through its gate; a published one through the
908
- // instance. Never both — publishing moves the handle from the gate to the instance.
909
- const gate = this.firstSyncGates.get(instance.strandId);
910
- if (gate) {
911
- gate.stop();
912
- this.firstSyncGates.delete(instance.strandId);
913
- await gate.database.close();
914
- }
915
- if (instance.database) {
916
- await instance.database.close();
917
- instance.database = undefined;
918
- }
919
- if (instance.libp2pNode) {
920
- await instance.libp2pNode.stop();
921
- instance.libp2pNode = undefined;
922
- }
923
- instance.connectedPeers = 0;
924
- }
925
-
926
- /**
927
- * One {@link superviseRelayReservation} per relay dial addr, each over exactly
928
- * that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
929
- * one relay re-drives only that one. Default timings — the control node's (2 s
930
- * doubling to 60 s between failed attempts, a 5 s liveness check while held).
931
- *
932
- * Each supervisor's `beforeRedrive` is the caller's
933
- * {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
934
- * node's peer id, so a re-drive after a relay restart is preceded by a fresh
935
- * delegate grant on the relay about to be dialed. Returns `[]` when no relay is
936
- * configured — the strand then simply has no circuit listener to fill.
937
- */
938
- private startRelaySupervisors(
939
- strandId: string,
940
- node: Libp2pNodeWithRepo,
941
- relayDialAddrs: readonly string[],
942
- config: StartStrandConfig
943
- ): RelayReservationSupervisor[] {
944
- if (relayDialAddrs.length === 0) {
945
- return [];
946
- }
947
- const announce = config.announceDelegateToRelay;
948
- const delegatePeerId = node.peerId.toString();
949
- return relayDialAddrs.map((relayAddr) => superviseRelayReservation(node, [relayAddr], {
950
- ...(announce && { beforeRedrive: () => announce(strandId, relayAddr, delegatePeerId) })
951
- }));
952
- }
953
-
954
- /**
955
- * Wait for every relay supervisor's first attempt to settle, concurrently, and log
956
- * the ones that landed nothing. Never throws — a strand whose relay is unreachable
957
- * at launch still comes up, with its supervisors retrying in the background.
958
- */
959
- private async awaitFirstRelayAttempts(strandId: string): Promise<void> {
960
- const supervisors = this.relaySupervisors.get(strandId) ?? [];
961
- if (supervisors.length === 0) {
962
- return;
963
- }
964
- await Promise.all(supervisors.map((supervisor) => supervisor.firstAttempt));
965
- supervisors.forEach((supervisor) => {
966
- if (supervisor.lastError !== null) {
967
- log('Strand %s: relay reservation not held after the first attempt (retrying in the background): %s',
968
- strandId, supervisor.lastError);
969
- }
970
- });
971
- }
972
-
973
- /**
974
- * Drop the strand's owned store and release this strand's claim on its cache. The
975
- * wrapper counts holders, so the cache is emptied and unregistered from the shared
976
- * pool only if no other scope still holds it. Called only where the strand's whole
977
- * lifetime ends (`stopStrand`, and the failed-launch rollback) — never on a quiesce.
978
- *
979
- * A dispose failure is logged, not thrown: the store is already unreferenced here,
980
- * and failing the stop over a cache-bookkeeping error would leave the caller unable
981
- * to tear the strand down. `disposeStorageCache` no-ops for an unwrapped store
982
- * (e.g. `MemoryRawStorage`), so no instanceof test is needed.
983
- */
984
- private async disposeStrandStorage(strandId: string): Promise<void> {
985
- const storage = this.strandStorages.get(strandId);
986
- if (!storage) {
987
- return;
988
- }
989
- this.strandStorages.delete(strandId);
990
- try {
991
- await disposeStorageCache(storage);
992
- } catch (error) {
993
- log('Failed to dispose storage cache for strand %s: %o', strandId, error);
994
- }
995
- }
996
-
997
- /**
998
- * Refresh a strand's revoked-peer deny set NOW — and, with it, run the
999
- * teardown sweep that hangs up every connected revoked peer.
1000
- *
1001
- * The gate is otherwise poll-driven (default
1002
- * `DEFAULT_REVOCATION_POLL_INTERVAL_MS`), because neither a replicated
1003
- * revocation nor a local `revokeMember`/`leaveStrand` raises anything this
1004
- * runtime can hook. A caller that just wrote a revocation against the
1005
- * strand's database should follow it with this call, which makes the cut
1006
- * immediate instead of waiting out the interval.
1007
- *
1008
- * Quiet no-op — matching `quiesceStrand`'s posture — when the strand is not
1009
- * tracked, is quiesced, is open, or has the gate disabled: in every one of
1010
- * those cases there is no enforcer to refresh. Never rejects (the enforcer's
1011
- * refresh contains its own failures), and resolves only once the sweep has
1012
- * finished.
1013
- */
1014
- async refreshRevocationEnforcement(strandId: string): Promise<void> {
1015
- const enforcer = this.revocationEnforcers.get(strandId);
1016
- if (!enforcer) {
1017
- log('refreshRevocationEnforcement: strand %s has no armed revocation enforcer', strandId);
1018
- return;
1019
- }
1020
- await enforcer.refresh();
1021
- }
1022
-
1023
- /**
1024
- * Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
1025
- * self arm of `removeMemberPeer`, signed with the retained launch config's party
1026
- * key. Called by `CadreNode.unpublishStrand` BEFORE the local stop, while the
1027
- * strand database and transport are still live: after the unpublish commits, the
1028
- * party's `StrandPartyKey` control row is gone, so no restart could ever sign
1029
- * this removal again — the in-memory retained key is the last chance.
1030
- *
1031
- * Never throws. Quiet no-op for a strand that is untracked, open, or launched
1032
- * without a party key; a quiesced (hibernating) strand, or a strand whose write
1033
- * quorum is already unreachable, leaves the stale binding behind with a log —
1034
- * it grants nothing today (admission is not allowlist-gated) and only
1035
- * mis-credits diversity. The strand's own membership reconciler is stopped —
1036
- * and its in-flight pass awaited — FIRST, so a pass that was already past its
1037
- * own stopped check cannot re-register the binding this is about to clear.
1038
- *
1039
- * NOTE: deliberately NOT handled here (or anywhere yet): clearing the bindings
1040
- * of a machine removed from the CADRE at the control layer, or of a party's
1041
- * OTHER machines when the party unpublishes — the party still holds the key on
1042
- * every surviving machine and can clear from any of them, and the stale rows
1043
- * grant nothing while admission is deny-list only. Revisit when
1044
- * `feat-strand-member-allowlist-admission` lands and a stale binding starts
1045
- * granting admission rather than merely mis-crediting diversity.
1046
- */
1047
- async clearOwnMemberPeerBinding(strandId: string): Promise<void> {
1048
- const instance = this.instances.get(strandId);
1049
- const config = this.launchConfigs.get(strandId);
1050
- if (!instance || !config || config.strandRow.Type !== 'c' || !config.partyMemberPrivateKey) {
1051
- return;
1052
- }
1053
- const reconciler = this.membershipReconcilers.get(strandId);
1054
- if (reconciler) {
1055
- reconciler.stop();
1056
- await reconciler.settle();
1057
- }
1058
- const db = instance.database?.getDatabase();
1059
- const peerId = instance.libp2pNode?.peerId.toString();
1060
- if (!db || !peerId) {
1061
- log('clearOwnMemberPeerBinding: strand %s has no live runtime — own MemberPeer binding left in place', strandId);
1062
- return;
1063
- }
1064
- try {
1065
- // Background write on the app's database: never join a transaction the app has open.
1066
- await removeMemberPeer(db, {
1067
- memberKeyPair: strandMemberKeyPair(config.partyMemberPrivateKey),
1068
- peerId
1069
- }, { joinOpenTransaction: false });
1070
- log('clearOwnMemberPeerBinding: strand %s — own binding for peer %s cleared', strandId, peerId);
1071
- } catch (error) {
1072
- log('clearOwnMemberPeerBinding: strand %s — best-effort clear failed (strand may already be unreachable): %o',
1073
- strandId, error);
1074
- }
1075
- }
1076
-
1077
- /**
1078
- * Quiesce a strand: release its strand-network resources (stop the libp2p node,
1079
- * close the StrandDatabase) while RETAINING the instance record — identity,
1080
- * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
1081
- * resumed later. Mechanically this is `stopStrand` minus the instance/config
1082
- * deletion. The caller sets the post-quiesce status (e.g. `hibernating`).
1083
- * No-ops when the strand is missing or already quiesced.
1084
- */
1085
- async quiesceStrand(strandId: string): Promise<void> {
1086
- const instance = this.instances.get(strandId);
1087
- if (!instance) {
1088
- log('quiesceStrand: strand %s not found', strandId);
1089
- return;
1090
- }
1091
- if (!instance.libp2pNode && !instance.database) {
1092
- log('quiesceStrand: strand %s already quiesced', strandId);
1093
- return;
1094
- }
1095
-
1096
- log('Quiescing strand instance: %s', strandId);
1097
- await this.releaseRuntime(instance);
1098
- log('Strand %s quiesced (resources released, instance retained)', strandId);
1099
- }
1100
-
1101
- /**
1102
- * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
1103
- * from the retained launch config and re-attach them, transitioning it back to
1104
- * `active`. `overrides` re-applies volatile inputs that may have changed since
1105
- * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
1106
- * count) and updates the retained config so a later resume reuses the latest
1107
- * values. Returns the live instance unchanged if it is already running.
1108
- */
1109
- async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
1110
- if (this.stopping) {
1111
- throw new Error('StrandInstanceManager is stopping');
1112
- }
1113
- const instance = this.instances.get(strandId);
1114
- if (!instance) {
1115
- throw new Error(`Cannot resume strand ${strandId}: not tracked`);
1116
- }
1117
- const launchConfig = this.launchConfigs.get(strandId);
1118
- if (!launchConfig) {
1119
- throw new Error(`Cannot resume strand ${strandId}: no retained launch config`);
1120
- }
1121
- if (instance.libp2pNode || instance.database) {
1122
- log('resumeStrand: strand %s already live', strandId);
1123
- return instance;
1124
- }
1125
-
1126
- log('Resuming strand instance: %s', strandId);
1127
- const tTotal = performance.now();
1128
-
1129
- // Re-apply volatile inputs and persist them so a subsequent resume reuses them.
1130
- // Each `??` matters: a resume that passes no override must keep the retained value,
1131
- // and a resume that passes one must leave it retained for the next resume — otherwise
1132
- // a later no-override wake silently reverts to whatever launch time saw.
1133
- const resumeConfig: StartStrandConfig = {
1134
- ...launchConfig,
1135
- bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes,
1136
- // NOTE: `??` retains, so an override can raise or lower the count but cannot CLEAR it
1137
- // back to "this node no longer knows" — the direction that would declare nothing. Moot
1138
- // while nothing feeds `servingMachines` at all; if a source lands
1139
- // (`backlog/feat-strand-yardstick-from-serving-machines`) that can legitimately lose the
1140
- // count — a strand whose member rows became unreadable — this merge must gain an explicit
1141
- // clear rather than silently declaring a stale number over a serving set it can no longer
1142
- // see. Same applies to `bootstrapNodes` above, where a stale seed is harmless.
1143
- servingMachines: overrides?.servingMachines ?? launchConfig.servingMachines
1144
- };
1145
- this.launchConfigs.set(strandId, resumeConfig);
1146
-
1147
- instance.status = 'starting';
1148
- try {
1149
- await this.buildStrandRuntime(instance, resumeConfig);
1150
- timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
1151
- log('Strand %s resumed successfully', strandId);
1152
- return instance;
1153
- } catch (error) {
1154
- instance.status = 'error';
1155
- instance.error = error instanceof Error ? error.message : String(error);
1156
- log('Failed to resume strand %s: %s', strandId, instance.error);
1157
- throw error;
1158
- }
1159
- }
1160
-
1161
- /**
1162
- * Honor a founder request against an ALREADY-TRACKED strand — the seam that
1163
- * closes the "whoever launches first decides whether the bootstrap runs" gap:
1164
- * an instance first launched as a joiner (an app's own attach, or a watcher
1165
- * poll winning the launch race) used to swallow a later founder request
1166
- * silently, leaving the strand active with no `Strand.Header`.
1167
- *
1168
- * Flips the RETAINED launch config's `founder` to true, so every later
1169
- * quiesce → resume rebuild founds as well (the bootstrap is insert-if-absent —
1170
- * {@link StrandDatabase.ensureFounderBootstrap} — so re-running it per rebuild
1171
- * writes nothing twice), and runs the bootstrap against the live database now.
1172
- *
1173
- * @returns how the request resolved:
1174
- * - `'already-founder'` — the retained config already founds; nothing to do.
1175
- * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1176
- * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1177
- * database), so the bootstrap could not run here: the CALLER must wake the
1178
- * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1179
- * manager does not) so the rebuild — which now founds — runs it.
1180
- * @param resolvePartyKey - Asked for the party's own membership key when (and only
1181
- * when) a CLOSED strand's retained config carries none — the instance was launched
1182
- * as a joiner before its `StrandPartyKey` row existed or replicated. The resolved
1183
- * key is retained alongside the founder flip so the bootstrap (now, or on the
1184
- * caller's wake for `'needs-resume'`) can seat the founding Member/Manager.
1185
- * @throws when the strand is not tracked — this seam exists only for the
1186
- * tracked-instance launch path; an untracked id is a caller bug.
1187
- * @throws whatever the live bootstrap throws (e.g. `PreSplitStrandIdentityError`),
1188
- * after {@link withdrawFounderRequest} — so a retry re-attempts the founding rather
1189
- * than resolving `'already-founder'` over an instance that never founded. A caller
1190
- * whose `'needs-resume'` wake fails owes the same withdrawal.
1191
- */
1192
- async foundExistingStrand(
1193
- strandId: string,
1194
- resolvePartyKey?: () => Promise<string | undefined>
1195
- ): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'> {
1196
- const instance = this.instances.get(strandId);
1197
- const config = this.launchConfigs.get(strandId);
1198
- if (!instance || !config) {
1199
- throw new Error(`Cannot found strand ${strandId}: not tracked`);
1200
- }
1201
- if (config.founder === true) {
1202
- return 'already-founder';
1203
- }
1204
- const partyMemberPrivateKey =
1205
- config.partyMemberPrivateKey
1206
- ?? (config.strandRow.Type === 'c' ? await resolvePartyKey?.() : undefined);
1207
- // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1208
- // config object, which is not ours to rewrite.
1209
- this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1210
- if (!instance.database && !this.firstSyncGates.has(strandId)) {
1211
- return 'needs-resume';
1212
- }
1213
- try {
1214
- await this.ensureFounderBootstrap(strandId);
1215
- } catch (error) {
1216
- this.withdrawFounderRequest(strandId);
1217
- throw error;
1218
- }
1219
- return 'bootstrapped';
1220
- }
1221
-
1222
- /**
1223
- * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1224
- * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
1225
- * the hibernation manager's idle timer fired on it). `false` for a writable, quiesced,
1226
- * or untracked strand. The predicate {@link whenWritable}'s callers gate on.
1227
- */
1228
- isAwaitingFirstSync(strandId: string): boolean {
1229
- const instance = this.instances.get(strandId);
1230
- return instance !== undefined && isAwaitingFirstSync(instance);
1231
- }
1232
-
1233
- /**
1234
- * Resolve once the strand's database is published to the app — immediately for a
1235
- * writable strand — or reject with {@link StrandAwaitingFirstSyncError} after
1236
- * `timeoutMs` (default: the retained launch config's `firstSync.timeoutMs`, else
1237
- * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}). The rejection is RETRYABLE: nothing is
1238
- * torn down, the gate keeps probing, and a later call waits afresh. A quiesced
1239
- * (hibernating) strand is not woken here — the wait spans a resume, so a caller that
1240
- * wants one wakes it (`CadreNode.wakeStrand`) and waits. Rejects immediately for an
1241
- * untracked strand, and whenever the strand is stopped while a wait is pending.
1242
- */
1243
- whenWritable(strandId: string, options?: { timeoutMs?: number }): Promise<StrandInstance> {
1244
- const instance = this.instances.get(strandId);
1245
- if (!instance) {
1246
- return Promise.reject(new Error(`Cannot wait for strand ${strandId} to become writable: not tracked`));
1247
- }
1248
- if (instance.database) {
1249
- return Promise.resolve(instance);
1250
- }
1251
- const timeoutMs = options?.timeoutMs
1252
- ?? this.launchConfigs.get(strandId)?.firstSync?.timeoutMs
1253
- ?? DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS;
1254
- const startedAt = Date.now();
1255
- return new Promise<StrandInstance>((resolve, reject) => {
1256
- const waiters = this.writableWaiters.get(strandId) ?? new Set<WritableWaiter>();
1257
- this.writableWaiters.set(strandId, waiters);
1258
- const settle = (): void => {
1259
- clearTimeout(timer);
1260
- waiters.delete(waiter);
1261
- if (waiters.size === 0) {
1262
- this.writableWaiters.delete(strandId);
1263
- }
1264
- };
1265
- const waiter: WritableWaiter = {
1266
- resolve: () => { settle(); resolve(instance); },
1267
- reject: (error) => { settle(); reject(error); }
1268
- };
1269
- const timer = setTimeout(
1270
- () => waiter.reject(new StrandAwaitingFirstSyncError(strandId, Date.now() - startedAt)),
1271
- timeoutMs);
1272
- (timer as { unref?: () => void }).unref?.();
1273
- waiters.add(waiter);
1274
- });
1275
- }
1276
-
1277
- /**
1278
- * Hand a launch's database to the app: set `instance.database`, retire the gate that
1279
- * held it, release every {@link whenWritable} waiter, and — when the launch had already
1280
- * been reported gated — flip it `'active'` and announce it through the retained config's
1281
- * `onWritable`. "Reported gated" is any status but `'starting'`: `'syncing'`, or the
1282
- * `'idle'` the hibernation manager's idle timer flips a long-gated joiner to (see
1283
- * {@link isAwaitingFirstSync}) — the Header arriving after that flip must announce too,
1284
- * or an app hanging on `strand:writable` never learns the strand opened. A publish that
1285
- * lands while the instance is still `'starting'` (the Header arrived during the rest of
1286
- * bring-up) announces nothing: `startStrand` / `resumeStrand` are about to report the
1287
- * strand `'active'`, and `strand:writable` is defined as the follow-up to a gated
1288
- * launch, never a duplicate of `strand:started`.
1289
- *
1290
- * It is also the one seam where a WITHHELD database becomes available, so it kicks the
1291
- * membership reconciler: on a gated launch that loop's own immediate pass ran while
1292
- * `instance.database` was still unset, found no database, and would otherwise not try
1293
- * again until its retry timer fired. A launch that was never gated publishes here BEFORE
1294
- * the reconciler is constructed, so there is nothing registered to kick and the loop's own
1295
- * immediate pass already sees the database — which is why the kick needs no `wasGated`
1296
- * test of its own.
1297
- */
1298
- private publishDatabase(instance: StrandInstance, database: StrandDatabase): void {
1299
- const strandId = instance.strandId;
1300
- const gate = this.firstSyncGates.get(strandId);
1301
- if (gate) {
1302
- gate.open();
1303
- this.firstSyncGates.delete(strandId);
1304
- }
1305
- instance.database = database;
1306
- instance.lastActivity = new Date();
1307
- const wasGated = instance.status !== 'starting';
1308
- if (wasGated) {
1309
- instance.status = 'active';
1310
- log('Strand %s is now writable: Strand.Header held', strandId);
1311
- }
1312
- const waiters = this.writableWaiters.get(strandId);
1313
- if (waiters) {
1314
- this.writableWaiters.delete(strandId);
1315
- waiters.forEach((waiter) => waiter.resolve());
1316
- }
1317
- if (wasGated) {
1318
- this.launchConfigs.get(strandId)?.onWritable?.(strandId);
1319
- }
1320
- // Not awaited: publishing must stay synchronous, and the pass chains on the loop's own
1321
- // tail, so it cannot overlap a pass already in flight.
1322
- void this.membershipReconcilers.get(strandId)?.reconcile();
1323
- }
1324
-
1325
- /** Reject every {@link whenWritable} waiter for a strand that is going away. */
1326
- private rejectWritableWaiters(strandId: string, reason: string): void {
1327
- const waiters = this.writableWaiters.get(strandId);
1328
- if (!waiters) {
1329
- return;
1330
- }
1331
- this.writableWaiters.delete(strandId);
1332
- waiters.forEach((waiter) => waiter.reject(new Error(`Strand ${strandId} ${reason} before becoming writable`)));
1333
- }
1334
-
1335
- /**
1336
- * Undo {@link foundExistingStrand}'s founder flip after the founding it promised
1337
- * failed, so the retained config again says what this instance actually runs as (a
1338
- * joiner) and the next founder request re-attempts the bootstrap. The resolved party
1339
- * key stays retained — it is this party's identity for the strand either way. No-op
1340
- * when the strand is untracked or its config does not found.
1341
- */
1342
- withdrawFounderRequest(strandId: string): void {
1343
- const config = this.launchConfigs.get(strandId);
1344
- if (config?.founder === true) {
1345
- this.launchConfigs.set(strandId, { ...config, founder: false });
1346
- }
1347
- }
1348
-
1349
- /**
1350
- * Run the (idempotent) founder bootstrap against a tracked strand's LIVE
1351
- * database, independently of what the retained launch config says.
1352
- *
1353
- * Separate from {@link foundExistingStrand} because a caller that resolved
1354
- * `'needs-resume'` and woke the strand must not assume the wake's own rebuild
1355
- * founded it: `HibernationManager` COALESCES wakes, so a wake already in flight
1356
- * when the config flipped had already read the PRE-flip config and rebuilt as a
1357
- * joiner. Re-running the bootstrap costs one insert-if-absent probe per table
1358
- * and is the only thing that makes "founding resolves once the Header is
1359
- * written" true on that path.
1360
- *
1361
- * A strand still `'syncing'` (its database withheld behind the first-sync gate) is
1362
- * founded through the gate: the bootstrap writes the Header this machine was waiting
1363
- * to receive, so a successful run publishes the database and the strand goes
1364
- * `'active'` — the one legitimate way a joiner launch becomes writable without a peer.
1365
- *
1366
- * @throws when the strand is not tracked, or is still quiesced (no live
1367
- * database) — both mean the bootstrap did NOT run, which a founder request
1368
- * must never swallow.
1369
- */
1370
- async ensureFounderBootstrap(strandId: string): Promise<void> {
1371
- const instance = this.instances.get(strandId);
1372
- if (!instance) {
1373
- throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: not tracked`);
1374
- }
1375
- const gate = this.firstSyncGates.get(strandId);
1376
- const database = instance.database ?? gate?.database;
1377
- if (!database) {
1378
- throw new Error(
1379
- `Cannot run the founder bootstrap for strand ${strandId}: it is quiesced, so there ` +
1380
- 'is no live database to write to — resume it first.'
1381
- );
1382
- }
1383
- // Forward the RETAINED config's party key: a foundExistingStrand that flipped a
1384
- // joiner launch to founder may have resolved a key the live database's captured
1385
- // config (built at the original launch) never saw.
1386
- await database.ensureFounderBootstrap(this.launchConfigs.get(strandId)?.partyMemberPrivateKey);
1387
- if (!instance.database) {
1388
- this.publishDatabase(instance, database);
1389
- }
1390
- }
1391
-
1392
- /**
1393
- * Stop a strand instance
1394
- */
1395
- async stopStrand(strandId: string): Promise<void> {
1396
- const instance = this.instances.get(strandId);
1397
- if (!instance) {
1398
- log('Strand %s not found', strandId);
1399
- return;
1400
- }
1401
-
1402
- log('Stopping strand instance: %s', strandId);
1403
- instance.status = 'stopping';
1404
- this.rejectWritableWaiters(strandId, 'was stopped');
1405
-
1406
- try {
1407
- await this.releaseRuntime(instance);
1408
- instance.status = 'stopped';
1409
- this.instances.delete(strandId);
1410
- this.launchConfigs.delete(strandId);
1411
- // Storage is released only here — NOT in releaseRuntime, which a quiesce shares.
1412
- await this.disposeStrandStorage(strandId);
1413
- log('Strand %s stopped successfully', strandId);
1414
- } catch (error) {
1415
- instance.status = 'error';
1416
- instance.error = error instanceof Error ? error.message : String(error);
1417
- log('Error stopping strand %s: %s', strandId, instance.error);
1418
- throw error;
1419
- }
1420
- }
1421
-
1422
- /**
1423
- * Stop all strand instances
1424
- */
1425
- async stopAll(): Promise<void> {
1426
- this.stopping = true;
1427
- log('Stopping all strand instances (%d)', this.instances.size);
1428
-
1429
- const stopPromises = Array.from(this.instances.keys()).map(id =>
1430
- this.stopStrand(id).catch(err => {
1431
- log('Error stopping strand %s during shutdown: %s', id, err);
1432
- })
1433
- );
1434
-
1435
- await Promise.all(stopPromises);
1436
- this.stopping = false;
1437
- log('All strand instances stopped');
1438
- }
1439
- }
1440
-
1
+ import debug from 'debug';
2
+ import type { PrivateKey } from '@libp2p/interface';
3
+ import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
4
+ import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
5
+ import { StrandDatabase } from './strand-database.js';
6
+ import { PeerJoinBackfill, type PeerJoinBackfillConfig } from './peer-join-backfill.js';
7
+ import {
8
+ StrandRevocationEnforcer,
9
+ createRevocationConnectionGater,
10
+ readStrandRevocationRows,
11
+ type StrandRevocationEnforcementConfig
12
+ } from './strand-revocation-enforcer.js';
13
+ import {
14
+ StrandMembershipReconciler,
15
+ type PendingMembershipInviteSource,
16
+ type StrandMembershipReconciliationConfig
17
+ } from './strand-membership-reconciler.js';
18
+ import { removeMemberPeer } from './strand-membership-writer.js';
19
+ import { strandMemberKeyPair } from './strand-member-key.js';
20
+ import { assertSchemaSignature } from './schema-verification.js';
21
+ import { assertStrandScopeKey } from './storage-scope.js';
22
+ import {
23
+ StrandFirstSyncGate,
24
+ StrandAwaitingFirstSyncError,
25
+ strandFirstSyncComplete,
26
+ DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS,
27
+ type StrandFirstSyncConfig
28
+ } from './strand-first-sync-gate.js';
29
+ import type {
30
+ StrandInstance,
31
+ StrandRow,
32
+ StorageConfig,
33
+ NetworkConfig,
34
+ LatencyHint,
35
+ NodeProfile,
36
+ SAppConfig,
37
+ SAppInfo,
38
+ RawStorageProvider,
39
+ Libp2pNodeWithRepo
40
+ } from './types.js';
41
+ import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
42
+ import { strandNodeAddrs } from './strand-network-config.js';
43
+ import { superviseRelayReservation, type RelayReservationSupervisor } from './relay-reservation.js';
44
+ import { peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
45
+
46
+ const log = debug('sereus:cadre:strand-manager');
47
+ const timing = debug('sereus:cadre:timing');
48
+
49
+ /** One caller blocked in {@link StrandInstanceManager.whenWritable}. */
50
+ interface WritableWaiter {
51
+ resolve: () => void;
52
+ reject: (error: Error) => void;
53
+ }
54
+
55
+ /**
56
+ * Whether a tracked instance is launched but waiting for its first sync: the runtime is
57
+ * up (`libp2pNode` set) while the database is still withheld (`database` unset). That
58
+ * is the first-sync gate's state regardless of what `status` says — the hibernation
59
+ * manager's idle timer can flip a long-gated joiner to `'idle'` without changing it.
60
+ *
61
+ * NOTE: `status` is therefore not the source of truth for "gated": a joiner that reaches
62
+ * nobody for `idleTimeout` (5 min at the interactive hint) reads `'idle'`, then
63
+ * `'hibernating'`, while still holding no database. Harmless today — every decision the
64
+ * runtime makes goes through this predicate, {@link StrandInstanceManager.publishDatabase}
65
+ * announces on the idle-flipped instance exactly as on a `'syncing'` one, and an app that
66
+ * shows "waiting for the other member" on `'syncing'` merely sees "idle" instead after
67
+ * five minutes. If an app ever needs the gated LABEL to survive the idle flip, make the
68
+ * hibernation manager's idle transition preserve `'syncing'` rather than adding a flag.
69
+ */
70
+ export function isAwaitingFirstSync(instance: StrandInstance): boolean {
71
+ return instance.libp2pNode !== undefined && instance.database === undefined;
72
+ }
73
+
74
+ /**
75
+ * The status a LIVE instance (runtime rebuilt or woken) should report: `'active'` once its
76
+ * database is published, `'syncing'` while the first-sync gate still withholds it. Every
77
+ * site that would otherwise write `'active'` after a build or a wake goes through this,
78
+ * so a gated joiner never reads as writable.
79
+ */
80
+ export function liveStrandStatus(instance: StrandInstance): 'active' | 'syncing' {
81
+ return instance.database ? 'active' : 'syncing';
82
+ }
83
+
84
+ /**
85
+ * Configuration for starting a strand instance
86
+ */
87
+ export interface StartStrandConfig {
88
+ strandRow: StrandRow;
89
+ /** sApp configuration provided by the hosting application */
90
+ sAppConfig: SAppConfig;
91
+ storage?: StorageConfig;
92
+ network?: NetworkConfig;
93
+ profile: NodeProfile;
94
+ defaultLatencyHint: LatencyHint;
95
+ privateKey?: PrivateKey;
96
+ /** Cohort-derived discovery seed (multiaddr strings). Defaults to [] when omitted. */
97
+ bootstrapNodes?: string[];
98
+ /**
99
+ * Require a valid author signature on the sApp schema before bring-up.
100
+ * Defaults to true (fail closed) when omitted; set false only for dev/test
101
+ * with unsigned demo schemas. Mirrors {@link CadreNodeConfig.requireSignedSchemas}.
102
+ */
103
+ requireSignedSchemas?: boolean;
104
+ /**
105
+ * Whether this node founds the strand (vs. joins it). Forwarded to the
106
+ * StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
107
+ * runs once at bring-up. Joiners leave this unset and write nothing. Callers
108
+ * resolve it BEFORE calling `startStrand` (`CadreNode.launchStrand` derives it
109
+ * from the row's `FounderOwnerKey` when no explicit flag is given — see
110
+ * {@link StrandConfig.founder}); a founder request that arrives while the
111
+ * instance is already tracked goes through {@link StrandInstanceManager.foundExistingStrand},
112
+ * never through a repeat `startStrand`.
113
+ */
114
+ founder?: boolean;
115
+ /**
116
+ * THIS party's own strand membership private key (base64 protobuf) for a closed
117
+ * strand — the identity the founder bootstrap derives `Member.Key` /
118
+ * `Manager.MemberKey` from. Resolved by the caller (`CadreNode.launchStrand`: an
119
+ * explicit attach-time key, else the control-layer `StrandPartyKey` row, healed by
120
+ * mint-on-launch for the founding machine) and retained with the launch config, so a
121
+ * hibernation wake rebuilds under the same identity. Deliberately NOT the strand
122
+ * row's shared `MemberPrivateKey` (see `StrandDatabaseConfig.partyMemberPrivateKey`).
123
+ * Absent for open strands, and for joiners with no persisted party key.
124
+ */
125
+ partyMemberPrivateKey?: string;
126
+ /**
127
+ * Number of nodes Optimystic is told this strand's replication cluster should
128
+ * have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
129
+ * forwards here: every node on the strand should use the same value, and it is
130
+ * frozen when the strand's libp2p node is created. Defaults to
131
+ * `DEFAULT_STRAND_CLUSTER_SIZE` (4); values below `MIN_CLUSTER_SIZE` (2) are
132
+ * rejected by `resolveStrandClusterSize`. Strand-only — the
133
+ * control network's breadth is the fixed `CONTROL_REPLICATION_BREADTH`.
134
+ */
135
+ clusterSize?: number;
136
+ /**
137
+ * Machines that SERVE THIS STRAND (this node included), for the strand node's
138
+ * block-repair corroboration yardstick — see `resolveRepairYardstick`. It must be
139
+ * an authenticated per-strand count: the machines that actually run a node for this
140
+ * strand, not the machines that exist.
141
+ *
142
+ * **The party's enrolled-machine count is NOT this number and must never be passed
143
+ * here.** A strand launches only on machines whose embedding app registered its sApp
144
+ * config (`CadreNode.addStrand`), so a closed strand shared by two machines of a
145
+ * three-machine party is served by two. Passing three over-declares, and
146
+ * over-declaring is the unsafe direction: at a declaration of 3 or more Optimystic
147
+ * pins the repair corroboration floor at two corroborating peers, which a cohort
148
+ * that can only ever field one peer can never reach — so the strand can never repair
149
+ * a block (`cluster-fetch:no-quorum`, surfacing as reads failing with `Missing
150
+ * block`). That regression is why this field was renamed off "enrolledMachines":
151
+ * `bug-strand-yardstick-counts-party-machines`.
152
+ *
153
+ * **Nothing feeds it in production today.** No authenticated per-strand serving count
154
+ * exists yet, so `CadreNode` passes nothing and the strand node runs the frozen
155
+ * `STRAND_CLUSTER_POLICY` — declaring no yardstick, which leaves the known,
156
+ * upstream-tracked single-voter exposure
157
+ * (`backlog/debt-read-repair-single-voter-corroboration`). Building the count is
158
+ * `backlog/feat-strand-yardstick-from-serving-machines`; this field and its
159
+ * threading are the seam it plugs into.
160
+ *
161
+ * Volatile when a source does exist: re-resolve it on every resume beside the cohort
162
+ * seed, since machines join and leave a strand while it hibernates. Omitting it means
163
+ * "this node does not know", which declares nothing.
164
+ */
165
+ servingMachines?: number;
166
+ /**
167
+ * Tuning for the strand peer-join block catch-up ({@link PeerJoinBackfill}),
168
+ * forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
169
+ * libp2p node connects to a peer this runtime has not yet caught up, every
170
+ * block in the strand's own raw store is pushed to it. Runs only on strands
171
+ * with per-strand storage; `{ enabled: false }` restores the pre-existing
172
+ * no-backfill behaviour.
173
+ */
174
+ backfill?: PeerJoinBackfillConfig;
175
+ /**
176
+ * Tuning for the CLOSED-strand revoked-peer gate
177
+ * ({@link StrandRevocationEnforcer}), forwarded from
178
+ * {@link CadreNodeConfig.strandRevocationEnforcement}. Armed only for
179
+ * `strandRow.Type === 'c'` (an open strand has no membership rows to derive a
180
+ * deny set from); `{ enabled: false }` restores the pre-existing behaviour
181
+ * (a removed party's peers keep being served).
182
+ */
183
+ revocationEnforcement?: StrandRevocationEnforcementConfig;
184
+
185
+ /**
186
+ * The staged formation membership invitation for THIS strand — the seam the
187
+ * bring-up membership reconciler (`strand-membership-reconciler.ts`) reads to
188
+ * redeem the party's `Strand.Member` seat, and clears once the invitation is
189
+ * spent, burned, or dead. `CadreNode.launchStrand` wires it over its in-memory
190
+ * `pendingMembershipInvites` map; read lazily per pass, so a re-formation that
191
+ * replaces the entry between passes is picked up. Only consulted for closed
192
+ * strands launched with a {@link partyMemberPrivateKey}; harmless otherwise.
193
+ */
194
+ pendingMembershipInvite?: PendingMembershipInviteSource;
195
+
196
+ /**
197
+ * Tuning for the CLOSED-strand membership reconciler, forwarded from
198
+ * {@link CadreNodeConfig.strandMembershipReconciliation}. Armed only for
199
+ * closed strands launched with a {@link partyMemberPrivateKey};
200
+ * `{ enabled: false }` disarms the loop (test fixtures that hand-drive the
201
+ * membership writers). When it names no `pollIntervalMs` the reconciler
202
+ * mirrors {@link revocationEnforcement}'s cadence — as its IDLE cadence and
203
+ * the cap of its unfinished-join retry ladder, not as a flat retry interval.
204
+ */
205
+ membershipReconciliation?: StrandMembershipReconciliationConfig;
206
+
207
+ /**
208
+ * Called when THIS node's own peer id turns up in the strand's revoked set —
209
+ * this node's party was removed from the closed strand, or left it. Fires once
210
+ * per removal (re-armed if the party is re-admitted; a resume also rebuilds the
211
+ * enforcer and may legitimately re-fire), and only for closed strands with the
212
+ * gate armed. `CadreNode`
213
+ * wires it to its `strand:revoked` event; nothing is stopped or torn down on
214
+ * this node's behalf — what to do about it is the app's call.
215
+ */
216
+ onSelfRevoked?: (strandId: string) => void;
217
+
218
+ /**
219
+ * Called when the membership reconciler reports a blocked re-join — this node holds a
220
+ * staged membership invitation for the strand that its writes cannot redeem, which is
221
+ * what a REMOVED party handed a fresh invitation hits (see "Reporting a blocked
222
+ * re-join" in `strand-membership-reconciler.ts` for the two triggers, one of them a
223
+ * suspicion rather than a verdict). At most once per re-arm of the loop — a further
224
+ * invitation staged while the loop is still running does not reset the report — and
225
+ * only for closed strands with a party key. `CadreNode` wires it to its `strand:rejoin-blocked`
226
+ * event; nothing is stopped or torn down — a remaining manager admitting this party's
227
+ * key directly is the remedy, and the loop finishes the join by itself once that lands.
228
+ */
229
+ onRejoinBlocked?: (strandId: string) => void;
230
+
231
+ /**
232
+ * Tuning for the JOINING machine's first-sync write gate (`strand-first-sync-gate.ts`),
233
+ * forwarded from {@link CadreNodeConfig.strandFirstSync}: the Header probe cadence while
234
+ * a non-founder launch is `'syncing'`, and the default budget {@link whenWritable} waits
235
+ * before rejecting. Founders and machines that already hold the Header are never gated.
236
+ */
237
+ firstSync?: StrandFirstSyncConfig;
238
+
239
+ /**
240
+ * Called when a launch that came up `'syncing'` (see {@link firstSync}) becomes
241
+ * writable: the strand's `Strand.Header` arrived from a peer — or a founder request
242
+ * ran the bootstrap against the gated database — and `instance.database` is now set.
243
+ * Never called for a launch that was writable when `startStrand`/`resumeStrand`
244
+ * resolved. `CadreNode` wires it to its `strand:writable` event. Retained with the
245
+ * launch config, so a hibernation resume that comes up gated announces too.
246
+ */
247
+ onWritable?: (strandId: string) => void;
248
+
249
+ /**
250
+ * Re-announce this strand's delegate peer id (`delegatePeerId` — the strand
251
+ * node's own transport peer id) to ONE relay (`relayAddr`, a direct dial addr
252
+ * with a trailing `/p2p/<relayPeerId>`), unthrottled. Awaited before every
253
+ * relay-reservation RE-drive of that relay — never before the first attempt,
254
+ * which the launch/resume seed pass has already announced for.
255
+ *
256
+ * Why: a party control node running the relay server admits the strand node on
257
+ * an in-memory delegate grant that a relay restart drops, so a re-drive that did
258
+ * not re-announce first would be denied at the relay's gate. `CadreNode` wires
259
+ * its own control-mesh announce here; a throw or rejection is logged by the
260
+ * supervisor and the re-drive still runs. Retained with the launch config, so a
261
+ * hibernation wake's rebuilt supervisors carry it too.
262
+ */
263
+ announceDelegateToRelay?: (strandId: string, relayAddr: string, delegatePeerId: string) => Promise<void>;
264
+ }
265
+
266
+ /**
267
+ * Volatile inputs re-resolved when resuming a quiesced strand — the ones that can
268
+ * have moved since it last ran, and that the rebuilt libp2p node freezes again. Each
269
+ * one omitted keeps the value the retained launch config already holds, so a resume
270
+ * that passes nothing rebuilds the strand exactly as it last ran.
271
+ */
272
+ export interface ResumeStrandOverrides {
273
+ /**
274
+ * Freshly-resolved cohort discovery seed (multiaddr strings). Grows as peers are
275
+ * learned since the strand first launched.
276
+ */
277
+ bootstrapNodes?: string[];
278
+ /**
279
+ * Freshly-read count of the machines serving this strand, for the repair yardstick
280
+ * (see {@link StartStrandConfig.servingMachines} — including why the party's
281
+ * enrolled-machine count is not it). Moves whenever a machine starts or stops
282
+ * serving the strand, which a hibernating strand does not otherwise notice. Nothing
283
+ * passes it today. Omitting it here does NOT mean "unknown" the way omitting it from
284
+ * {@link StartStrandConfig} does — it retains the last value; see `resumeStrand`.
285
+ */
286
+ servingMachines?: number;
287
+ }
288
+
289
+ /**
290
+ * Get the isolated storage path for a specific strand.
291
+ *
292
+ * @deprecated This helper is Node-only and throws in React Native (it assumes a
293
+ * filesystem layout). Use a storage provider factory function instead, which
294
+ * receives the strandId and can create strand-specific storage paths using
295
+ * platform-appropriate methods.
296
+ *
297
+ * @example
298
+ * // Instead of using getStrandStoragePath, use a storage provider factory:
299
+ * const storage = {
300
+ * provider: (strandId: string) => new FileRawStorage(`./data/strands/${strandId}`)
301
+ * };
302
+ */
303
+ export function getStrandStoragePath(basePath: string, strandId: string): string {
304
+ // Check if we're in a Node.js environment
305
+ if (typeof process === 'undefined' || !process.versions?.node) {
306
+ throw new Error(
307
+ 'getStrandStoragePath is not available in React Native. ' +
308
+ 'Use a storage provider factory function instead.'
309
+ );
310
+ }
311
+
312
+ // Sanitize strandId for filesystem safety (UUIDs should be safe, but just in case)
313
+ const safeId = strandId.replace(/[^a-zA-Z0-9-]/g, '_');
314
+
315
+ // Build the path with plain string joins rather than the Node `path` module.
316
+ // A static `require('path')` forces RN bundlers (e.g. Metro) to *resolve* the
317
+ // module at bundle time even though this Node-only helper throws above before
318
+ // ever reaching here — joining by hand keeps the module free of any Node
319
+ // built-in reference, so RN bundles need no `path` shim.
320
+ const trimmedBase = basePath.replace(/[\\/]+$/, '');
321
+ return `${trimmedBase}/strands/${safeId}`;
322
+ }
323
+
324
+ /**
325
+ * Resolve a storage provider for a specific strand.
326
+ * If the provider is a factory function, call it with the strandId.
327
+ *
328
+ * Called only from {@link StrandInstanceManager.startStrand}, which owns the result
329
+ * for the instance's lifetime — see `strandStorages`, and which has already asserted
330
+ * the id is usable as a scope key. Do not move that assertion here: this function
331
+ * returns early when no provider is configured, and the id becomes a libp2p protocol
332
+ * prefix on that path too.
333
+ *
334
+ * @param provider - Storage provider (instance or factory)
335
+ * @param strandId - The strand ID to create storage for
336
+ * @returns The resolved IRawStorage instance, or undefined if no provider
337
+ */
338
+ function resolveStrandStorage(
339
+ provider: RawStorageProvider | undefined,
340
+ strandId: string
341
+ ): IRawStorage | undefined {
342
+ if (!provider) {
343
+ return undefined;
344
+ }
345
+ const storage = typeof provider === 'function' ? provider(strandId) : provider;
346
+ // Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's cached-storage.ts).
347
+ // Called ONCE per strand launch — `startStrand` keeps the result for the instance's
348
+ // lifetime — so the wrap survives every runtime rebuild the instance goes through.
349
+ return wrapStorageWithCache(storage, strandId);
350
+ }
351
+
352
+ /**
353
+ * Manages individual strand instances - creates and destroys isolated libp2p nodes
354
+ * for each strand the cadre participates in.
355
+ */
356
+ export class StrandInstanceManager {
357
+ private instances: Map<string, StrandInstance> = new Map();
358
+ /**
359
+ * Retained launch config per strand, captured in `startStrand` and cleared in
360
+ * `stopStrand`. `resumeStrand` reuses it to rebuild a quiesced strand's runtime
361
+ * without the caller re-threading storage/network/profile/key/sApp config.
362
+ */
363
+ private launchConfigs: Map<string, StartStrandConfig> = new Map();
364
+ /**
365
+ * The per-strand peer-join block catch-up, keyed by strand id. Private (not on
366
+ * the public {@link StrandInstance}) because it is runtime plumbing with the
367
+ * same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
368
+ * stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
369
+ * with a fresh caught-up-peer memo, which is intended (a resumed node may have
370
+ * missed writes).
371
+ */
372
+ private backfills: Map<string, PeerJoinBackfill> = new Map();
373
+ /**
374
+ * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
375
+ * id. Same lifecycle rationale as {@link backfills}: created in
376
+ * `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so quiesce →
377
+ * resume rebuilds it with a fresh (initially empty, fail-open) snapshot.
378
+ */
379
+ private revocationEnforcers: Map<string, StrandRevocationEnforcer> = new Map();
380
+ /**
381
+ * The per-strand membership reconciler (closed strands launched with a party
382
+ * key only), keyed by strand id. Same lifecycle rationale as {@link backfills}:
383
+ * created in `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so
384
+ * quiesce → resume rebuilds it and re-runs the idempotent join ladder (redeem a
385
+ * staged invitation, write this machine's own `MemberPeer` binding) from scratch.
386
+ */
387
+ private membershipReconcilers: Map<string, StrandMembershipReconciler> = new Map();
388
+ /**
389
+ * The per-strand first-sync gate (`strand-first-sync-gate.ts`), keyed by strand id:
390
+ * holds a launch's `StrandDatabase` while it is NOT yet published on the instance.
391
+ * Present from the moment the database object exists in `buildStrandRuntime` until
392
+ * {@link publishDatabase} hands it to the app — so it doubles as the rollback handle
393
+ * for a failed `initialize()` — and, for a joiner that holds no `Strand.Header` yet,
394
+ * for as long as the launch stays `'syncing'`. Same lifecycle as {@link backfills}:
395
+ * stopped (its database closed) and dropped in `releaseRuntime`, so a quiesce →
396
+ * resume rebuild re-probes over the same store. An entry and `instance.database`
397
+ * are mutually exclusive.
398
+ */
399
+ private firstSyncGates: Map<string, StrandFirstSyncGate> = new Map();
400
+ /**
401
+ * Callers blocked in {@link whenWritable}, keyed by strand id. Resolved by
402
+ * {@link publishDatabase}, rejected by {@link stopStrand}; each waiter also carries
403
+ * its own timeout. Deliberately NOT cleared by `releaseRuntime`: a waiter outlives a
404
+ * quiesce → resume cycle, since the rebuild is what may finally publish the database.
405
+ */
406
+ private writableWaiters: Map<string, Set<WritableWaiter>> = new Map();
407
+ /**
408
+ * The per-strand relay-reservation supervisors — ONE PER CONFIGURED RELAY, each
409
+ * over the node's own bare `/p2p-circuit` listener for that relay
410
+ * (`strand-network-config.ts`) — keyed by strand id. Same lifecycle rationale as
411
+ * {@link backfills}: started in `buildStrandRuntime` right after the libp2p node
412
+ * exists, stopped FIRST and dropped in `releaseRuntime` (before the database
413
+ * closes and the node stops, so no re-drive dials a node being torn down), so
414
+ * quiesce → resume rebuilds them over the new node. What makes a strand node's
415
+ * lost relay slot come back on its own — the control node has the same loop in
416
+ * `CadreNode.reserveRelays`.
417
+ */
418
+ private relaySupervisors: Map<string, RelayReservationSupervisor[]> = new Map();
419
+ /**
420
+ * The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
421
+ * store, resolved once in `startStrand` and held until `stopStrand` disposes it.
422
+ * Private for the same reason `backfills` is: runtime plumbing, not part of the
423
+ * public {@link StrandInstance}.
424
+ *
425
+ * Deliberately NOT touched by `releaseRuntime`: that is what lets a quiesce →
426
+ * resume cycle rebuild the libp2p node over the SAME store, keeping its
427
+ * write-through cache warm (and, on an in-memory backend, keeping the strand's
428
+ * blocks at all). An entry exists iff `instances` does AND the launch config
429
+ * supplied a storage provider.
430
+ *
431
+ * NOTE: a hibernating strand therefore keeps its store — and its share of the
432
+ * process-wide cache pool — resident for as long as the instance is tracked. That
433
+ * is the point (a warm wake), and the pool evicts under pressure, so the cost is
434
+ * one map entry per hibernating strand today. If a device ever hibernates strands
435
+ * by the hundred, revisit: dropping the store at quiesce and paying for a cold
436
+ * wake becomes the better trade.
437
+ */
438
+ private strandStorages: Map<string, IRawStorage> = new Map();
439
+ private stopping = false;
440
+
441
+ constructor() {
442
+ log('StrandInstanceManager created');
443
+ }
444
+
445
+ /**
446
+ * Get all current strand instances
447
+ */
448
+ getInstances(): Map<string, StrandInstance> {
449
+ return new Map(this.instances);
450
+ }
451
+
452
+ /**
453
+ * Get a specific strand instance
454
+ */
455
+ getInstance(strandId: string): StrandInstance | undefined {
456
+ return this.instances.get(strandId);
457
+ }
458
+
459
+ /**
460
+ * Check if a strand is currently running
461
+ */
462
+ hasStrand(strandId: string): boolean {
463
+ return this.instances.has(strandId);
464
+ }
465
+
466
+ /**
467
+ * Start a new strand instance.
468
+ *
469
+ * A failed launch leaves NOTHING tracked: the instance and its retained launch
470
+ * config are both dropped before the error is rethrown, so the strand id is
471
+ * free for a genuine retry. This matches the pre-registration failure path
472
+ * (a rejected schema signature, which throws before anything is recorded) —
473
+ * both failure modes of this call leave the same residue: none. Callers learn
474
+ * of the failure from the rejected promise (and, on the control-discovered
475
+ * path, from CadreNode's `strand:error` event), not from an error record left
476
+ * behind in `instances`.
477
+ *
478
+ * The strand's raw storage is resolved HERE, once, and held for the instance's
479
+ * lifetime (see `strandStorages`) — `buildStrandRuntime` only reads it, so a
480
+ * hibernation wake never re-enters the embedder's provider.
481
+ */
482
+ async startStrand(config: StartStrandConfig): Promise<StrandInstance> {
483
+ const { strandRow, sAppConfig } = config;
484
+ const strandId = strandRow.Id;
485
+
486
+ if (this.stopping) {
487
+ throw new Error('StrandInstanceManager is stopping');
488
+ }
489
+
490
+ if (this.instances.has(strandId)) {
491
+ // Callers resolve founder-ness BEFORE reaching here (CadreNode.launchStrand) and
492
+ // honor a founder request on a tracked instance via foundExistingStrand — so a
493
+ // founder flag arriving at this early return against a non-founder retained
494
+ // config is a dropped bootstrap, and must never again be silent.
495
+ if (config.founder === true && this.launchConfigs.get(strandId)?.founder !== true) {
496
+ log('startStrand: strand %s is already running but was NOT launched as founder — ' +
497
+ 'this early return DROPS the founder request; use foundExistingStrand', strandId);
498
+ }
499
+ log('Strand %s already running', strandId);
500
+ return this.instances.get(strandId)!;
501
+ }
502
+
503
+ log('Starting strand instance: %s (sApp: %s v%s)', strandId, sAppConfig.id, sAppConfig.version);
504
+ const tTotal = performance.now();
505
+
506
+ // The id becomes two names below: the storage scope key the embedder's provider
507
+ // turns into a directory or database name, and `networkName` in buildStrandRuntime,
508
+ // from which the libp2p protocol prefix `/optimystic/strand-<id>` is built. A
509
+ // replicated strand row carries whatever id the founding node wrote, so neither is
510
+ // safe unchecked. Asserted HERE rather than in `resolveStrandStorage` because that
511
+ // function returns early when no provider is configured — a node with no storage
512
+ // still reaches buildStrandRuntime and still mints the protocol prefix.
513
+ assertStrandScopeKey(strandId);
514
+
515
+ // Verify schema signature before proceeding (fail-closed by default)
516
+ const requireSignature = config.requireSignedSchemas ?? true;
517
+ assertSchemaSignature(sAppConfig, { requireSignature });
518
+ log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
519
+
520
+ // Convert SAppConfig to SAppInfo for the instance
521
+ const sAppInfo: SAppInfo = {
522
+ id: sAppConfig.id,
523
+ version: sAppConfig.version,
524
+ schema: sAppConfig.schema,
525
+ signature: sAppConfig.signature
526
+ };
527
+
528
+ // Resolve this strand's storage ONCE, before anything is recorded. If a factory
529
+ // function is provided, it is called with the strandId to create strand-specific
530
+ // storage (e.g. strand-isolated directories). A provider that throws therefore
531
+ // fails the launch alongside the schema-signature check, leaving nothing tracked.
532
+ const strandStorage = resolveStrandStorage(config.storage?.provider, strandId);
533
+ if (strandStorage) {
534
+ log('Strand %s using provided storage provider', strandId);
535
+ }
536
+
537
+ // Determine latency hint: sApp config > default
538
+ const latencyHint = sAppConfig.latencyHint ?? config.defaultLatencyHint;
539
+
540
+ const instance: StrandInstance = {
541
+ strandId,
542
+ status: 'starting',
543
+ sAppInfo,
544
+ memberPrivateKey: strandRow.MemberPrivateKey ?? undefined,
545
+ connectedPeers: 0,
546
+ lastActivity: new Date(),
547
+ latencyHint
548
+ };
549
+
550
+ this.instances.set(strandId, instance);
551
+ this.launchConfigs.set(strandId, config);
552
+ if (strandStorage) {
553
+ this.strandStorages.set(strandId, strandStorage);
554
+ }
555
+
556
+ try {
557
+ await this.buildStrandRuntime(instance, config);
558
+ timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
559
+ log('Strand %s started successfully with sApp %s', strandId, sAppConfig.id);
560
+ return instance;
561
+ } catch (error) {
562
+ // Status/error first — the (now discarded) record is still what `log` reports on.
563
+ instance.status = 'error';
564
+ instance.error = error instanceof Error ? error.message : String(error);
565
+ log('Failed to start strand %s: %s', strandId, instance.error);
566
+ // Drop the dead record so this strand id can be launched again. Keep the
567
+ // `launchConfigs` has an entry iff `instances` does invariant — resumeStrand
568
+ // reads both, and a config without an instance would strand the config.
569
+ this.instances.delete(strandId);
570
+ this.launchConfigs.delete(strandId);
571
+ // The store this launch resolved goes with it: nothing owns it any more, and a
572
+ // retained cache wrapper would be handed back (already retired) on a retry.
573
+ await this.disposeStrandStorage(strandId);
574
+ throw error;
575
+ }
576
+ }
577
+
578
+ /**
579
+ * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
580
+ * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
581
+ * launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
582
+ * volatile inputs (bootstrapNodes, servingMachines, network, profile,
583
+ * privateKey, sApp config) from `config`, so the caller controls the
584
+ * cohort-derived values. Storage is the
585
+ * one input it does NOT re-read from `config`: that belongs to the instance and
586
+ * comes from `strandStorages`.
587
+ */
588
+ private async buildStrandRuntime(instance: StrandInstance, config: StartStrandConfig): Promise<void> {
589
+ const strandId = instance.strandId;
590
+ const { sAppConfig } = config;
591
+
592
+ // Refuse a bad `linkRoundTripMs` before any strand bring-up, for the reason
593
+ // `CadreNode.start()` does: every budget derived from it here is behind a condition
594
+ // (backfill disabled, no relay addrs), so a zero or NaN declaration would otherwise
595
+ // surface later inside a path that logs and carries on. `link-budget.ts`.
596
+ resolveLinkRoundTripMs(config.network?.linkRoundTripMs);
597
+
598
+ // The store the instance OWNS (resolved once in `startStrand`), not a fresh
599
+ // resolution: a rebuild must reach the same backend through the same warm cache.
600
+ const strandStorage = this.strandStorages.get(strandId);
601
+
602
+ // db-p2p namespaces every one of the node's protocol ids by network name
603
+ // (`/optimystic/<networkName>/...`), so anything dialing this node's own
604
+ // services must derive its prefix from the SAME string the node was built
605
+ // with — hence one binding for both, not two literals that can drift.
606
+ const networkName = `strand-${strandId}`;
607
+ const protocolPrefix = `/optimystic/${networkName}`;
608
+
609
+ // Determine relay mode: if explicitly set in config, use that;
610
+ // otherwise default to true for storage profile nodes.
611
+ const enableRelay = config.network?.enableRelay ?? (config.profile === 'storage');
612
+ // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
613
+ // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
614
+ // bind one port), the announce config is dropped (it names the control node's
615
+ // address), and every configured relay becomes one bare `/p2p-circuit` SEARCH
616
+ // listener plus a dial addr in `relayAddrs` for the per-relay supervisor started
617
+ // below — the same route the control node takes, because the configured
618
+ // `<relay>/p2p-circuit` shape loses its address on a relay restart, a hangup, or
619
+ // libp2p's own reservation refresh and recovers from none of them. See
620
+ // `strand-network-config.ts` for all three, and `relay-reservation.ts`.
621
+ //
622
+ // `relayAddrs` is destructured OFF: it is not a `createLibp2pNode` option, and
623
+ // spreading it would hand db-p2p a key it does not know.
624
+ const { relayAddrs: relayDialAddrs = [], ...addrOptions } = strandNodeAddrs(config.network);
625
+
626
+ // The CLOSED-strand revoked-peer gate (see strand-revocation-enforcer.ts).
627
+ // Constructed BEFORE the libp2p node because the node's options embed its
628
+ // predicates; its deny set starts empty (admit everything — fail-open, same
629
+ // posture as bring-up everywhere here) and is first populated after the
630
+ // strand database initializes below. Registered in the map immediately so
631
+ // the failure rollback (releaseRuntime) tears it down like every other
632
+ // runtime component. Open strands skip it entirely: their Member/MemberPeer
633
+ // tables are empty by schema (`OnlyClosed`), so there is nothing to derive
634
+ // a deny set from — arming would be pointless polling.
635
+ const revocationEnforcer =
636
+ config.strandRow.Type === 'c' && config.revocationEnforcement?.enabled !== false
637
+ ? new StrandRevocationEnforcer({
638
+ label: strandId,
639
+ readRows: () => {
640
+ const database = instance.database;
641
+ if (!database) {
642
+ throw new Error(`strand ${strandId} has no live database`);
643
+ }
644
+ return readStrandRevocationRows(database.getDatabase());
645
+ },
646
+ // Read per sweep, never captured: this closure is built BEFORE the
647
+ // libp2p node exists (its options embed the enforcer's predicates),
648
+ // and `releaseRuntime` clears the field again on quiesce. Undefined
649
+ // therefore means "no transport right now", which the enforcer
650
+ // treats as "nothing to tear down".
651
+ getNetwork: () => instance.libp2pNode,
652
+ onSelfRevoked: () => config.onSelfRevoked?.(strandId)
653
+ }, config.revocationEnforcement)
654
+ : undefined;
655
+ if (revocationEnforcer) {
656
+ this.revocationEnforcers.set(strandId, revocationEnforcer);
657
+ }
658
+
659
+ // Closed strands compose revoked-peer denial onto the caller's gater and arm
660
+ // the fail-closed per-stream gate; open strands keep the raw configured gater
661
+ // (their peers are legitimately cross-party and nothing is ever revoked).
662
+ const revocationGateOptions = revocationEnforcer
663
+ ? {
664
+ connectionGater: createRevocationConnectionGater(revocationEnforcer, config.network?.connectionGater),
665
+ authorizeInboundStream: (remotePeerId: string, protocol: string) =>
666
+ revocationEnforcer.authorizeStream(remotePeerId, protocol)
667
+ }
668
+ : config.network?.connectionGater
669
+ ? { connectionGater: config.network.connectionGater }
670
+ : {};
671
+
672
+ try {
673
+ // Bound once: the breadth is also the ceiling on the repair yardstick below, and the
674
+ // two must be derived from the same resolution. Inside the `try` deliberately — a
675
+ // rejected clusterSize is a build failure that runs the same cleanup as any other.
676
+ const strandClusterSize = resolveStrandClusterSize(config.clusterSize);
677
+
678
+ let t0 = performance.now();
679
+ const node = await createLibp2pNode({
680
+ port: 0, // Random port
681
+ bootstrapNodes: config.bootstrapNodes ?? [],
682
+ networkName,
683
+ storage: strandStorage,
684
+ fretProfile: config.profile === 'storage' ? 'core' : 'edge',
685
+ relay: enableRelay,
686
+ clusterSize: strandClusterSize,
687
+ // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
688
+ // breadth reasoning, and the shape match with the control policy is a coincidence.
689
+ //
690
+ // The builder declares this node's block-repair corroboration yardstick from the
691
+ // count of machines SERVING this strand, capped at the breadth above (a block
692
+ // never lives on more machines than the cohort is wide). Given no count — the
693
+ // production path today, since no per-strand serving count exists yet — it
694
+ // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
695
+ // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
696
+ // serving set that changed while the strand slept.
697
+ // The read deadline is the same field the control node reads, because the two
698
+ // networks ride one link; absent, the base policy's COHORT_READ_DEADLINE_MS stands.
699
+ clusterPolicy: strandClusterPolicy(strandClusterSize, {
700
+ servingMachines: config.servingMachines,
701
+ cohortQueryTimeoutMs: config.network?.cohortQueryTimeoutMs
702
+ }),
703
+ arachnode: {
704
+ enableRingZulu: config.profile === 'storage'
705
+ },
706
+ ...(config.privateKey && { privateKey: config.privateKey }),
707
+ ...(config.network?.transports && { transports: config.network.transports }),
708
+ ...(config.network?.noiseCrypto && { noiseCrypto: config.network.noiseCrypto }),
709
+ // Unconditional, and the same default the control node takes: every node of
710
+ // every party runs the monitor over its connection to a slow phone, so the
711
+ // widened ping deadline has to reach the strand nodes too (see
712
+ // DEFAULT_CONNECTION_MONITOR).
713
+ connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
714
+ // Listen entries plus the WebSocket transport switch they imply — a strand node
715
+ // announces nothing the operator configured (`strand-network-config.ts`), and
716
+ // spreads AFTER `transports` above because the switch is a no-op whenever the
717
+ // embedder supplied its own factories. The bare `/p2p-circuit` entries (one per
718
+ // relay) open no connection here; each registers a pending reservation that the
719
+ // matching supervisor below fills, which is what gives a NAT'd strand node a
720
+ // reachable relay slot — and works against a party-run relay because the launch
721
+ // path announced this strand's derived peerId to it first (delegate admission;
722
+ // see cadre-node.ts).
723
+ ...addrOptions,
724
+ // The raw configured gater (open strands), or the revocation-composed
725
+ // gater plus the fail-closed per-stream revoked-peer gate (closed
726
+ // strands) — resolved above, before the try.
727
+ ...revocationGateOptions
728
+ }) as Libp2pNodeWithRepo;
729
+ timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
730
+
731
+ instance.libp2pNode = node;
732
+
733
+ // One reservation supervisor PER RELAY, started now so the first attempts
734
+ // overlap the database bring-up below (the relay is never in a strand's
735
+ // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
736
+ // — so an open relay connection cannot disturb that bring-up). Their first
737
+ // attempts are awaited before the strand goes `active`, below. Registered in the
738
+ // map immediately so the failure rollback (releaseRuntime) stops them like every
739
+ // other runtime component.
740
+ this.relaySupervisors.set(strandId, this.startRelaySupervisors(strandId, node, relayDialAddrs, config));
741
+
742
+ // Create and initialize the StrandDatabase.
743
+ //
744
+ // Held by the first-sync gate (registered before initialize) rather than attached
745
+ // to the instance: a failed init is then cleaned up by releaseRuntime below through
746
+ // the gate (close() is safe on a partially-initialized db), and the app only ever
747
+ // sees `instance.database` once this machine is allowed to write to the strand.
748
+ t0 = performance.now();
749
+ const strandDb = new StrandDatabase({
750
+ strandId,
751
+ sAppConfig,
752
+ libp2pNode: node,
753
+ coordinatedRepo: node.coordinatedRepo,
754
+ // Founder bootstrap inputs: the strand's type drives which membership rows
755
+ // are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
756
+ // caller from the control-layer StrandPartyKey row) derives the founding
757
+ // Member/Manager key. The row's shared MemberPrivateKey rides along as the
758
+ // read-gating secret only — it derives nobody's identity.
759
+ strandType: config.strandRow.Type,
760
+ memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
761
+ partyMemberPrivateKey: config.partyMemberPrivateKey,
762
+ founder: config.founder
763
+ });
764
+ const gate = new StrandFirstSyncGate({
765
+ label: strandId,
766
+ database: strandDb,
767
+ onHeaderHeld: () => this.publishDatabase(instance, strandDb)
768
+ }, config.firstSync);
769
+ this.firstSyncGates.set(strandId, gate);
770
+ await strandDb.initialize();
771
+ timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
772
+
773
+ // The first-sync write gate. A founder just wrote the Header in its bootstrap; any
774
+ // other machine may commit only once it holds the Header — which it can only have
775
+ // received from a peer (now, or on an earlier run over the same store) — and has read
776
+ // each App table once, so their collections are fetched too. Publishing the database
777
+ // is what makes the strand writable to the app; until then the instance stays
778
+ // `'syncing'` and the gate re-probes on its cadence. Everything armed below reads
779
+ // `instance.database` lazily, so the reconciler and enforcer wait with it — and
780
+ // `publishDatabase` kicks the reconciler when the wait ends, so a joiner's membership
781
+ // rows land with the strand rather than one retry later.
782
+ t0 = performance.now();
783
+ if (config.founder === true || await strandFirstSyncComplete(strandDb.getDatabase(), strandId)) {
784
+ this.publishDatabase(instance, strandDb);
785
+ } else {
786
+ gate.start();
787
+ }
788
+ timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
789
+
790
+ // The membership reconciler: finish this party's join on every machine —
791
+ // redeem a staged formation invitation if one is pending, then write the
792
+ // durable machine→party binding (Strand.MemberPeer). Armed here, the one
793
+ // seam where the strand's transport peer id and a live Database both exist
794
+ // for launch AND hibernation resume, for closed strands that carry the
795
+ // party's own membership key. start() kicks an immediate pass WITHOUT
796
+ // awaiting it — a joiner at this instant has not synced the founder's rows
797
+ // and may lack write quorum, so bring-up is never blocked; an unfinished join
798
+ // then retries on a short doubling ladder capped at the configured
799
+ // pollIntervalMs (which is also the flat cadence while nobody has admitted this
800
+ // party yet), and the loop stops once the member row and this machine's own
801
+ // binding are both in place. Armed AFTER the gate above, so a gated launch
802
+ // always has a reconciler registered by the time the gate can publish.
803
+ if (config.strandRow.Type === 'c' && config.partyMemberPrivateKey
804
+ && config.membershipReconciliation?.enabled !== false) {
805
+ const reconciler = new StrandMembershipReconciler({
806
+ label: strandId,
807
+ partyMemberPrivateKey: config.partyMemberPrivateKey,
808
+ // Read per pass, never captured — same lifecycle argument as the
809
+ // enforcer's getNetwork: quiesce drops both handles.
810
+ getDatabase: () => instance.database?.getDatabase(),
811
+ getOwnPeerId: () => instance.libp2pNode?.peerId.toString(),
812
+ pendingInvite: config.pendingMembershipInvite,
813
+ // The enforcer's CURRENT snapshot view of this node's own peer id — the
814
+ // reconciler stops rather than fight a self-revocation. Absent when the
815
+ // gate is disarmed (fail-open: the loop just runs).
816
+ isSelfRevoked: revocationEnforcer
817
+ ? () => {
818
+ const ownPeerId = instance.libp2pNode?.peerId.toString();
819
+ return ownPeerId !== undefined && revocationEnforcer.isRevoked(ownPeerId);
820
+ }
821
+ : undefined,
822
+ onRejoinBlocked: () => config.onRejoinBlocked?.(strandId)
823
+ }, {
824
+ pollIntervalMs: config.membershipReconciliation?.pollIntervalMs
825
+ ?? config.revocationEnforcement?.pollIntervalMs
826
+ });
827
+ this.membershipReconcilers.set(strandId, reconciler);
828
+ reconciler.start();
829
+ }
830
+
831
+ // Arm the revoked-peer deny-set poll now that the database it reads
832
+ // exists. start() kicks an immediate refresh WITHOUT awaiting it —
833
+ // bring-up and resume are never blocked on a membership read; until the
834
+ // first read lands the empty snapshot admits everyone (fail-open).
835
+ revocationEnforcer?.start();
836
+
837
+ // Peer-join block catch-up: push this strand's own blocks to each newly
838
+ // connected peer, so a machine that joined after blocks were committed
839
+ // still ends up physically holding them (without per-strand storage there
840
+ // is nothing to copy). Armed for EVERY stored strand: `PeerJoinBackfill`
841
+ // only does work when the strand's libp2p node reports a peer connection,
842
+ // so on a device that is genuinely alone it is inert, and arming it at
843
+ // launch is what closes the "founded alone, never replicates" hole — a
844
+ // peer that joins later gets the founder's blocks without any relaunch.
845
+ // No `authorizePeer` gate, deliberately — see the module comment in
846
+ // peer-join-backfill.ts for the strand-side argument (and why the control
847
+ // network, which DOES gate, is different).
848
+ //
849
+ // NOTE: cost is one PeerJoinBackfill object + one `connection:open` listener
850
+ // per running strand — linear in strand count, negligible at the handful a
851
+ // device or host runs today. If a node ever hosts strands by the hundred,
852
+ // move to one shared listener that dispatches by strand id.
853
+ if (strandStorage && config.backfill?.enabled !== false) {
854
+ if (node.keyNetwork) {
855
+ const backfill = new PeerJoinBackfill({
856
+ label: strandId,
857
+ libp2p: node,
858
+ peerNetwork: node.keyNetwork,
859
+ storage: strandStorage,
860
+ // The same prefix the receiver registered its block-transfer handler
861
+ // under — derived from networkName above, never re-spelled here.
862
+ protocolPrefix
863
+ }, {
864
+ // Dial and response deadlines counted in link round trips, not fixed milliseconds:
865
+ // a relayed dial costs a fixed number of exchanges, so a fixed budget has a link
866
+ // speed above which this catch-up can never reach the peer at all. See
867
+ // `link-budget.ts`. Spread BEFORE the host's own config so an explicit
868
+ // `strandBackfill.dialTimeoutMs` still wins.
869
+ ...peerJoinPushBudget(config.network?.linkRoundTripMs),
870
+ ...config.backfill
871
+ });
872
+ backfill.start();
873
+ this.backfills.set(strandId, backfill);
874
+ } else {
875
+ log('Strand %s: libp2p node exposes no keyNetwork; peer-join block catch-up is inert', strandId);
876
+ }
877
+ }
878
+
879
+ // The relay supervisors' FIRST attempts, all of them, before `active`: the happy
880
+ // path still publishes its circuit addr before `addStrand` resolves (the
881
+ // strand-addr RPC answers read it, and the same-party circuit scenario asserts
882
+ // it). Fail-SOFT — a first attempt that lands nothing does not fail the launch.
883
+ // The strand's database is up, and the supervisor keeps trying on its backoff;
884
+ // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
885
+ //
886
+ // NOTE: a relay that is down costs this launch one full drive — the reservation budget
887
+ // counted from the declared link round trip, 8 s at its default (`link-budget.ts`) — and
888
+ // `StrandWatcher` launches strands one at a time, so N strands cost N of those in
889
+ // bring-up during a relay outage, and MORE on a host that declared a slower link. If that
890
+ // ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
891
+ // than shortening the drive.
892
+ t0 = performance.now();
893
+ await this.awaitFirstRelayAttempts(strandId);
894
+ timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
895
+
896
+ instance.status = liveStrandStatus(instance);
897
+ instance.lastActivity = new Date();
898
+ if (instance.status === 'syncing') {
899
+ log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
900
+ 'a member of the strand is reached', strandId);
901
+ }
902
+ } catch (error) {
903
+ // Roll back any partially-attached runtime so the instance is left with
904
+ // NEITHER handle. Otherwise the `libp2pNode || database` "already live"
905
+ // guard in resumeStrand/handleStrandWake would treat a half-built strand
906
+ // as healthy — leaking the libp2p node and never retrying the rebuild.
907
+ await this.releaseRuntime(instance).catch((cleanupErr) => {
908
+ log('buildStrandRuntime cleanup for strand %s also failed: %o', strandId, cleanupErr);
909
+ });
910
+ throw error;
911
+ }
912
+ }
913
+
914
+ /**
915
+ * Release an instance's strand-network runtime: stop the relay supervisors and
916
+ * the other background loops, close the StrandDatabase, then stop the libp2p
917
+ * node (construction order in reverse), clearing both fields and zeroing
918
+ * connectedPeers. Tolerant of partially-built state — either handle may be
919
+ * absent — so it doubles as rollback for a failed `buildStrandRuntime`. Shared
920
+ * by `quiesceStrand`, `stopStrand`, and that rollback path.
921
+ *
922
+ * Leaves `strandStorages` untouched by design — the store outlives the runtime it
923
+ * was built into, which is what makes a resume warm. Only `stopStrand` disposes it.
924
+ */
925
+ private async releaseRuntime(instance: StrandInstance): Promise<void> {
926
+ // Relay supervisors FIRST of all — before anything below is torn down — so no
927
+ // re-drive is scheduled against a node being stopped. `stop()` is synchronous
928
+ // and never awaits a drive, so a relay that is down cannot delay this teardown.
929
+ // `stop()` also cancels the drive already in flight, so an attempt started
930
+ // moments before this does not keep dialing and polling the node the lines
931
+ // below are stopping; its result is discarded either way.
932
+ const relaySupervisors = this.relaySupervisors.get(instance.strandId);
933
+ if (relaySupervisors) {
934
+ relaySupervisors.forEach((supervisor) => supervisor.stop());
935
+ this.relaySupervisors.delete(instance.strandId);
936
+ }
937
+ // Backfill next — before the database closes and the libp2p node stops — so
938
+ // no NEW catch-up push is issued against a torn-down transport. A push already
939
+ // in flight is not awaited; it fails into the module's own per-chunk catch.
940
+ const backfill = this.backfills.get(instance.strandId);
941
+ if (backfill) {
942
+ backfill.stop();
943
+ this.backfills.delete(instance.strandId);
944
+ }
945
+ // The revoked-peer enforcer goes with the runtime it gated: a resume
946
+ // rebuilds it with a fresh (initially empty, fail-open) snapshot.
947
+ const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
948
+ if (revocationEnforcer) {
949
+ revocationEnforcer.stop();
950
+ this.revocationEnforcers.delete(instance.strandId);
951
+ }
952
+ // The membership reconciler likewise: a resume rebuilds it and re-runs the
953
+ // idempotent ladder, which is what makes resume-after-partial-join heal.
954
+ const membershipReconciler = this.membershipReconcilers.get(instance.strandId);
955
+ if (membershipReconciler) {
956
+ membershipReconciler.stop();
957
+ this.membershipReconcilers.delete(instance.strandId);
958
+ }
959
+ // A still-gated database (a joiner that never received the Header, or a launch whose
960
+ // initialize() failed) is closed through its gate; a published one through the
961
+ // instance. Never both — publishing moves the handle from the gate to the instance.
962
+ const gate = this.firstSyncGates.get(instance.strandId);
963
+ if (gate) {
964
+ gate.stop();
965
+ this.firstSyncGates.delete(instance.strandId);
966
+ await gate.database.close();
967
+ }
968
+ if (instance.database) {
969
+ await instance.database.close();
970
+ instance.database = undefined;
971
+ }
972
+ if (instance.libp2pNode) {
973
+ await instance.libp2pNode.stop();
974
+ instance.libp2pNode = undefined;
975
+ }
976
+ instance.connectedPeers = 0;
977
+ }
978
+
979
+ /**
980
+ * One {@link superviseRelayReservation} per relay dial addr, each over exactly
981
+ * that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
982
+ * one relay re-drives only that one. Default retry timings — the control node's (2 s
983
+ * doubling to 60 s between failed attempts, a 5 s liveness check while held). Each DRIVE's
984
+ * own deadline is counted in link round trips at this host's declared
985
+ * `network.linkRoundTripMs` (`link-budget.ts`), so the same declaration that lengthens the
986
+ * control node's drive lengthens these.
987
+ *
988
+ * Each supervisor's `beforeRedrive` is the caller's
989
+ * {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
990
+ * node's peer id, so a re-drive after a relay restart is preceded by a fresh
991
+ * delegate grant on the relay about to be dialed. Returns `[]` when no relay is
992
+ * configured — the strand then simply has no circuit listener to fill.
993
+ */
994
+ private startRelaySupervisors(
995
+ strandId: string,
996
+ node: Libp2pNodeWithRepo,
997
+ relayDialAddrs: readonly string[],
998
+ config: StartStrandConfig
999
+ ): RelayReservationSupervisor[] {
1000
+ if (relayDialAddrs.length === 0) {
1001
+ return [];
1002
+ }
1003
+ const announce = config.announceDelegateToRelay;
1004
+ const delegatePeerId = node.peerId.toString();
1005
+ const timeoutMs = relayReservationBudgetMs(config.network?.linkRoundTripMs);
1006
+ return relayDialAddrs.map((relayAddr) => superviseRelayReservation(node, [relayAddr], {
1007
+ timeoutMs,
1008
+ ...(announce && { beforeRedrive: () => announce(strandId, relayAddr, delegatePeerId) })
1009
+ }));
1010
+ }
1011
+
1012
+ /**
1013
+ * Wait for every relay supervisor's first attempt to settle, concurrently, and log
1014
+ * the ones that landed nothing. Never throws — a strand whose relay is unreachable
1015
+ * at launch still comes up, with its supervisors retrying in the background.
1016
+ */
1017
+ private async awaitFirstRelayAttempts(strandId: string): Promise<void> {
1018
+ const supervisors = this.relaySupervisors.get(strandId) ?? [];
1019
+ if (supervisors.length === 0) {
1020
+ return;
1021
+ }
1022
+ await Promise.all(supervisors.map((supervisor) => supervisor.firstAttempt));
1023
+ supervisors.forEach((supervisor) => {
1024
+ if (supervisor.lastError !== null) {
1025
+ log('Strand %s: relay reservation not held after the first attempt (retrying in the background): %s',
1026
+ strandId, supervisor.lastError);
1027
+ }
1028
+ });
1029
+ }
1030
+
1031
+ /**
1032
+ * Drop the strand's owned store and release this strand's claim on its cache. The
1033
+ * wrapper counts holders, so the cache is emptied and unregistered from the shared
1034
+ * pool only if no other scope still holds it. Called only where the strand's whole
1035
+ * lifetime ends (`stopStrand`, and the failed-launch rollback) — never on a quiesce.
1036
+ *
1037
+ * A dispose failure is logged, not thrown: the store is already unreferenced here,
1038
+ * and failing the stop over a cache-bookkeeping error would leave the caller unable
1039
+ * to tear the strand down. `disposeStorageCache` no-ops for an unwrapped store
1040
+ * (e.g. `MemoryRawStorage`), so no instanceof test is needed.
1041
+ */
1042
+ private async disposeStrandStorage(strandId: string): Promise<void> {
1043
+ const storage = this.strandStorages.get(strandId);
1044
+ if (!storage) {
1045
+ return;
1046
+ }
1047
+ this.strandStorages.delete(strandId);
1048
+ try {
1049
+ await disposeStorageCache(storage);
1050
+ } catch (error) {
1051
+ log('Failed to dispose storage cache for strand %s: %o', strandId, error);
1052
+ }
1053
+ }
1054
+
1055
+ /**
1056
+ * Refresh a strand's revoked-peer deny set NOW — and, with it, run the
1057
+ * teardown sweep that hangs up every connected revoked peer.
1058
+ *
1059
+ * The gate is otherwise poll-driven (default
1060
+ * `DEFAULT_REVOCATION_POLL_INTERVAL_MS`), because neither a replicated
1061
+ * revocation nor a local `revokeMember`/`leaveStrand` raises anything this
1062
+ * runtime can hook. A caller that just wrote a revocation against the
1063
+ * strand's database should follow it with this call, which makes the cut
1064
+ * immediate instead of waiting out the interval.
1065
+ *
1066
+ * Quiet no-op — matching `quiesceStrand`'s posture — when the strand is not
1067
+ * tracked, is quiesced, is open, or has the gate disabled: in every one of
1068
+ * those cases there is no enforcer to refresh. Never rejects (the enforcer's
1069
+ * refresh contains its own failures), and resolves only once the sweep has
1070
+ * finished.
1071
+ */
1072
+ async refreshRevocationEnforcement(strandId: string): Promise<void> {
1073
+ const enforcer = this.revocationEnforcers.get(strandId);
1074
+ if (!enforcer) {
1075
+ log('refreshRevocationEnforcement: strand %s has no armed revocation enforcer', strandId);
1076
+ return;
1077
+ }
1078
+ await enforcer.refresh();
1079
+ }
1080
+
1081
+ /**
1082
+ * A fresh membership invitation was staged for `strandId` (`CadreNode`'s
1083
+ * `adoptFormationMembershipInvite`): re-arm the strand's membership reconciler so the
1084
+ * invitation is attempted now rather than never — the loop finished during the first
1085
+ * join and would otherwise stay stopped until a relaunch. Quiet no-op when no reconciler
1086
+ * is armed: a first formation stages before the strand is added, and bring-up then arms
1087
+ * a loop that finds the invitation by itself. Returns at once; the pass runs on the
1088
+ * loop's own serialized chain and never rejects.
1089
+ */
1090
+ notifyMembershipInviteStaged(strandId: string): void {
1091
+ const reconciler = this.membershipReconcilers.get(strandId);
1092
+ if (!reconciler) {
1093
+ log('notifyMembershipInviteStaged: strand %s has no armed membership reconciler — bring-up will find the invitation', strandId);
1094
+ return;
1095
+ }
1096
+ void reconciler.rearm();
1097
+ }
1098
+
1099
+ /**
1100
+ * Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
1101
+ * self arm of `removeMemberPeer`, signed with the retained launch config's party
1102
+ * key. Called by `CadreNode.unpublishStrand` BEFORE the local stop, while the
1103
+ * strand database and transport are still live: after the unpublish commits, the
1104
+ * party's `StrandPartyKey` control row is gone, so no restart could ever sign
1105
+ * this removal again — the in-memory retained key is the last chance.
1106
+ *
1107
+ * Never throws. Quiet no-op for a strand that is untracked, open, or launched
1108
+ * without a party key; a quiesced (hibernating) strand, or a strand whose write
1109
+ * quorum is already unreachable, leaves the stale binding behind with a log —
1110
+ * it grants nothing today (admission is not allowlist-gated) and only
1111
+ * mis-credits diversity. The strand's own membership reconciler is stopped —
1112
+ * and its in-flight pass awaited — FIRST, so a pass that was already past its
1113
+ * own stopped check cannot re-register the binding this is about to clear.
1114
+ *
1115
+ * NOTE: deliberately NOT handled here (or anywhere yet): clearing the bindings
1116
+ * of a machine removed from the CADRE at the control layer, or of a party's
1117
+ * OTHER machines when the party unpublishes — the party still holds the key on
1118
+ * every surviving machine and can clear from any of them, and the stale rows
1119
+ * grant nothing while admission is deny-list only. Revisit when
1120
+ * `feat-strand-member-allowlist-admission` lands and a stale binding starts
1121
+ * granting admission rather than merely mis-crediting diversity.
1122
+ */
1123
+ async clearOwnMemberPeerBinding(strandId: string): Promise<void> {
1124
+ const instance = this.instances.get(strandId);
1125
+ const config = this.launchConfigs.get(strandId);
1126
+ if (!instance || !config || config.strandRow.Type !== 'c' || !config.partyMemberPrivateKey) {
1127
+ return;
1128
+ }
1129
+ const reconciler = this.membershipReconcilers.get(strandId);
1130
+ if (reconciler) {
1131
+ reconciler.stop();
1132
+ await reconciler.settle();
1133
+ }
1134
+ const db = instance.database?.getDatabase();
1135
+ const peerId = instance.libp2pNode?.peerId.toString();
1136
+ if (!db || !peerId) {
1137
+ log('clearOwnMemberPeerBinding: strand %s has no live runtime — own MemberPeer binding left in place', strandId);
1138
+ return;
1139
+ }
1140
+ try {
1141
+ // Background write on the app's database: never join a transaction the app has open.
1142
+ await removeMemberPeer(db, {
1143
+ memberKeyPair: strandMemberKeyPair(config.partyMemberPrivateKey),
1144
+ peerId
1145
+ }, { joinOpenTransaction: false });
1146
+ log('clearOwnMemberPeerBinding: strand %s — own binding for peer %s cleared', strandId, peerId);
1147
+ } catch (error) {
1148
+ log('clearOwnMemberPeerBinding: strand %s — best-effort clear failed (strand may already be unreachable): %o',
1149
+ strandId, error);
1150
+ }
1151
+ }
1152
+
1153
+ /**
1154
+ * Quiesce a strand: release its strand-network resources (stop the libp2p node,
1155
+ * close the StrandDatabase) while RETAINING the instance record — identity,
1156
+ * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
1157
+ * resumed later. Mechanically this is `stopStrand` minus the instance/config
1158
+ * deletion. The caller sets the post-quiesce status (e.g. `hibernating`).
1159
+ * No-ops when the strand is missing or already quiesced.
1160
+ */
1161
+ async quiesceStrand(strandId: string): Promise<void> {
1162
+ const instance = this.instances.get(strandId);
1163
+ if (!instance) {
1164
+ log('quiesceStrand: strand %s not found', strandId);
1165
+ return;
1166
+ }
1167
+ if (!instance.libp2pNode && !instance.database) {
1168
+ log('quiesceStrand: strand %s already quiesced', strandId);
1169
+ return;
1170
+ }
1171
+
1172
+ log('Quiescing strand instance: %s', strandId);
1173
+ await this.releaseRuntime(instance);
1174
+ log('Strand %s quiesced (resources released, instance retained)', strandId);
1175
+ }
1176
+
1177
+ /**
1178
+ * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
1179
+ * from the retained launch config and re-attach them, transitioning it back to
1180
+ * `active`. `overrides` re-applies volatile inputs that may have changed since
1181
+ * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
1182
+ * count) and updates the retained config so a later resume reuses the latest
1183
+ * values. Returns the live instance unchanged if it is already running.
1184
+ */
1185
+ async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
1186
+ if (this.stopping) {
1187
+ throw new Error('StrandInstanceManager is stopping');
1188
+ }
1189
+ const instance = this.instances.get(strandId);
1190
+ if (!instance) {
1191
+ throw new Error(`Cannot resume strand ${strandId}: not tracked`);
1192
+ }
1193
+ const launchConfig = this.launchConfigs.get(strandId);
1194
+ if (!launchConfig) {
1195
+ throw new Error(`Cannot resume strand ${strandId}: no retained launch config`);
1196
+ }
1197
+ if (instance.libp2pNode || instance.database) {
1198
+ log('resumeStrand: strand %s already live', strandId);
1199
+ return instance;
1200
+ }
1201
+
1202
+ log('Resuming strand instance: %s', strandId);
1203
+ const tTotal = performance.now();
1204
+
1205
+ // Re-apply volatile inputs and persist them so a subsequent resume reuses them.
1206
+ // Each `??` matters: a resume that passes no override must keep the retained value,
1207
+ // and a resume that passes one must leave it retained for the next resume — otherwise
1208
+ // a later no-override wake silently reverts to whatever launch time saw.
1209
+ const resumeConfig: StartStrandConfig = {
1210
+ ...launchConfig,
1211
+ bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes,
1212
+ // NOTE: `??` retains, so an override can raise or lower the count but cannot CLEAR it
1213
+ // back to "this node no longer knows" — the direction that would declare nothing. Moot
1214
+ // while nothing feeds `servingMachines` at all; if a source lands
1215
+ // (`backlog/feat-strand-yardstick-from-serving-machines`) that can legitimately lose the
1216
+ // count — a strand whose member rows became unreadable — this merge must gain an explicit
1217
+ // clear rather than silently declaring a stale number over a serving set it can no longer
1218
+ // see. Same applies to `bootstrapNodes` above, where a stale seed is harmless.
1219
+ servingMachines: overrides?.servingMachines ?? launchConfig.servingMachines
1220
+ };
1221
+ this.launchConfigs.set(strandId, resumeConfig);
1222
+
1223
+ instance.status = 'starting';
1224
+ try {
1225
+ await this.buildStrandRuntime(instance, resumeConfig);
1226
+ timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
1227
+ log('Strand %s resumed successfully', strandId);
1228
+ return instance;
1229
+ } catch (error) {
1230
+ instance.status = 'error';
1231
+ instance.error = error instanceof Error ? error.message : String(error);
1232
+ log('Failed to resume strand %s: %s', strandId, instance.error);
1233
+ throw error;
1234
+ }
1235
+ }
1236
+
1237
+ /**
1238
+ * Honor a founder request against an ALREADY-TRACKED strand — the seam that
1239
+ * closes the "whoever launches first decides whether the bootstrap runs" gap:
1240
+ * an instance first launched as a joiner (an app's own attach, or a watcher
1241
+ * poll winning the launch race) used to swallow a later founder request
1242
+ * silently, leaving the strand active with no `Strand.Header`.
1243
+ *
1244
+ * Flips the RETAINED launch config's `founder` to true, so every later
1245
+ * quiesce → resume rebuild founds as well (the bootstrap is insert-if-absent —
1246
+ * {@link StrandDatabase.ensureFounderBootstrap} — so re-running it per rebuild
1247
+ * writes nothing twice), and runs the bootstrap against the live database now.
1248
+ *
1249
+ * @returns how the request resolved:
1250
+ * - `'already-founder'` — the retained config already founds; nothing to do.
1251
+ * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1252
+ * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1253
+ * database), so the bootstrap could not run here: the CALLER must wake the
1254
+ * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1255
+ * manager does not) so the rebuild — which now founds — runs it.
1256
+ * @param resolvePartyKey - Asked for the party's own membership key when (and only
1257
+ * when) a CLOSED strand's retained config carries none — the instance was launched
1258
+ * as a joiner before its `StrandPartyKey` row existed or replicated. The resolved
1259
+ * key is retained alongside the founder flip so the bootstrap (now, or on the
1260
+ * caller's wake for `'needs-resume'`) can seat the founding Member/Manager.
1261
+ * @throws when the strand is not tracked — this seam exists only for the
1262
+ * tracked-instance launch path; an untracked id is a caller bug.
1263
+ * @throws whatever the live bootstrap throws (e.g. `PreSplitStrandIdentityError`),
1264
+ * after {@link withdrawFounderRequest} — so a retry re-attempts the founding rather
1265
+ * than resolving `'already-founder'` over an instance that never founded. A caller
1266
+ * whose `'needs-resume'` wake fails owes the same withdrawal.
1267
+ */
1268
+ async foundExistingStrand(
1269
+ strandId: string,
1270
+ resolvePartyKey?: () => Promise<string | undefined>
1271
+ ): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'> {
1272
+ const instance = this.instances.get(strandId);
1273
+ const config = this.launchConfigs.get(strandId);
1274
+ if (!instance || !config) {
1275
+ throw new Error(`Cannot found strand ${strandId}: not tracked`);
1276
+ }
1277
+ if (config.founder === true) {
1278
+ return 'already-founder';
1279
+ }
1280
+ const partyMemberPrivateKey =
1281
+ config.partyMemberPrivateKey
1282
+ ?? (config.strandRow.Type === 'c' ? await resolvePartyKey?.() : undefined);
1283
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1284
+ // config object, which is not ours to rewrite.
1285
+ this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1286
+ if (!instance.database && !this.firstSyncGates.has(strandId)) {
1287
+ return 'needs-resume';
1288
+ }
1289
+ try {
1290
+ await this.ensureFounderBootstrap(strandId);
1291
+ } catch (error) {
1292
+ this.withdrawFounderRequest(strandId);
1293
+ throw error;
1294
+ }
1295
+ return 'bootstrapped';
1296
+ }
1297
+
1298
+ /**
1299
+ * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1300
+ * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
1301
+ * the hibernation manager's idle timer fired on it). `false` for a writable, quiesced,
1302
+ * or untracked strand. The predicate {@link whenWritable}'s callers gate on.
1303
+ */
1304
+ isAwaitingFirstSync(strandId: string): boolean {
1305
+ const instance = this.instances.get(strandId);
1306
+ return instance !== undefined && isAwaitingFirstSync(instance);
1307
+ }
1308
+
1309
+ /**
1310
+ * Resolve once the strand's database is published to the app — immediately for a
1311
+ * writable strand — or reject with {@link StrandAwaitingFirstSyncError} after
1312
+ * `timeoutMs` (default: the retained launch config's `firstSync.timeoutMs`, else
1313
+ * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}). The rejection is RETRYABLE: nothing is
1314
+ * torn down, the gate keeps probing, and a later call waits afresh. A quiesced
1315
+ * (hibernating) strand is not woken here — the wait spans a resume, so a caller that
1316
+ * wants one wakes it (`CadreNode.wakeStrand`) and waits. Rejects immediately for an
1317
+ * untracked strand, and whenever the strand is stopped while a wait is pending.
1318
+ */
1319
+ whenWritable(strandId: string, options?: { timeoutMs?: number }): Promise<StrandInstance> {
1320
+ const instance = this.instances.get(strandId);
1321
+ if (!instance) {
1322
+ return Promise.reject(new Error(`Cannot wait for strand ${strandId} to become writable: not tracked`));
1323
+ }
1324
+ if (instance.database) {
1325
+ return Promise.resolve(instance);
1326
+ }
1327
+ const timeoutMs = options?.timeoutMs
1328
+ ?? this.launchConfigs.get(strandId)?.firstSync?.timeoutMs
1329
+ ?? DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS;
1330
+ const startedAt = Date.now();
1331
+ return new Promise<StrandInstance>((resolve, reject) => {
1332
+ const waiters = this.writableWaiters.get(strandId) ?? new Set<WritableWaiter>();
1333
+ this.writableWaiters.set(strandId, waiters);
1334
+ const settle = (): void => {
1335
+ clearTimeout(timer);
1336
+ waiters.delete(waiter);
1337
+ if (waiters.size === 0) {
1338
+ this.writableWaiters.delete(strandId);
1339
+ }
1340
+ };
1341
+ const waiter: WritableWaiter = {
1342
+ resolve: () => { settle(); resolve(instance); },
1343
+ reject: (error) => { settle(); reject(error); }
1344
+ };
1345
+ const timer = setTimeout(
1346
+ () => waiter.reject(new StrandAwaitingFirstSyncError(strandId, Date.now() - startedAt)),
1347
+ timeoutMs);
1348
+ (timer as { unref?: () => void }).unref?.();
1349
+ waiters.add(waiter);
1350
+ });
1351
+ }
1352
+
1353
+ /**
1354
+ * Hand a launch's database to the app: set `instance.database`, retire the gate that
1355
+ * held it, release every {@link whenWritable} waiter, and — when the launch had already
1356
+ * been reported gated — flip it `'active'` and announce it through the retained config's
1357
+ * `onWritable`. "Reported gated" is any status but `'starting'`: `'syncing'`, or the
1358
+ * `'idle'` the hibernation manager's idle timer flips a long-gated joiner to (see
1359
+ * {@link isAwaitingFirstSync}) — the Header arriving after that flip must announce too,
1360
+ * or an app hanging on `strand:writable` never learns the strand opened. A publish that
1361
+ * lands while the instance is still `'starting'` (the Header arrived during the rest of
1362
+ * bring-up) announces nothing: `startStrand` / `resumeStrand` are about to report the
1363
+ * strand `'active'`, and `strand:writable` is defined as the follow-up to a gated
1364
+ * launch, never a duplicate of `strand:started`.
1365
+ *
1366
+ * It is also the one seam where a WITHHELD database becomes available, so it kicks the
1367
+ * membership reconciler: on a gated launch that loop's own immediate pass ran while
1368
+ * `instance.database` was still unset, found no database, and would otherwise not try
1369
+ * again until its retry timer fired. A launch that was never gated publishes here BEFORE
1370
+ * the reconciler is constructed, so there is nothing registered to kick and the loop's own
1371
+ * immediate pass already sees the database — which is why the kick needs no `wasGated`
1372
+ * test of its own.
1373
+ */
1374
+ private publishDatabase(instance: StrandInstance, database: StrandDatabase): void {
1375
+ const strandId = instance.strandId;
1376
+ const gate = this.firstSyncGates.get(strandId);
1377
+ if (gate) {
1378
+ gate.open();
1379
+ this.firstSyncGates.delete(strandId);
1380
+ }
1381
+ instance.database = database;
1382
+ instance.lastActivity = new Date();
1383
+ const wasGated = instance.status !== 'starting';
1384
+ if (wasGated) {
1385
+ instance.status = 'active';
1386
+ log('Strand %s is now writable: Strand.Header held', strandId);
1387
+ }
1388
+ const waiters = this.writableWaiters.get(strandId);
1389
+ if (waiters) {
1390
+ this.writableWaiters.delete(strandId);
1391
+ waiters.forEach((waiter) => waiter.resolve());
1392
+ }
1393
+ if (wasGated) {
1394
+ this.launchConfigs.get(strandId)?.onWritable?.(strandId);
1395
+ }
1396
+ // Not awaited: publishing must stay synchronous, and the pass chains on the loop's own
1397
+ // tail, so it cannot overlap a pass already in flight.
1398
+ void this.membershipReconcilers.get(strandId)?.reconcile();
1399
+ }
1400
+
1401
+ /** Reject every {@link whenWritable} waiter for a strand that is going away. */
1402
+ private rejectWritableWaiters(strandId: string, reason: string): void {
1403
+ const waiters = this.writableWaiters.get(strandId);
1404
+ if (!waiters) {
1405
+ return;
1406
+ }
1407
+ this.writableWaiters.delete(strandId);
1408
+ waiters.forEach((waiter) => waiter.reject(new Error(`Strand ${strandId} ${reason} before becoming writable`)));
1409
+ }
1410
+
1411
+ /**
1412
+ * Undo {@link foundExistingStrand}'s founder flip after the founding it promised
1413
+ * failed, so the retained config again says what this instance actually runs as (a
1414
+ * joiner) and the next founder request re-attempts the bootstrap. The resolved party
1415
+ * key stays retained — it is this party's identity for the strand either way. No-op
1416
+ * when the strand is untracked or its config does not found.
1417
+ */
1418
+ withdrawFounderRequest(strandId: string): void {
1419
+ const config = this.launchConfigs.get(strandId);
1420
+ if (config?.founder === true) {
1421
+ this.launchConfigs.set(strandId, { ...config, founder: false });
1422
+ }
1423
+ }
1424
+
1425
+ /**
1426
+ * Run the (idempotent) founder bootstrap against a tracked strand's LIVE
1427
+ * database, independently of what the retained launch config says.
1428
+ *
1429
+ * Separate from {@link foundExistingStrand} because a caller that resolved
1430
+ * `'needs-resume'` and woke the strand must not assume the wake's own rebuild
1431
+ * founded it: `HibernationManager` COALESCES wakes, so a wake already in flight
1432
+ * when the config flipped had already read the PRE-flip config and rebuilt as a
1433
+ * joiner. Re-running the bootstrap costs one insert-if-absent probe per table
1434
+ * and is the only thing that makes "founding resolves once the Header is
1435
+ * written" true on that path.
1436
+ *
1437
+ * A strand still `'syncing'` (its database withheld behind the first-sync gate) is
1438
+ * founded through the gate: the bootstrap writes the Header this machine was waiting
1439
+ * to receive, so a successful run publishes the database and the strand goes
1440
+ * `'active'` — the one legitimate way a joiner launch becomes writable without a peer.
1441
+ *
1442
+ * @throws when the strand is not tracked, or is still quiesced (no live
1443
+ * database) — both mean the bootstrap did NOT run, which a founder request
1444
+ * must never swallow.
1445
+ */
1446
+ async ensureFounderBootstrap(strandId: string): Promise<void> {
1447
+ const instance = this.instances.get(strandId);
1448
+ if (!instance) {
1449
+ throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: not tracked`);
1450
+ }
1451
+ const gate = this.firstSyncGates.get(strandId);
1452
+ const database = instance.database ?? gate?.database;
1453
+ if (!database) {
1454
+ throw new Error(
1455
+ `Cannot run the founder bootstrap for strand ${strandId}: it is quiesced, so there ` +
1456
+ 'is no live database to write to — resume it first.'
1457
+ );
1458
+ }
1459
+ // Forward the RETAINED config's party key: a foundExistingStrand that flipped a
1460
+ // joiner launch to founder may have resolved a key the live database's captured
1461
+ // config (built at the original launch) never saw.
1462
+ await database.ensureFounderBootstrap(this.launchConfigs.get(strandId)?.partyMemberPrivateKey);
1463
+ if (!instance.database) {
1464
+ this.publishDatabase(instance, database);
1465
+ }
1466
+ }
1467
+
1468
+ /**
1469
+ * Stop a strand instance
1470
+ */
1471
+ async stopStrand(strandId: string): Promise<void> {
1472
+ const instance = this.instances.get(strandId);
1473
+ if (!instance) {
1474
+ log('Strand %s not found', strandId);
1475
+ return;
1476
+ }
1477
+
1478
+ log('Stopping strand instance: %s', strandId);
1479
+ instance.status = 'stopping';
1480
+ this.rejectWritableWaiters(strandId, 'was stopped');
1481
+
1482
+ try {
1483
+ await this.releaseRuntime(instance);
1484
+ instance.status = 'stopped';
1485
+ this.instances.delete(strandId);
1486
+ this.launchConfigs.delete(strandId);
1487
+ // Storage is released only here — NOT in releaseRuntime, which a quiesce shares.
1488
+ await this.disposeStrandStorage(strandId);
1489
+ log('Strand %s stopped successfully', strandId);
1490
+ } catch (error) {
1491
+ instance.status = 'error';
1492
+ instance.error = error instanceof Error ? error.message : String(error);
1493
+ log('Error stopping strand %s: %s', strandId, instance.error);
1494
+ throw error;
1495
+ }
1496
+ }
1497
+
1498
+ /**
1499
+ * Stop all strand instances
1500
+ */
1501
+ async stopAll(): Promise<void> {
1502
+ this.stopping = true;
1503
+ log('Stopping all strand instances (%d)', this.instances.size);
1504
+
1505
+ const stopPromises = Array.from(this.instances.keys()).map(id =>
1506
+ this.stopStrand(id).catch(err => {
1507
+ log('Error stopping strand %s during shutdown: %s', id, err);
1508
+ })
1509
+ );
1510
+
1511
+ await Promise.all(stopPromises);
1512
+ this.stopping = false;
1513
+ log('All strand instances stopped');
1514
+ }
1515
+ }
1516
+