@serfab/cadre-core 1.12.0 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1804 +1,1883 @@
1
- import debug from 'debug';
2
- import type { PrivateKey } from '@libp2p/interface';
3
- import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
4
- import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
5
- import { StrandDatabase } from './strand-database.js';
6
- import { PeerJoinBackfill, type PeerJoinBackfillConfig } from './peer-join-backfill.js';
7
- import { strandNetworkStatePersistence, type StrandNetworkStateStore } from './strand-network-state.js';
8
- import {
9
- StrandRevocationEnforcer,
10
- createRevocationConnectionGater,
11
- readStrandRevocationRows,
12
- type StrandRevocationEnforcementConfig
13
- } from './strand-revocation-enforcer.js';
14
- import {
15
- StrandMembershipReconciler,
16
- type PendingMembershipInviteSource,
17
- type StrandMembershipReconciliationConfig
18
- } from './strand-membership-reconciler.js';
19
- import { removeMemberPeer } from './strand-membership-writer.js';
20
- import { strandMemberKeyPair } from './strand-member-key.js';
21
- import { assertSchemaSignature } from './schema-verification.js';
22
- import { assertStrandScopeKey } from './storage-scope.js';
23
- import {
24
- StrandFirstSyncGate,
25
- StrandAwaitingFirstSyncError,
26
- strandFirstSyncComplete,
27
- DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS,
28
- type StrandFirstSyncConfig
29
- } from './strand-first-sync-gate.js';
30
- import type {
31
- StrandInstance,
32
- StrandRow,
33
- StorageConfig,
34
- NetworkConfig,
35
- LatencyHint,
36
- NodeProfile,
37
- SAppConfig,
38
- SAppInfo,
39
- RawStorageProvider,
40
- Libp2pNodeWithRepo
41
- } from './types.js';
42
- import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
43
- import { strandNodeAddrs } from './strand-network-config.js';
44
- import { resolveRelayServer } from './relay-server.js';
45
- import { strandCohortTopicOption, type StrandReactivityConfig } from './strand-reactivity.js';
46
- import { superviseRelayReservation, type RelayReservationSupervisor } from './relay-reservation.js';
47
- import { declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
48
-
49
- const log = debug('sereus:cadre:strand-manager');
50
- const timing = debug('sereus:cadre:timing');
51
-
52
- /** One caller blocked in {@link StrandInstanceManager.whenWritable}. */
53
- interface WritableWaiter {
54
- resolve: () => void;
55
- reject: (error: Error) => void;
56
- }
57
-
58
- /**
59
- * Whether a tracked instance is launched but waiting for its first sync: the runtime is
60
- * up (`libp2pNode` set) while the database is still withheld (`database` unset). That
61
- * is the first-sync gate's state regardless of what `status` says — the hibernation
62
- * manager's idle timer can flip a long-gated joiner to `'idle'` without changing it.
63
- *
64
- * NOTE: `status` is therefore not the source of truth for "gated": a joiner that reaches
65
- * nobody for `idleTimeout` (5 min at the interactive hint) reads `'idle'`, then
66
- * `'hibernating'`, while still holding no database. Harmless today — every decision the
67
- * runtime makes goes through this predicate, {@link StrandInstanceManager.publishDatabase}
68
- * announces on the idle-flipped instance exactly as on a `'syncing'` one, and an app that
69
- * shows "waiting for the other member" on `'syncing'` merely sees "idle" instead after
70
- * five minutes. If an app ever needs the gated LABEL to survive the idle flip, make the
71
- * hibernation manager's idle transition preserve `'syncing'` rather than adding a flag.
72
- */
73
- export function isAwaitingFirstSync(instance: StrandInstance): boolean {
74
- return instance.libp2pNode !== undefined && instance.database === undefined;
75
- }
76
-
77
- /**
78
- * The status a LIVE instance (runtime rebuilt or woken) should report: `'active'` once its
79
- * database is published, `'syncing'` while the first-sync gate still withholds it. Every
80
- * site that would otherwise write `'active'` after a build or a wake goes through this,
81
- * so a gated joiner never reads as writable.
82
- */
83
- export function liveStrandStatus(instance: StrandInstance): 'active' | 'syncing' {
84
- return instance.database ? 'active' : 'syncing';
85
- }
86
-
87
- /**
88
- * Configuration for starting a strand instance
89
- */
90
- export interface StartStrandConfig {
91
- strandRow: StrandRow;
92
- /**
93
- * sApp configuration provided by the hosting application. Absent for a **storage
94
- * replica** — a strand this node stores and serves without the app installed
95
- * (`CadreNodeConfig.hostUnclaimedStrands`): no schema signature check, no `App`
96
- * schema, no `sAppInfo` on the instance, and the default latency hint. A replica is
97
- * always a joiner; `StrandDatabase` refuses to found one. An app that later claims the
98
- * strand upgrades the running replica in place ({@link StrandInstanceManager.attachSApp}).
99
- */
100
- sAppConfig?: SAppConfig;
101
- storage?: StorageConfig;
102
- network?: NetworkConfig;
103
- profile: NodeProfile;
104
- defaultLatencyHint: LatencyHint;
105
- privateKey?: PrivateKey;
106
- /** Cohort-derived discovery seed (multiaddr strings). Defaults to [] when omitted. */
107
- bootstrapNodes?: string[];
108
- /**
109
- * Require a valid author signature on the sApp schema before bring-up.
110
- * Defaults to true (fail closed) when omitted; set false only for dev/test
111
- * with unsigned demo schemas. Mirrors {@link CadreNodeConfig.requireSignedSchemas}.
112
- */
113
- requireSignedSchemas?: boolean;
114
- /**
115
- * Whether this node founds the strand (vs. joins it). Forwarded to the
116
- * StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
117
- * runs once at bring-up. Joiners leave this unset and write nothing. Callers
118
- * resolve it BEFORE calling `startStrand` (`CadreNode.launchStrand` derives it
119
- * from the row's `FounderOwnerKey` when no explicit flag is given — see
120
- * {@link StrandConfig.founder}); a founder request that arrives while the
121
- * instance is already tracked goes through {@link StrandInstanceManager.foundExistingStrand},
122
- * never through a repeat `startStrand`.
123
- */
124
- founder?: boolean;
125
- /**
126
- * THIS party's own strand membership private key (base64 protobuf) for a closed
127
- * strand — the identity the founder bootstrap derives `Member.Key` /
128
- * `Manager.MemberKey` from. Resolved by the caller (`CadreNode.launchStrand`: an
129
- * explicit attach-time key, else the control-layer `StrandPartyKey` row, healed by
130
- * mint-on-launch for the founding machine) and retained with the launch config, so a
131
- * hibernation wake rebuilds under the same identity. Deliberately NOT the strand
132
- * row's shared `MemberPrivateKey` (see `StrandDatabaseConfig.partyMemberPrivateKey`).
133
- * Absent for open strands, and for joiners with no persisted party key.
134
- */
135
- partyMemberPrivateKey?: string;
136
- /**
137
- * Number of nodes Optimystic is told this strand's replication cluster should
138
- * have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
139
- * forwards here: every node on the strand should use the same value, and it is
140
- * frozen when the strand's libp2p node is created. Defaults to
141
- * `DEFAULT_STRAND_CLUSTER_SIZE` (4); values below `MIN_CLUSTER_SIZE` (2) are
142
- * rejected by `resolveStrandClusterSize`. Strand-only — the
143
- * control network's breadth is the fixed `CONTROL_REPLICATION_BREADTH`.
144
- */
145
- clusterSize?: number;
146
- /**
147
- * Machines that SERVE THIS STRAND (this node included), for the strand node's
148
- * block-repair corroboration yardstick — see `resolveRepairYardstick`. It must be
149
- * an authenticated per-strand count: the machines that actually run a node for this
150
- * strand, not the machines that exist.
151
- *
152
- * **The party's enrolled-machine count is NOT this number and must never be passed
153
- * here.** A strand launches only on machines whose embedding app registered its sApp
154
- * config (`CadreNode.addStrand`), so a closed strand shared by two machines of a
155
- * three-machine party is served by two. Passing three over-declares, and
156
- * over-declaring is the unsafe direction: at a declaration of 3 or more Optimystic
157
- * pins the repair corroboration floor at two corroborating peers, which a cohort
158
- * that can only ever field one peer can never reach — so the strand can never repair
159
- * a block (`cluster-fetch:no-quorum`, surfacing as reads failing with `Missing
160
- * block`). That regression is why this field was renamed off "enrolledMachines":
161
- * `bug-strand-yardstick-counts-party-machines`.
162
- *
163
- * **Nothing feeds it in production today.** No authenticated per-strand serving count
164
- * exists yet, so `CadreNode` passes nothing and the strand node runs the frozen
165
- * `STRAND_CLUSTER_POLICY` — declaring no yardstick, which leaves the known,
166
- * upstream-tracked single-voter exposure
167
- * (`backlog/debt-read-repair-single-voter-corroboration`). Building the count is
168
- * `backlog/feat-strand-yardstick-from-serving-machines`; this field and its
169
- * threading are the seam it plugs into.
170
- *
171
- * Volatile when a source does exist: re-resolve it on every resume beside the cohort
172
- * seed, since machines join and leave a strand while it hibernates. Omitting it means
173
- * "this node does not know", which declares nothing.
174
- */
175
- servingMachines?: number;
176
- /**
177
- * Tuning for the strand peer-join block catch-up ({@link PeerJoinBackfill}),
178
- * forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
179
- * libp2p node connects to a peer this runtime has not yet caught up, every
180
- * block in the strand's own raw store is pushed to it. Runs only on strands
181
- * with per-strand storage; `{ enabled: false }` restores the pre-existing
182
- * no-backfill behaviour.
183
- */
184
- backfill?: PeerJoinBackfillConfig;
185
- /**
186
- * Whether this strand's libp2p node is built with Optimystic's change notifications,
187
- * forwarded from {@link CadreNodeConfig.strandReactivity} and resolved per strand by
188
- * `strandCohortTopicOption`. Retained with the launch config, so a hibernation wake
189
- * rebuilds the node with the same setting.
190
- */
191
- reactivity?: StrandReactivityConfig;
192
- /**
193
- * Tuning for the CLOSED-strand revoked-peer gate
194
- * ({@link StrandRevocationEnforcer}), forwarded from
195
- * {@link CadreNodeConfig.strandRevocationEnforcement}. Armed only for
196
- * `strandRow.Type === 'c'` (an open strand has no membership rows to derive a
197
- * deny set from); `{ enabled: false }` restores the pre-existing behaviour
198
- * (a removed party's peers keep being served).
199
- */
200
- revocationEnforcement?: StrandRevocationEnforcementConfig;
201
-
202
- /**
203
- * The staged formation membership invitation for THIS strand — the seam the
204
- * bring-up membership reconciler (`strand-membership-reconciler.ts`) reads to
205
- * redeem the party's `Strand.Member` seat, and clears once the invitation is
206
- * spent, burned, or dead. `CadreNode.launchStrand` wires it over its in-memory
207
- * `pendingMembershipInvites` map; read lazily per pass, so a re-formation that
208
- * replaces the entry between passes is picked up. Only consulted for closed
209
- * strands launched with a {@link partyMemberPrivateKey}; harmless otherwise.
210
- */
211
- pendingMembershipInvite?: PendingMembershipInviteSource;
212
-
213
- /**
214
- * Tuning for the CLOSED-strand membership reconciler, forwarded from
215
- * {@link CadreNodeConfig.strandMembershipReconciliation}. Armed only for
216
- * closed strands launched with a {@link partyMemberPrivateKey};
217
- * `{ enabled: false }` disarms the loop (test fixtures that hand-drive the
218
- * membership writers). When it names no `pollIntervalMs` the reconciler
219
- * mirrors {@link revocationEnforcement}'s cadence — as its IDLE cadence and
220
- * the cap of its unfinished-join retry ladder, not as a flat retry interval.
221
- */
222
- membershipReconciliation?: StrandMembershipReconciliationConfig;
223
-
224
- /**
225
- * Called when THIS node's own peer id turns up in the strand's revoked set —
226
- * this node's party was removed from the closed strand, or left it. Fires once
227
- * per removal (re-armed if the party is re-admitted; a resume also rebuilds the
228
- * enforcer and may legitimately re-fire), and only for closed strands with the
229
- * gate armed. `CadreNode`
230
- * wires it to its `strand:revoked` event; nothing is stopped or torn down on
231
- * this node's behalf — what to do about it is the app's call.
232
- */
233
- onSelfRevoked?: (strandId: string) => void;
234
-
235
- /**
236
- * Called when the membership reconciler reports a blocked re-join — this node holds a
237
- * staged membership invitation for the strand that its writes cannot redeem, which is
238
- * what a REMOVED party handed a fresh invitation hits (see "Reporting a blocked
239
- * re-join" in `strand-membership-reconciler.ts` for the two triggers, one of them a
240
- * suspicion rather than a verdict). At most once per re-arm of the loop — a further
241
- * invitation staged while the loop is still running does not reset the report — and
242
- * only for closed strands with a party key. `CadreNode` wires it to its `strand:rejoin-blocked`
243
- * event; nothing is stopped or torn down — a remaining manager admitting this party's
244
- * key directly is the remedy, and the loop finishes the join by itself once that lands.
245
- */
246
- onRejoinBlocked?: (strandId: string) => void;
247
-
248
- /**
249
- * Tuning for the JOINING machine's first-sync write gate (`strand-first-sync-gate.ts`),
250
- * forwarded from {@link CadreNodeConfig.strandFirstSync}: the Header probe cadence while
251
- * a non-founder launch is `'syncing'`, and the default budget {@link whenWritable} waits
252
- * before rejecting. Founders and machines that already hold the Header are never gated.
253
- */
254
- firstSync?: StrandFirstSyncConfig;
255
-
256
- /**
257
- * Called when a launch that came up `'syncing'` (see {@link firstSync}) becomes
258
- * writable: the strand's `Strand.Header` arrived from a peer — or a founder request
259
- * ran the bootstrap against the gated database — and `instance.database` is now set.
260
- * Never called for a launch that was writable when `startStrand`/`resumeStrand`
261
- * resolved. `CadreNode` wires it to its `strand:writable` event. Retained with the
262
- * launch config, so a hibernation resume that comes up gated announces too.
263
- */
264
- onWritable?: (strandId: string) => void;
265
-
266
- /**
267
- * Re-announce this strand's delegate peer id (`delegatePeerId` — the strand
268
- * node's own transport peer id) to ONE relay (`relayAddr`, a direct dial addr
269
- * with a trailing `/p2p/<relayPeerId>`), unthrottled. Awaited before every
270
- * relay-reservation RE-drive of that relay — never before the first attempt,
271
- * which the launch/resume seed pass has already announced for.
272
- *
273
- * Why: a party control node running the relay server admits the strand node on
274
- * an in-memory delegate grant that a relay restart drops, so a re-drive that did
275
- * not re-announce first would be denied at the relay's gate. `CadreNode` wires
276
- * its own control-mesh announce here; a throw or rejection is logged by the
277
- * supervisor and the re-drive still runs. Retained with the launch config, so a
278
- * hibernation wake's rebuilt supervisors carry it too.
279
- */
280
- announceDelegateToRelay?: (strandId: string, relayAddr: string, delegatePeerId: string) => Promise<void>;
281
-
282
- /**
283
- * The node-local strand network state (`strand-network-state.ts`): this strand's
284
- * slice of it is handed to db-p2p as `persistence`, so the strand node saves its
285
- * FRET routing table (with each peer's signed address record) on every connection
286
- * and re-imports it when it is next built. Absent ⇒ nothing is saved and the node
287
- * starts with an empty table. `CadreNode` passes its store. Retained with the launch
288
- * config, so a hibernation wake rebuilds the node over the same state.
289
- */
290
- networkState?: StrandNetworkStateStore;
291
- }
292
-
293
- /**
294
- * Volatile inputs re-resolved when resuming a quiesced strand — the ones that can
295
- * have moved since it last ran, and that the rebuilt libp2p node freezes again. Each
296
- * one omitted keeps the value the retained launch config already holds, so a resume
297
- * that passes nothing rebuilds the strand exactly as it last ran.
298
- */
299
- export interface ResumeStrandOverrides {
300
- /**
301
- * Freshly-resolved cohort discovery seed (multiaddr strings). Grows as peers are
302
- * learned since the strand first launched.
303
- */
304
- bootstrapNodes?: string[];
305
- /**
306
- * Freshly-read count of the machines serving this strand, for the repair yardstick
307
- * (see {@link StartStrandConfig.servingMachines} — including why the party's
308
- * enrolled-machine count is not it). Moves whenever a machine starts or stops
309
- * serving the strand, which a hibernating strand does not otherwise notice. Nothing
310
- * passes it today. Omitting it here does NOT mean "unknown" the way omitting it from
311
- * {@link StartStrandConfig} does — it retains the last value; see `resumeStrand`.
312
- */
313
- servingMachines?: number;
314
- }
315
-
316
- /** A {@link StrandInstanceManager.resumeStrand} or {@link StrandInstanceManager.quiesceStrand} of one strand. */
317
- type RuntimeTransition =
318
- | { kind: 'resume'; operation: Promise<StrandInstance> }
319
- | { kind: 'quiesce'; operation: Promise<void> };
320
-
321
- /**
322
- * Get the isolated storage path for a specific strand.
323
- *
324
- * @deprecated This helper is Node-only and throws in React Native (it assumes a
325
- * filesystem layout). Use a storage provider factory function instead, which
326
- * receives the strandId and can create strand-specific storage paths using
327
- * platform-appropriate methods.
328
- *
329
- * @example
330
- * // Instead of using getStrandStoragePath, use a storage provider factory:
331
- * const storage = {
332
- * provider: (strandId: string) => new FileRawStorage(`./data/strands/${strandId}`)
333
- * };
334
- */
335
- export function getStrandStoragePath(basePath: string, strandId: string): string {
336
- // Check if we're in a Node.js environment
337
- if (typeof process === 'undefined' || !process.versions?.node) {
338
- throw new Error(
339
- 'getStrandStoragePath is not available in React Native. ' +
340
- 'Use a storage provider factory function instead.'
341
- );
342
- }
343
-
344
- // Sanitize strandId for filesystem safety (UUIDs should be safe, but just in case)
345
- const safeId = strandId.replace(/[^a-zA-Z0-9-]/g, '_');
346
-
347
- // Build the path with plain string joins rather than the Node `path` module.
348
- // A static `require('path')` forces RN bundlers (e.g. Metro) to *resolve* the
349
- // module at bundle time even though this Node-only helper throws above before
350
- // ever reaching here — joining by hand keeps the module free of any Node
351
- // built-in reference, so RN bundles need no `path` shim.
352
- const trimmedBase = basePath.replace(/[\\/]+$/, '');
353
- return `${trimmedBase}/strands/${safeId}`;
354
- }
355
-
356
- /**
357
- * Resolve a storage provider for a specific strand.
358
- * If the provider is a factory function, call it with the strandId.
359
- *
360
- * Called only from {@link StrandInstanceManager.startStrand}, which owns the result
361
- * for the instance's lifetime — see `strandStorages`, and which has already asserted
362
- * the id is usable as a scope key. Do not move that assertion here: this function
363
- * returns early when no provider is configured, and the id becomes a libp2p protocol
364
- * prefix on that path too.
365
- *
366
- * @param provider - Storage provider (instance or factory)
367
- * @param strandId - The strand ID to create storage for
368
- * @returns The resolved IRawStorage instance, or undefined if no provider
369
- */
370
- function resolveStrandStorage(
371
- provider: RawStorageProvider | undefined,
372
- strandId: string
373
- ): IRawStorage | undefined {
374
- if (!provider) {
375
- return undefined;
376
- }
377
- const storage = typeof provider === 'function' ? provider(strandId) : provider;
378
- // Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's cached-storage.ts).
379
- // Called ONCE per strand launch — `startStrand` keeps the result for the instance's
380
- // lifetime — so the wrap survives every runtime rebuild the instance goes through.
381
- return wrapStorageWithCache(storage, strandId);
382
- }
383
-
384
- /**
385
- * Verify an sApp's schema signature (fail-closed unless `requireSignedSchemas` is
386
- * false) and project the config onto the instance's `sAppInfo`.
387
- */
388
- function verifiedSAppInfo(strandId: string, sAppConfig: SAppConfig, requireSignedSchemas: boolean | undefined): SAppInfo {
389
- assertSchemaSignature(sAppConfig, { requireSignature: requireSignedSchemas ?? true });
390
- log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
391
- return {
392
- id: sAppConfig.id,
393
- version: sAppConfig.version,
394
- schema: sAppConfig.schema,
395
- signature: sAppConfig.signature
396
- };
397
- }
398
-
399
- /** Log wording for what a launch runs: the sApp and its version, or a storage replica. */
400
- function describeSApp(sAppConfig: SAppConfig | undefined): string {
401
- return sAppConfig ? `sApp: ${sAppConfig.id} v${sAppConfig.version}` : 'storage replica';
402
- }
403
-
404
- /**
405
- * Manages individual strand instances - creates and destroys isolated libp2p nodes
406
- * for each strand the cadre participates in.
407
- */
408
- export class StrandInstanceManager {
409
- private instances: Map<string, StrandInstance> = new Map();
410
- /**
411
- * Retained launch config per strand, captured in `startStrand` and cleared in
412
- * `stopStrand`. `resumeStrand` reuses it to rebuild a quiesced strand's runtime
413
- * without the caller re-threading storage/network/profile/key/sApp config.
414
- */
415
- private launchConfigs: Map<string, StartStrandConfig> = new Map();
416
- /**
417
- * The per-strand peer-join block catch-up, keyed by strand id. Private (not on
418
- * the public {@link StrandInstance}) because it is runtime plumbing with the
419
- * same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
420
- * stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
421
- * with a fresh caught-up-peer memo, which is intended (a resumed node may have
422
- * missed writes).
423
- */
424
- private backfills: Map<string, PeerJoinBackfill> = new Map();
425
- /**
426
- * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
427
- * id. Same lifecycle rationale as {@link backfills}: created in
428
- * `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so quiesce →
429
- * resume rebuilds it with a fresh (initially empty, fail-open) snapshot.
430
- */
431
- private revocationEnforcers: Map<string, StrandRevocationEnforcer> = new Map();
432
- /**
433
- * The per-strand membership reconciler (closed strands launched with a party
434
- * key only), keyed by strand id. Same lifecycle rationale as {@link backfills}:
435
- * created in `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so
436
- * quiesce → resume rebuilds it and re-runs the idempotent join ladder (redeem a
437
- * staged invitation, write this machine's own `MemberPeer` binding) from scratch.
438
- */
439
- private membershipReconcilers: Map<string, StrandMembershipReconciler> = new Map();
440
- /**
441
- * The per-strand first-sync gate (`strand-first-sync-gate.ts`), keyed by strand id:
442
- * holds a launch's `StrandDatabase` while it is NOT yet published on the instance.
443
- * Present from the moment the database object exists in `buildStrandRuntime` until
444
- * {@link publishDatabase} hands it to the app — so it doubles as the rollback handle
445
- * for a failed `initialize()` — and, for a joiner that holds no `Strand.Header` yet,
446
- * for as long as the launch stays `'syncing'`. Same lifecycle as {@link backfills}:
447
- * stopped (its database closed) and dropped in `releaseRuntime`, so a quiesce →
448
- * resume rebuild re-probes over the same store. An entry and `instance.database`
449
- * are mutually exclusive.
450
- */
451
- private firstSyncGates: Map<string, StrandFirstSyncGate> = new Map();
452
- /**
453
- * Callers blocked in {@link whenWritable}, keyed by strand id. Resolved by
454
- * {@link publishDatabase}, rejected by {@link stopStrand}; each waiter also carries
455
- * its own timeout. Deliberately NOT cleared by `releaseRuntime`: a waiter outlives a
456
- * quiesce → resume cycle, since the rebuild is what may finally publish the database.
457
- */
458
- private writableWaiters: Map<string, Set<WritableWaiter>> = new Map();
459
- /**
460
- * The per-strand relay-reservation supervisors — ONE PER CONFIGURED RELAY, each
461
- * over the node's own bare `/p2p-circuit` listener for that relay
462
- * (`strand-network-config.ts`) — keyed by strand id. Same lifecycle rationale as
463
- * {@link backfills}: started in `buildStrandRuntime` right after the libp2p node
464
- * exists, stopped FIRST and dropped in `releaseRuntime` (before the database
465
- * closes and the node stops, so no re-drive dials a node being torn down), so
466
- * quiesce → resume rebuilds them over the new node. What makes a strand node's
467
- * lost relay slot come back on its own — the control node has the same loop in
468
- * `CadreNode.reserveRelays`.
469
- */
470
- private relaySupervisors: Map<string, RelayReservationSupervisor[]> = new Map();
471
- /**
472
- * The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
473
- * store, resolved once in `startStrand` and held until `stopStrand` disposes it.
474
- * Private for the same reason `backfills` is: runtime plumbing, not part of the
475
- * public {@link StrandInstance}.
476
- *
477
- * Deliberately NOT touched by `releaseRuntime`: that is what lets a quiesce →
478
- * resume cycle rebuild the libp2p node over the SAME store, keeping its
479
- * write-through cache warm (and, on an in-memory backend, keeping the strand's
480
- * blocks at all). An entry exists iff `instances` does AND the launch config
481
- * supplied a storage provider.
482
- *
483
- * NOTE: a hibernating strand therefore keeps its store — and its share of the
484
- * process-wide cache pool — resident for as long as the instance is tracked. That
485
- * is the point (a warm wake), and the pool evicts under pressure, so the cost is
486
- * one map entry per hibernating strand today. If a device ever hibernates strands
487
- * by the hundred, revisit: dropping the store at quiesce and paying for a cold
488
- * wake becomes the better trade.
489
- */
490
- private strandStorages: Map<string, IRawStorage> = new Map();
491
- /**
492
- * The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
493
- * `resumeStrand`), present until it settles. The instance is tracked for the whole build,
494
- * with its database not yet constructed or not yet initialized, so a caller that must act
495
- * on the FINISHED runtime ({@link attachSApp}, {@link whenRuntimeBuilt}) waits on this instead. A failed build's
496
- * rejection belongs to the call that started it.
497
- */
498
- private runtimeBuilds: Map<string, Promise<void>> = new Map();
499
- /**
500
- * The latest resume or quiesce requested per strand id, present until it settles. Each one
501
- * starts only once the one requested before it has settled, so a wake that lands during a
502
- * quiesce rebuilds after it, and a quiesce requested during a rebuild releases what that
503
- * rebuild finished. The WHOLE operation, not its entry in {@link runtimeBuilds}: that settles
504
- * before a resume's own catch records `'error'`, so a joiner awaiting it could read a stale
505
- * status.
506
- */
507
- private transitions: Map<string, RuntimeTransition> = new Map();
508
- private stopping = false;
509
-
510
- constructor() {
511
- log('StrandInstanceManager created');
512
- }
513
-
514
- /**
515
- * Get all current strand instances
516
- */
517
- getInstances(): Map<string, StrandInstance> {
518
- return new Map(this.instances);
519
- }
520
-
521
- /**
522
- * Get a specific strand instance
523
- */
524
- getInstance(strandId: string): StrandInstance | undefined {
525
- return this.instances.get(strandId);
526
- }
527
-
528
- /**
529
- * Check if a strand is currently running
530
- */
531
- hasStrand(strandId: string): boolean {
532
- return this.instances.has(strandId);
533
- }
534
-
535
- /**
536
- * Start a new strand instance.
537
- *
538
- * A failed launch leaves NOTHING tracked: the instance and its retained launch
539
- * config are both dropped before the error is rethrown, so the strand id is
540
- * free for a genuine retry. This matches the pre-registration failure path
541
- * (a rejected schema signature, which throws before anything is recorded) —
542
- * both failure modes of this call leave the same residue: none. Callers learn
543
- * of the failure from the rejected promise (and, on the control-discovered
544
- * path, from CadreNode's `strand:error` event), not from an error record left
545
- * behind in `instances`.
546
- *
547
- * The strand's raw storage is resolved HERE, once, and held for the instance's
548
- * lifetime (see `strandStorages`) — `buildStrandRuntime` only reads it, so a
549
- * hibernation wake never re-enters the embedder's provider.
550
- */
551
- async startStrand(config: StartStrandConfig): Promise<StrandInstance> {
552
- const { strandRow, sAppConfig } = config;
553
- const strandId = strandRow.Id;
554
-
555
- if (this.stopping) {
556
- throw new Error('StrandInstanceManager is stopping');
557
- }
558
-
559
- if (this.instances.has(strandId)) {
560
- // Callers resolve founder-ness BEFORE reaching here (CadreNode.launchStrand) and
561
- // honor a founder request on a tracked instance via foundExistingStrand — so a
562
- // founder flag arriving at this early return against a non-founder retained
563
- // config is a dropped bootstrap, and must never again be silent.
564
- if (config.founder === true && this.launchConfigs.get(strandId)?.founder !== true) {
565
- log('startStrand: strand %s is already running but was NOT launched as founder — ' +
566
- 'this early return DROPS the founder request; use foundExistingStrand', strandId);
567
- }
568
- log('Strand %s already running', strandId);
569
- return this.instances.get(strandId)!;
570
- }
571
-
572
- log('Starting strand instance: %s (%s)', strandId, describeSApp(sAppConfig));
573
- const tTotal = performance.now();
574
-
575
- // The id becomes two names below: the storage scope key the embedder's provider
576
- // turns into a directory or database name, and `networkName` in buildStrandRuntime,
577
- // from which the libp2p protocol prefix `/optimystic/strand-<id>` is built. A
578
- // replicated strand row carries whatever id the founding node wrote, so neither is
579
- // safe unchecked. Asserted HERE rather than in `resolveStrandStorage` because that
580
- // function returns early when no provider is configured — a node with no storage
581
- // still reaches buildStrandRuntime and still mints the protocol prefix.
582
- assertStrandScopeKey(strandId);
583
-
584
- const sAppInfo = sAppConfig ? verifiedSAppInfo(strandId, sAppConfig, config.requireSignedSchemas) : undefined;
585
-
586
- // Resolve this strand's storage ONCE, before anything is recorded. If a factory
587
- // function is provided, it is called with the strandId to create strand-specific
588
- // storage (e.g. strand-isolated directories). A provider that throws therefore
589
- // fails the launch alongside the schema-signature check, leaving nothing tracked.
590
- const strandStorage = resolveStrandStorage(config.storage?.provider, strandId);
591
- if (strandStorage) {
592
- log('Strand %s using provided storage provider', strandId);
593
- }
594
-
595
- // Determine latency hint: sApp config > default
596
- const latencyHint = sAppConfig?.latencyHint ?? config.defaultLatencyHint;
597
-
598
- const instance: StrandInstance = {
599
- strandId,
600
- status: 'starting',
601
- sAppInfo,
602
- memberPrivateKey: strandRow.MemberPrivateKey ?? undefined,
603
- connectedPeers: 0,
604
- lastActivity: new Date(),
605
- latencyHint
606
- };
607
-
608
- this.instances.set(strandId, instance);
609
- this.launchConfigs.set(strandId, config);
610
- if (strandStorage) {
611
- this.strandStorages.set(strandId, strandStorage);
612
- }
613
-
614
- try {
615
- await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, config));
616
- timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
617
- log('Strand %s started successfully (%s)', strandId, describeSApp(sAppConfig));
618
- return instance;
619
- } catch (error) {
620
- // Status/error first — the (now discarded) record is still what `log` reports on.
621
- instance.status = 'error';
622
- instance.error = error instanceof Error ? error.message : String(error);
623
- log('Failed to start strand %s: %s', strandId, instance.error);
624
- // Drop the dead record so this strand id can be launched again. Keep the
625
- // `launchConfigs` has an entry iff `instances` does invariant — resumeStrand
626
- // reads both, and a config without an instance would strand the config.
627
- this.instances.delete(strandId);
628
- this.launchConfigs.delete(strandId);
629
- // The store this launch resolved goes with it: nothing owns it any more, and a
630
- // retained cache wrapper would be handed back (already retired) on a retry.
631
- await this.disposeStrandStorage(strandId);
632
- throw error;
633
- }
634
- }
635
-
636
- /** Hold `build` in {@link runtimeBuilds} until it settles, and settle as it does. */
637
- private async trackRuntimeBuild(strandId: string, build: Promise<void>): Promise<void> {
638
- this.runtimeBuilds.set(strandId, build);
639
- try {
640
- await build;
641
- } finally {
642
- if (this.runtimeBuilds.get(strandId) === build) {
643
- this.runtimeBuilds.delete(strandId);
644
- }
645
- }
646
- }
647
-
648
- /**
649
- * Wait until no runtime build of `strandId` is in flight. Never rejects: a failed build
650
- * is reported by the call that started it, and the caller re-reads the instance after.
651
- *
652
- * NOTE: after a failed launch the re-read sees the instance gone only because
653
- * `startStrand`'s catch deletes it synchronously, and its continuation is queued before
654
- * this waiter's. If that cleanup ever awaits before `instances.delete`, a waiter can act
655
- * on the dying record; move the cleanup inside the tracked promise then.
656
- */
657
- private async settleRuntimeBuilds(strandId: string): Promise<void> {
658
- for (let build = this.runtimeBuilds.get(strandId); build; build = this.runtimeBuilds.get(strandId)) {
659
- await build.catch((error: unknown) => {
660
- log('Strand %s: the runtime build waited on failed (reported by its launcher): %o', strandId, error);
661
- });
662
- }
663
- }
664
-
665
- /**
666
- * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
667
- * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
668
- * launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
669
- * volatile inputs (bootstrapNodes, servingMachines, network, profile,
670
- * privateKey, sApp config) from `config`, so the caller controls the
671
- * cohort-derived values. Storage is the
672
- * one input it does NOT re-read from `config`: that belongs to the instance and
673
- * comes from `strandStorages`.
674
- */
675
- private async buildStrandRuntime(instance: StrandInstance, config: StartStrandConfig): Promise<void> {
676
- const strandId = instance.strandId;
677
- const { sAppConfig } = config;
678
-
679
- // Refuse a bad `linkRoundTripMs` before any strand bring-up, for the reason
680
- // `CadreNode.start()` does: every budget derived from it here is behind a condition
681
- // (backfill disabled, no relay addrs), so a zero or NaN declaration would otherwise
682
- // surface later inside a path that logs and carries on. `link-budget.ts`.
683
- resolveLinkRoundTripMs(config.network?.linkRoundTripMs);
684
-
685
- // The store the instance OWNS (resolved once in `startStrand`), not a fresh
686
- // resolution: a rebuild must reach the same backend through the same warm cache.
687
- const strandStorage = this.strandStorages.get(strandId);
688
-
689
- // db-p2p namespaces every one of the node's protocol ids by network name
690
- // (`/optimystic/<networkName>/...`), so anything dialing this node's own
691
- // services must derive its prefix from the SAME string the node was built
692
- // with — hence one binding for both, not two literals that can drift.
693
- const networkName = `strand-${strandId}`;
694
- const protocolPrefix = `/optimystic/${networkName}`;
695
-
696
- // Whether this node runs the circuit-relay server, and its init — the same
697
- // resolution the control node takes from the same config (`relay-server.ts`).
698
- const relayServer = resolveRelayServer(config.network, config.profile);
699
- // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
700
- // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
701
- // bind one port), the announce config is dropped (it names the control node's
702
- // address), and every configured relay becomes one bare `/p2p-circuit` SEARCH
703
- // listener plus a dial addr in `relayAddrs` for the per-relay supervisor started
704
- // below — the same route the control node takes, because the configured
705
- // `<relay>/p2p-circuit` shape loses its address on a relay restart, a hangup, or
706
- // libp2p's own reservation refresh and recovers from none of them. See
707
- // `strand-network-config.ts` for all three, and `relay-reservation.ts`.
708
- //
709
- // `relayAddrs` is destructured OFF: it is not a `createLibp2pNode` option, and
710
- // spreading it would hand db-p2p a key it does not know.
711
- const { relayAddrs: relayDialAddrs = [], ...addrOptions } = strandNodeAddrs(config.network);
712
-
713
- // The CLOSED-strand revoked-peer gate (see strand-revocation-enforcer.ts).
714
- // Constructed BEFORE the libp2p node because the node's options embed its
715
- // predicates; its deny set starts empty (admit everything — fail-open, same
716
- // posture as bring-up everywhere here) and is first populated after the
717
- // strand database initializes below. Registered in the map immediately so
718
- // the failure rollback (releaseRuntime) tears it down like every other
719
- // runtime component. Open strands skip it entirely: their Member/MemberPeer
720
- // tables are empty by schema (`OnlyClosed`), so there is nothing to derive
721
- // a deny set from — arming would be pointless polling.
722
- const revocationEnforcer =
723
- config.strandRow.Type === 'c' && config.revocationEnforcement?.enabled !== false
724
- ? new StrandRevocationEnforcer({
725
- label: strandId,
726
- readRows: () => {
727
- const database = instance.database;
728
- if (!database) {
729
- throw new Error(`strand ${strandId} has no live database`);
730
- }
731
- return readStrandRevocationRows(database.getDatabase());
732
- },
733
- // Read per sweep, never captured: this closure is built BEFORE the
734
- // libp2p node exists (its options embed the enforcer's predicates),
735
- // and `releaseRuntime` clears the field again on quiesce. Undefined
736
- // therefore means "no transport right now", which the enforcer
737
- // treats as "nothing to tear down".
738
- getNetwork: () => instance.libp2pNode,
739
- onSelfRevoked: () => config.onSelfRevoked?.(strandId)
740
- }, config.revocationEnforcement)
741
- : undefined;
742
- if (revocationEnforcer) {
743
- this.revocationEnforcers.set(strandId, revocationEnforcer);
744
- }
745
-
746
- // Closed strands compose revoked-peer denial onto the caller's gater and arm
747
- // the fail-closed per-stream gate; open strands keep the raw configured gater
748
- // (their peers are legitimately cross-party and nothing is ever revoked).
749
- const revocationGateOptions = revocationEnforcer
750
- ? {
751
- connectionGater: createRevocationConnectionGater(revocationEnforcer, config.network?.connectionGater),
752
- authorizeInboundStream: (remotePeerId: string, protocol: string) =>
753
- revocationEnforcer.authorizeStream(remotePeerId, protocol)
754
- }
755
- : config.network?.connectionGater
756
- ? { connectionGater: config.network.connectionGater }
757
- : {};
758
-
759
- const reactivityOption = strandCohortTopicOption(config.reactivity, strandId);
760
- if (reactivityOption.cohortTopic) {
761
- log('strand %s: change notifications on', strandId);
762
- }
763
-
764
- try {
765
- // Bound once: the breadth is also the ceiling on the repair yardstick below, and the
766
- // two must be derived from the same resolution. Inside the `try` deliberately — a
767
- // rejected clusterSize is a build failure that runs the same cleanup as any other.
768
- const strandClusterSize = resolveStrandClusterSize(config.clusterSize);
769
-
770
- let t0 = performance.now();
771
- const node = await createLibp2pNode({
772
- port: 0, // Random port
773
- bootstrapNodes: config.bootstrapNodes ?? [],
774
- networkName,
775
- storage: strandStorage,
776
- fretProfile: config.profile === 'storage' ? 'core' : 'edge',
777
- relay: relayServer.enabled,
778
- // NOTE: unlike the control node, a strand node has no unauthorized-reservation budget,
779
- // so with the per-connection cap off any peer that reaches this node can hold one of
780
- // `maxReservations` slots and forward uncapped through it. If strand relays are ever
781
- // abused for bandwidth, give strand nodes a budget or a capped init of their own.
782
- ...(relayServer.enabled && { relayServerInit: relayServer.init }),
783
- clusterSize: strandClusterSize,
784
- // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
785
- // breadth reasoning, and the shape match with the control policy is a coincidence.
786
- //
787
- // The builder declares this node's block-repair corroboration yardstick from the
788
- // count of machines SERVING this strand, capped at the breadth above (a block
789
- // never lives on more machines than the cohort is wide). Given no count — the
790
- // production path today, since no per-strand serving count exists yet — it
791
- // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
792
- // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
793
- // serving set that changed while the strand slept.
794
- // The read deadline is settled by the same helper the control node uses, because the
795
- // two networks ride one link: the host's own, else derived from its declared
796
- // `linkRoundTripMs`, else nothing, and the base policy's COHORT_READ_DEADLINE_MS stands.
797
- clusterPolicy: strandClusterPolicy(strandClusterSize, {
798
- servingMachines: config.servingMachines,
799
- cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(config.network)
800
- }),
801
- arachnode: {
802
- enableRingZulu: config.profile === 'storage'
803
- },
804
- ...(config.privateKey && { privateKey: config.privateKey }),
805
- // Built per runtime, not per launch: the adapter stops saving once the strand's
806
- // state is forgotten, and a rebuilt runtime is what starts it saving again.
807
- //
808
- // NOTE: the saved state also restores which peers db-p2p saw serving the strand,
809
- // and it counts them in the cohort until FRET finds them unreachable. A node built
810
- // while every other member is offline therefore fails its first writes ("Failed to
811
- // get super-majority") — for 4-6 s on loopback, the window a running node already
812
- // sees when a member drops. If apps report failed sends after relaunching alone on
813
- // a slow relayed link, retry the write in the app or raise it upstream (db-p2p's
814
- // remembered-serving fallback); dropping `persistence` brings back the split
815
- // restart.
816
- ...(config.networkState && { persistence: strandNetworkStatePersistence(config.networkState, strandId) }),
817
- ...(config.network?.transports && { transports: config.network.transports }),
818
- ...(config.network?.noiseCrypto && { noiseCrypto: config.network.noiseCrypto }),
819
- // Unconditional, and the same default the control node takes: every node of
820
- // every party runs the monitor over its connection to a slow phone, so the
821
- // widened ping deadline has to reach the strand nodes too (see
822
- // DEFAULT_CONNECTION_MONITOR).
823
- connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
824
- // The same declared link the control node states, and for the same reason: a strand
825
- // node is also the listener for every other member's strand node, and Optimystic
826
- // derives that listener's limit from this (`link-budget.ts`).
827
- linkRoundTripMs: resolveLinkRoundTripMs(config.network?.linkRoundTripMs),
828
- // The same dial limits as the control node: a strand node dials the same machines,
829
- // through the same relays and gates.
830
- ...optimysticDialLimits(config.network?.linkRoundTripMs),
831
- // Listen entries plus the WebSocket transport switch they imply — a strand node
832
- // announces nothing the operator configured (`strand-network-config.ts`), and
833
- // spreads AFTER `transports` above because the switch is a no-op whenever the
834
- // embedder supplied its own factories. The bare `/p2p-circuit` entries (one per
835
- // relay) open no connection here; each registers a pending reservation that the
836
- // matching supervisor below fills, which is what gives a NAT'd strand node a
837
- // reachable relay slot — and works against a party-run relay because the launch
838
- // path announced this strand's derived peerId to it first (delegate admission;
839
- // see cadre-node.ts).
840
- ...addrOptions,
841
- // The raw configured gater (open strands), or the revocation-composed
842
- // gater plus the fail-closed per-stream revoked-peer gate (closed
843
- // strands) — resolved above, before the try.
844
- ...revocationGateOptions,
845
- // `{}` unless the operator enabled change notifications for this strand (`strand-reactivity.ts`).
846
- ...reactivityOption
847
- }) as Libp2pNodeWithRepo;
848
- timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
849
-
850
- instance.libp2pNode = node;
851
-
852
- // One reservation supervisor PER RELAY, started now so the first attempts
853
- // overlap the database bring-up below (the relay is never in a strand's
854
- // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
855
- // — so an open relay connection cannot disturb that bring-up). Their first
856
- // attempts are awaited before the strand goes `active`, below. Registered in the
857
- // map immediately so the failure rollback (releaseRuntime) stops them like every
858
- // other runtime component.
859
- this.relaySupervisors.set(strandId, this.startRelaySupervisors(strandId, node, relayDialAddrs, config));
860
-
861
- // Create and initialize the StrandDatabase.
862
- //
863
- // Held by the first-sync gate (registered before initialize) rather than attached
864
- // to the instance: a failed init is then cleaned up by releaseRuntime below through
865
- // the gate (close() is safe on a partially-initialized db), and the app only ever
866
- // sees `instance.database` once this machine is allowed to write to the strand.
867
- t0 = performance.now();
868
- const strandDb = new StrandDatabase({
869
- strandId,
870
- sAppConfig,
871
- libp2pNode: node,
872
- coordinatedRepo: node.coordinatedRepo,
873
- // Founder bootstrap inputs: the strand's type drives which membership rows
874
- // are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
875
- // caller from the control-layer StrandPartyKey row) derives the founding
876
- // Member/Manager key. The row's shared MemberPrivateKey rides along as the
877
- // read-gating secret only — it derives nobody's identity.
878
- strandType: config.strandRow.Type,
879
- memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
880
- partyMemberPrivateKey: config.partyMemberPrivateKey,
881
- founder: config.founder
882
- });
883
- const gate = new StrandFirstSyncGate({
884
- label: strandId,
885
- database: strandDb,
886
- onHeaderHeld: () => this.publishDatabase(instance, strandDb)
887
- }, config.firstSync);
888
- this.firstSyncGates.set(strandId, gate);
889
- await strandDb.initialize();
890
- timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
891
-
892
- // The first-sync write gate. A founder just wrote the Header in its bootstrap; any
893
- // other machine may commit only once it holds the Header — which it can only have
894
- // received from a peer (now, or on an earlier run over the same store) — and has read
895
- // each App table once, so their collections are fetched too. Publishing the database
896
- // is what makes the strand writable to the app; until then the instance stays
897
- // `'syncing'` and the gate re-probes on its cadence. Everything armed below reads
898
- // `instance.database` lazily, so the reconciler and enforcer wait with it — and
899
- // `publishDatabase` kicks the reconciler when the wait ends, so a joiner's membership
900
- // rows land with the strand rather than one retry later.
901
- t0 = performance.now();
902
- if (config.founder === true || await strandFirstSyncComplete(strandDb.getDatabase(), strandId)) {
903
- this.publishDatabase(instance, strandDb);
904
- } else {
905
- gate.start();
906
- }
907
- timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
908
-
909
- // The membership reconciler: finish this party's join on every machine —
910
- // redeem a staged formation invitation if one is pending, then write the
911
- // durable machine→party binding (Strand.MemberPeer). Armed here, the one
912
- // seam where the strand's transport peer id and a live Database both exist
913
- // for launch AND hibernation resume, for closed strands that carry the
914
- // party's own membership key. start() kicks an immediate pass WITHOUT
915
- // awaiting it — a joiner at this instant has not synced the founder's rows
916
- // and may lack write quorum, so bring-up is never blocked; an unfinished join
917
- // then retries on a short doubling ladder capped at the configured
918
- // pollIntervalMs (which is also the flat cadence while nobody has admitted this
919
- // party yet), and the loop stops once the member row and this machine's own
920
- // binding are both in place. Armed AFTER the gate above, so a gated launch
921
- // always has a reconciler registered by the time the gate can publish.
922
- if (config.strandRow.Type === 'c' && config.partyMemberPrivateKey
923
- && config.membershipReconciliation?.enabled !== false) {
924
- const reconciler = new StrandMembershipReconciler({
925
- label: strandId,
926
- partyMemberPrivateKey: config.partyMemberPrivateKey,
927
- // Read per pass, never captured — same lifecycle argument as the
928
- // enforcer's getNetwork: quiesce drops both handles.
929
- getDatabase: () => instance.database?.getDatabase(),
930
- getOwnPeerId: () => instance.libp2pNode?.peerId.toString(),
931
- pendingInvite: config.pendingMembershipInvite,
932
- // The enforcer's CURRENT snapshot view of this node's own peer id — the
933
- // reconciler stops rather than fight a self-revocation. Absent when the
934
- // gate is disarmed (fail-open: the loop just runs).
935
- isSelfRevoked: revocationEnforcer
936
- ? () => {
937
- const ownPeerId = instance.libp2pNode?.peerId.toString();
938
- return ownPeerId !== undefined && revocationEnforcer.isRevoked(ownPeerId);
939
- }
940
- : undefined,
941
- onRejoinBlocked: () => config.onRejoinBlocked?.(strandId)
942
- }, {
943
- pollIntervalMs: config.membershipReconciliation?.pollIntervalMs
944
- ?? config.revocationEnforcement?.pollIntervalMs
945
- });
946
- this.membershipReconcilers.set(strandId, reconciler);
947
- reconciler.start();
948
- }
949
-
950
- // Arm the revoked-peer deny-set poll now that the database it reads
951
- // exists. start() kicks an immediate refresh WITHOUT awaiting it —
952
- // bring-up and resume are never blocked on a membership read; until the
953
- // first read lands the empty snapshot admits everyone (fail-open).
954
- revocationEnforcer?.start();
955
-
956
- // Peer-join block catch-up: push this strand's own blocks to each newly
957
- // connected peer, so a machine that joined after blocks were committed
958
- // still ends up physically holding them (without per-strand storage there
959
- // is nothing to copy). Armed for EVERY stored strand: `PeerJoinBackfill`
960
- // only does work when the strand's libp2p node reports a peer connection,
961
- // so on a device that is genuinely alone it is inert, and arming it at
962
- // launch is what closes the "founded alone, never replicates" hole — a
963
- // peer that joins later gets the founder's blocks without any relaunch.
964
- // No `authorizePeer` gate, deliberately — see the module comment in
965
- // peer-join-backfill.ts for the strand-side argument (and why the control
966
- // network, which DOES gate, is different).
967
- //
968
- // NOTE: cost is one PeerJoinBackfill object + one `peer:identify` listener
969
- // per running strand — linear in strand count, negligible at the handful a
970
- // device or host runs today. If a node ever hosts strands by the hundred,
971
- // move to one shared listener that dispatches by strand id.
972
- if (strandStorage && config.backfill?.enabled !== false) {
973
- if (node.keyNetwork) {
974
- const backfill = new PeerJoinBackfill({
975
- label: strandId,
976
- libp2p: node,
977
- peerNetwork: node.keyNetwork,
978
- storage: strandStorage,
979
- // The same prefix the receiver registered its block-transfer handler
980
- // under — derived from networkName above, never re-spelled here.
981
- protocolPrefix
982
- }, {
983
- // Dial and response deadlines counted in link round trips, not fixed milliseconds:
984
- // a relayed dial costs a fixed number of exchanges, so a fixed budget has a link
985
- // speed above which this catch-up can never reach the peer at all. See
986
- // `link-budget.ts`. Spread BEFORE the host's own config so an explicit
987
- // `strandBackfill.dialTimeoutMs` still wins.
988
- ...peerJoinPushBudget(config.network?.linkRoundTripMs),
989
- ...config.backfill
990
- });
991
- backfill.start();
992
- this.backfills.set(strandId, backfill);
993
- } else {
994
- log('Strand %s: libp2p node exposes no keyNetwork; peer-join block catch-up is inert', strandId);
995
- }
996
- }
997
-
998
- // The relay supervisors' FIRST attempts, all of them, before `active`: the happy
999
- // path still publishes its circuit addr before `addStrand` resolves (the
1000
- // strand-addr RPC answers read it, and the same-party circuit scenario asserts
1001
- // it). Fail-SOFT — a first attempt that lands nothing does not fail the launch.
1002
- // The strand's database is up, and the supervisor keeps trying on its backoff;
1003
- // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
1004
- //
1005
- // NOTE: a relay that is down costs this launch one full drive — the reservation budget
1006
- // counted from the declared link round trip, 18 s at its default (`link-budget.ts`) — and
1007
- // `StrandWatcher` launches strands one at a time, so N strands cost N of those in
1008
- // bring-up during a relay outage, and MORE on a host that declared a slower link. If that
1009
- // ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
1010
- // than shortening the drive.
1011
- t0 = performance.now();
1012
- await this.awaitFirstRelayAttempts(strandId);
1013
- timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
1014
-
1015
- // No `lastActivity` stamp: bringing a runtime up is not activity. A check-in marks
1016
- // activity before its resume and must see only what landed while the strand was up.
1017
- instance.status = liveStrandStatus(instance);
1018
- if (instance.status === 'syncing') {
1019
- log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
1020
- 'a member of the strand is reached', strandId);
1021
- }
1022
- } catch (error) {
1023
- // Roll back any partially-attached runtime so the instance is left with
1024
- // NEITHER handle. Otherwise the `libp2pNode || database` "already live"
1025
- // guard in resumeStrand/handleStrandWake would treat a half-built strand
1026
- // as healthy — leaking the libp2p node and never retrying the rebuild.
1027
- await this.releaseRuntime(instance).catch((cleanupErr) => {
1028
- log('buildStrandRuntime cleanup for strand %s also failed: %o', strandId, cleanupErr);
1029
- });
1030
- throw error;
1031
- }
1032
- }
1033
-
1034
- /**
1035
- * Release an instance's strand-network runtime: stop the relay supervisors and
1036
- * the other background loops, close the StrandDatabase, then stop the libp2p
1037
- * node (construction order in reverse), clearing both fields and zeroing
1038
- * connectedPeers. Tolerant of partially-built state — either handle may be
1039
- * absent — so it doubles as rollback for a failed `buildStrandRuntime`. Shared
1040
- * by `quiesceStrand`, `stopStrand`, and that rollback path.
1041
- *
1042
- * Leaves `strandStorages` untouched by design — the store outlives the runtime it
1043
- * was built into, which is what makes a resume warm. Only `stopStrand` disposes it.
1044
- */
1045
- private async releaseRuntime(instance: StrandInstance): Promise<void> {
1046
- // Relay supervisors FIRST of all — before anything below is torn down — so no
1047
- // re-drive is scheduled against a node being stopped. `stop()` is synchronous
1048
- // and never awaits a drive, so a relay that is down cannot delay this teardown.
1049
- // `stop()` also cancels the drive already in flight, so an attempt started
1050
- // moments before this does not keep dialing and polling the node the lines
1051
- // below are stopping; its result is discarded either way.
1052
- const relaySupervisors = this.relaySupervisors.get(instance.strandId);
1053
- if (relaySupervisors) {
1054
- relaySupervisors.forEach((supervisor) => supervisor.stop());
1055
- this.relaySupervisors.delete(instance.strandId);
1056
- }
1057
- // Backfill next — before the database closes and the libp2p node stops — so
1058
- // no NEW catch-up push is issued against a torn-down transport. A push already
1059
- // in flight is not awaited; it fails into the module's own per-chunk catch.
1060
- const backfill = this.backfills.get(instance.strandId);
1061
- if (backfill) {
1062
- backfill.stop();
1063
- this.backfills.delete(instance.strandId);
1064
- }
1065
- // The revoked-peer enforcer goes with the runtime it gated: a resume
1066
- // rebuilds it with a fresh (initially empty, fail-open) snapshot.
1067
- const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
1068
- if (revocationEnforcer) {
1069
- revocationEnforcer.stop();
1070
- this.revocationEnforcers.delete(instance.strandId);
1071
- }
1072
- // The membership reconciler likewise: a resume rebuilds it and re-runs the
1073
- // idempotent ladder, which is what makes resume-after-partial-join heal.
1074
- const membershipReconciler = this.membershipReconcilers.get(instance.strandId);
1075
- if (membershipReconciler) {
1076
- membershipReconciler.stop();
1077
- this.membershipReconcilers.delete(instance.strandId);
1078
- }
1079
- // A still-gated database (a joiner that never received the Header, or a launch whose
1080
- // initialize() failed) is closed through its gate; a published one through the
1081
- // instance. Never both — publishing moves the handle from the gate to the instance.
1082
- const gate = this.firstSyncGates.get(instance.strandId);
1083
- if (gate) {
1084
- gate.stop();
1085
- this.firstSyncGates.delete(instance.strandId);
1086
- await gate.database.close();
1087
- }
1088
- if (instance.database) {
1089
- await instance.database.close();
1090
- instance.database = undefined;
1091
- }
1092
- if (instance.libp2pNode) {
1093
- await instance.libp2pNode.stop();
1094
- instance.libp2pNode = undefined;
1095
- }
1096
- instance.connectedPeers = 0;
1097
- }
1098
-
1099
- /**
1100
- * One {@link superviseRelayReservation} per relay dial addr, each over exactly
1101
- * that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
1102
- * one relay re-drives only that one. Default retry timings — the control node's (2 s
1103
- * doubling to 60 s between failed attempts, a 5 s liveness check while held). Each DRIVE's
1104
- * own deadline is counted in link round trips at this host's declared
1105
- * `network.linkRoundTripMs` (`link-budget.ts`), so the same declaration that lengthens the
1106
- * control node's drive lengthens these.
1107
- *
1108
- * Each supervisor's `beforeRedrive` is the caller's
1109
- * {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
1110
- * node's peer id, so a re-drive after a relay restart is preceded by a fresh
1111
- * delegate grant on the relay about to be dialed. Returns `[]` when no relay is
1112
- * configured — the strand then simply has no circuit listener to fill.
1113
- */
1114
- private startRelaySupervisors(
1115
- strandId: string,
1116
- node: Libp2pNodeWithRepo,
1117
- relayDialAddrs: readonly string[],
1118
- config: StartStrandConfig
1119
- ): RelayReservationSupervisor[] {
1120
- if (relayDialAddrs.length === 0) {
1121
- return [];
1122
- }
1123
- const announce = config.announceDelegateToRelay;
1124
- const delegatePeerId = node.peerId.toString();
1125
- const timeoutMs = relayReservationBudgetMs(config.network?.linkRoundTripMs);
1126
- return relayDialAddrs.map((relayAddr) => superviseRelayReservation(node, [relayAddr], {
1127
- timeoutMs,
1128
- ...(announce && { beforeRedrive: () => announce(strandId, relayAddr, delegatePeerId) })
1129
- }));
1130
- }
1131
-
1132
- /**
1133
- * Wait for every relay supervisor's first attempt to settle, concurrently, and log
1134
- * the ones that landed nothing. Never throws — a strand whose relay is unreachable
1135
- * at launch still comes up, with its supervisors retrying in the background.
1136
- */
1137
- private async awaitFirstRelayAttempts(strandId: string): Promise<void> {
1138
- const supervisors = this.relaySupervisors.get(strandId) ?? [];
1139
- if (supervisors.length === 0) {
1140
- return;
1141
- }
1142
- await Promise.all(supervisors.map((supervisor) => supervisor.firstAttempt));
1143
- supervisors.forEach((supervisor) => {
1144
- if (supervisor.lastError !== null) {
1145
- log('Strand %s: relay reservation not held after the first attempt (retrying in the background): %s',
1146
- strandId, supervisor.lastError);
1147
- }
1148
- });
1149
- }
1150
-
1151
- /**
1152
- * Drop the strand's owned store and release this strand's claim on its cache. The
1153
- * wrapper counts holders, so the cache is emptied and unregistered from the shared
1154
- * pool only if no other scope still holds it. Called only where the strand's whole
1155
- * lifetime ends (`stopStrand`, and the failed-launch rollback) — never on a quiesce.
1156
- *
1157
- * A dispose failure is logged, not thrown: the store is already unreferenced here,
1158
- * and failing the stop over a cache-bookkeeping error would leave the caller unable
1159
- * to tear the strand down. `disposeStorageCache` no-ops for an unwrapped store
1160
- * (e.g. `MemoryRawStorage`), so no instanceof test is needed.
1161
- */
1162
- private async disposeStrandStorage(strandId: string): Promise<void> {
1163
- const storage = this.strandStorages.get(strandId);
1164
- if (!storage) {
1165
- return;
1166
- }
1167
- this.strandStorages.delete(strandId);
1168
- try {
1169
- await disposeStorageCache(storage);
1170
- } catch (error) {
1171
- log('Failed to dispose storage cache for strand %s: %o', strandId, error);
1172
- }
1173
- }
1174
-
1175
- /**
1176
- * Refresh a strand's revoked-peer deny set NOW — and, with it, run the
1177
- * teardown sweep that hangs up every connected revoked peer.
1178
- *
1179
- * The gate is otherwise poll-driven (default
1180
- * `DEFAULT_REVOCATION_POLL_INTERVAL_MS`), because neither a replicated
1181
- * revocation nor a local `revokeMember`/`leaveStrand` raises anything this
1182
- * runtime can hook. A caller that just wrote a revocation against the
1183
- * strand's database should follow it with this call, which makes the cut
1184
- * immediate instead of waiting out the interval.
1185
- *
1186
- * Quiet no-op — matching `quiesceStrand`'s posture — when the strand is not
1187
- * tracked, is quiesced, is open, or has the gate disabled: in every one of
1188
- * those cases there is no enforcer to refresh. Never rejects (the enforcer's
1189
- * refresh contains its own failures), and resolves only once the sweep has
1190
- * finished.
1191
- */
1192
- async refreshRevocationEnforcement(strandId: string): Promise<void> {
1193
- const enforcer = this.revocationEnforcers.get(strandId);
1194
- if (!enforcer) {
1195
- log('refreshRevocationEnforcement: strand %s has no armed revocation enforcer', strandId);
1196
- return;
1197
- }
1198
- await enforcer.refresh();
1199
- }
1200
-
1201
- /**
1202
- * A fresh membership invitation was staged for `strandId` (`CadreNode`'s
1203
- * `adoptFormationMembershipInvite`): re-arm the strand's membership reconciler so the
1204
- * invitation is attempted now rather than never — the loop finished during the first
1205
- * join and would otherwise stay stopped until a relaunch. Quiet no-op when no reconciler
1206
- * is armed: a first formation stages before the strand is added, and bring-up then arms
1207
- * a loop that finds the invitation by itself. Returns at once; the pass runs on the
1208
- * loop's own serialized chain and never rejects.
1209
- */
1210
- notifyMembershipInviteStaged(strandId: string): void {
1211
- const reconciler = this.membershipReconcilers.get(strandId);
1212
- if (!reconciler) {
1213
- log('notifyMembershipInviteStaged: strand %s has no armed membership reconciler — bring-up will find the invitation', strandId);
1214
- return;
1215
- }
1216
- void reconciler.rearm();
1217
- }
1218
-
1219
- /**
1220
- * Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
1221
- * self arm of `removeMemberPeer`, signed with the retained launch config's party
1222
- * key. Called by `CadreNode.unpublishStrand` BEFORE the local stop, while the
1223
- * strand database and transport are still live: after the unpublish commits, the
1224
- * party's `StrandPartyKey` control row is gone, so no restart could ever sign
1225
- * this removal again — the in-memory retained key is the last chance.
1226
- *
1227
- * Never throws. Quiet no-op for a strand that is untracked, open, or launched
1228
- * without a party key; a quiesced (hibernating) strand, or a strand whose write
1229
- * quorum is already unreachable, leaves the stale binding behind with a log —
1230
- * it grants nothing today (admission is not allowlist-gated) and only
1231
- * mis-credits diversity. The strand's own membership reconciler is stopped —
1232
- * and its in-flight pass awaited — FIRST, so a pass that was already past its
1233
- * own stopped check cannot re-register the binding this is about to clear.
1234
- *
1235
- * NOTE: deliberately NOT handled here (or anywhere yet): clearing the bindings
1236
- * of a machine removed from the CADRE at the control layer, or of a party's
1237
- * OTHER machines when the party unpublishes — the party still holds the key on
1238
- * every surviving machine and can clear from any of them, and the stale rows
1239
- * grant nothing while admission is deny-list only. Revisit when
1240
- * `feat-strand-member-allowlist-admission` lands and a stale binding starts
1241
- * granting admission rather than merely mis-crediting diversity.
1242
- */
1243
- async clearOwnMemberPeerBinding(strandId: string): Promise<void> {
1244
- const instance = this.instances.get(strandId);
1245
- const config = this.launchConfigs.get(strandId);
1246
- if (!instance || !config || config.strandRow.Type !== 'c' || !config.partyMemberPrivateKey) {
1247
- return;
1248
- }
1249
- const reconciler = this.membershipReconcilers.get(strandId);
1250
- if (reconciler) {
1251
- reconciler.stop();
1252
- await reconciler.settle();
1253
- }
1254
- const db = instance.database?.getDatabase();
1255
- const peerId = instance.libp2pNode?.peerId.toString();
1256
- if (!db || !peerId) {
1257
- log('clearOwnMemberPeerBinding: strand %s has no live runtime — own MemberPeer binding left in place', strandId);
1258
- return;
1259
- }
1260
- try {
1261
- // Background write on the app's database: never join a transaction the app has open.
1262
- await removeMemberPeer(db, {
1263
- memberKeyPair: strandMemberKeyPair(config.partyMemberPrivateKey),
1264
- peerId
1265
- }, { joinOpenTransaction: false });
1266
- log('clearOwnMemberPeerBinding: strand %s — own binding for peer %s cleared', strandId, peerId);
1267
- } catch (error) {
1268
- log('clearOwnMemberPeerBinding: strand %s — best-effort clear failed (strand may already be unreachable): %o',
1269
- strandId, error);
1270
- }
1271
- }
1272
-
1273
- /**
1274
- * Quiesce a strand: release its strand-network resources (stop the libp2p node,
1275
- * close the StrandDatabase) while RETAINING the instance record — identity,
1276
- * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
1277
- * resumed later, then mark it `hibernating`. Mechanically this is `stopStrand` minus
1278
- * the instance/config deletion. A tracked strand with no runtime left (a failed
1279
- * rebuild already released it) is only marked; an untracked one is left alone.
1280
- *
1281
- * Ordered with {@link resumeStrand} (see {@link transitions}): it starts once the resume
1282
- * or launch build before it has settled, so it releases a finished runtime, and the
1283
- * status write is its last step, so a resume requested meanwhile writes after it. A call
1284
- * made while another quiesce is the latest request joins it. A failed release leaves the
1285
- * status as it was and rejects.
1286
- */
1287
- async quiesceStrand(strandId: string): Promise<void> {
1288
- const previous = this.transitions.get(strandId);
1289
- if (previous?.kind === 'quiesce') {
1290
- log('quiesceStrand: strand %s — joining the quiesce already requested', strandId);
1291
- return previous.operation;
1292
- }
1293
- const operation = this.runQuiesce(strandId, previous);
1294
- this.holdTransition(strandId, { kind: 'quiesce', operation });
1295
- return operation;
1296
- }
1297
-
1298
- /** Body of {@link quiesceStrand}. */
1299
- private async runQuiesce(strandId: string, previous: RuntimeTransition | undefined): Promise<void> {
1300
- await this.settleTransition(strandId, previous);
1301
- await this.settleRuntimeBuilds(strandId);
1302
- const instance = this.instances.get(strandId);
1303
- if (!instance) {
1304
- log('quiesceStrand: strand %s not found', strandId);
1305
- return;
1306
- }
1307
- if (instance.libp2pNode || instance.database) {
1308
- log('Quiescing strand instance: %s', strandId);
1309
- await this.releaseRuntime(instance);
1310
- log('Strand %s quiesced (resources released, instance retained)', strandId);
1311
- } else {
1312
- log('quiesceStrand: strand %s already quiesced', strandId);
1313
- }
1314
- instance.status = 'hibernating';
1315
- }
1316
-
1317
- /**
1318
- * Whether the latest runtime transition requested for `strandId` is a quiesce that has
1319
- * not finished. Its handles may still be attached and its status still live, but it is on
1320
- * its way to `hibernating`, so a caller that needs it up must {@link resumeStrand} it — the
1321
- * resume runs after the quiesce — rather than treat it as live.
1322
- */
1323
- isQuiescing(strandId: string): boolean {
1324
- return this.transitions.get(strandId)?.kind === 'quiesce';
1325
- }
1326
-
1327
- /**
1328
- * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
1329
- * from the retained launch config and re-attach them, transitioning it back to
1330
- * `active`. `overrides` re-applies volatile inputs that may have changed since
1331
- * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
1332
- * count) and updates the retained config so a later resume reuses the latest
1333
- * values. Returns the live instance unchanged if it is already running.
1334
- *
1335
- * Overlapping calls share one rebuild: a call made while a resume is the latest request
1336
- * for the strand — a wake landing during a check-in — joins it and settles exactly as it
1337
- * does, `'error'` status included. The joiner's own `overrides` are ignored; the first
1338
- * resume's seed wins, and both callers resolved the cohort seed moments apart. A call made
1339
- * while a quiesce is the latest request runs after that quiesce and rebuilds (see
1340
- * {@link transitions}), and one made while `startStrand` is still building waits for that
1341
- * build rather than starting a second.
1342
- */
1343
- async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
1344
- if (this.stopping) {
1345
- throw new Error('StrandInstanceManager is stopping');
1346
- }
1347
- // No `await` between this read and `holdTransition`, or a second caller slips through
1348
- // and builds.
1349
- const previous = this.transitions.get(strandId);
1350
- if (previous?.kind === 'resume') {
1351
- log('resumeStrand: strand %s — joining the resume already in flight', strandId);
1352
- return previous.operation;
1353
- }
1354
- const operation = this.runResume(strandId, previous, overrides);
1355
- this.holdTransition(strandId, { kind: 'resume', operation });
1356
- return operation;
1357
- }
1358
-
1359
- /** Record `transition` as the latest for `strandId` until it settles. */
1360
- private holdTransition(strandId: string, transition: RuntimeTransition): void {
1361
- this.transitions.set(strandId, transition);
1362
- const release = (): void => {
1363
- if (this.transitions.get(strandId) === transition) {
1364
- this.transitions.delete(strandId);
1365
- }
1366
- };
1367
- // The rejection itself reaches the caller through the operation returned to it.
1368
- void transition.operation.then(release, release);
1369
- }
1370
-
1371
- /** Wait for the transition requested before this one. Never rejects: its failure belongs to its caller. */
1372
- private async settleTransition(strandId: string, previous: RuntimeTransition | undefined): Promise<void> {
1373
- if (!previous) {
1374
- return;
1375
- }
1376
- await (previous.operation as Promise<unknown>).catch((error: unknown) => {
1377
- log('Strand %s: the %s queued ahead failed (reported by its caller): %o', strandId, previous.kind, error);
1378
- });
1379
- }
1380
-
1381
- /** Body of {@link resumeStrand}; at most one runs per strand at a time. */
1382
- private async runResume(
1383
- strandId: string,
1384
- previous: RuntimeTransition | undefined,
1385
- overrides?: ResumeStrandOverrides
1386
- ): Promise<StrandInstance> {
1387
- // Before any "already live" test: a quiesce ahead still has the handles attached, and
1388
- // mid-build (a launch still building) the libp2p node is attached while the database is
1389
- // not. A failed launch has already dropped the instance, and the check below reports it
1390
- // untracked.
1391
- await this.settleTransition(strandId, previous);
1392
- await this.settleRuntimeBuilds(strandId);
1393
- const instance = this.instances.get(strandId);
1394
- if (!instance) {
1395
- throw new Error(`Cannot resume strand ${strandId}: not tracked`);
1396
- }
1397
- const launchConfig = this.launchConfigs.get(strandId);
1398
- if (!launchConfig) {
1399
- throw new Error(`Cannot resume strand ${strandId}: no retained launch config`);
1400
- }
1401
- if (instance.libp2pNode || instance.database) {
1402
- log('resumeStrand: strand %s already live', strandId);
1403
- return instance;
1404
- }
1405
-
1406
- log('Resuming strand instance: %s', strandId);
1407
- const tTotal = performance.now();
1408
-
1409
- // Re-apply volatile inputs and persist them so a subsequent resume reuses them.
1410
- // Each `??` matters: a resume that passes no override must keep the retained value,
1411
- // and a resume that passes one must leave it retained for the next resume — otherwise
1412
- // a later no-override wake silently reverts to whatever launch time saw.
1413
- const resumeConfig: StartStrandConfig = {
1414
- ...launchConfig,
1415
- bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes,
1416
- // NOTE: `??` retains, so an override can raise or lower the count but cannot CLEAR it
1417
- // back to "this node no longer knows" — the direction that would declare nothing. Moot
1418
- // while nothing feeds `servingMachines` at all; if a source lands
1419
- // (`backlog/feat-strand-yardstick-from-serving-machines`) that can legitimately lose the
1420
- // count — a strand whose member rows became unreadable — this merge must gain an explicit
1421
- // clear rather than silently declaring a stale number over a serving set it can no longer
1422
- // see. Same applies to `bootstrapNodes` above, where a stale seed is harmless.
1423
- servingMachines: overrides?.servingMachines ?? launchConfig.servingMachines
1424
- };
1425
- this.launchConfigs.set(strandId, resumeConfig);
1426
-
1427
- instance.status = 'starting';
1428
- try {
1429
- await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, resumeConfig));
1430
- timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
1431
- log('Strand %s resumed successfully', strandId);
1432
- return instance;
1433
- } catch (error) {
1434
- instance.status = 'error';
1435
- instance.error = error instanceof Error ? error.message : String(error);
1436
- log('Failed to resume strand %s: %s', strandId, instance.error);
1437
- throw error;
1438
- }
1439
- }
1440
-
1441
- /**
1442
- * Honor a founder request against an ALREADY-TRACKED strand — the seam that
1443
- * closes the "whoever launches first decides whether the bootstrap runs" gap:
1444
- * an instance first launched as a joiner (an app's own attach, or a watcher
1445
- * poll winning the launch race) used to swallow a later founder request
1446
- * silently, leaving the strand active with no `Strand.Header`.
1447
- *
1448
- * Flips the RETAINED launch config's `founder` to true, so every later
1449
- * quiesce → resume rebuild founds as well (the bootstrap is insert-if-absent —
1450
- * {@link StrandDatabase.ensureFounderBootstrap} — so re-running it per rebuild
1451
- * writes nothing twice), and runs the bootstrap against the live database now.
1452
- *
1453
- * @returns how the request resolved:
1454
- * - `'already-founder'` — the retained config already founds; nothing to do.
1455
- * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1456
- * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1457
- * database) or being quiesced (its database is closing), so the bootstrap could
1458
- * not run here: the CALLER must wake the
1459
- * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1460
- * manager does not) so the rebuild — which now founds — runs it.
1461
- * @param resolvePartyKey - Asked for the party's own membership key when (and only
1462
- * when) a CLOSED strand's retained config carries none — the instance was launched
1463
- * as a joiner before its `StrandPartyKey` row existed or replicated. The resolved
1464
- * key is retained alongside the founder flip so the bootstrap (now, or on the
1465
- * caller's wake for `'needs-resume'`) can seat the founding Member/Manager.
1466
- * @throws when the strand is not tracked — this seam exists only for the
1467
- * tracked-instance launch path; an untracked id is a caller bug.
1468
- * @throws whatever the live bootstrap throws (e.g. `PreSplitStrandIdentityError`),
1469
- * after {@link withdrawFounderRequest} — so a retry re-attempts the founding rather
1470
- * than resolving `'already-founder'` over an instance that never founded. A caller
1471
- * whose `'needs-resume'` wake fails owes the same withdrawal.
1472
- */
1473
- async foundExistingStrand(
1474
- strandId: string,
1475
- resolvePartyKey?: () => Promise<string | undefined>
1476
- ): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'> {
1477
- const instance = this.instances.get(strandId);
1478
- const config = this.launchConfigs.get(strandId);
1479
- if (!instance || !config) {
1480
- throw new Error(`Cannot found strand ${strandId}: not tracked`);
1481
- }
1482
- if (config.founder === true) {
1483
- return 'already-founder';
1484
- }
1485
- const partyMemberPrivateKey =
1486
- config.partyMemberPrivateKey
1487
- ?? (config.strandRow.Type === 'c' ? await resolvePartyKey?.() : undefined);
1488
- // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1489
- // config object, which is not ours to rewrite.
1490
- this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1491
- if (this.isQuiescing(strandId) || (!instance.database && !this.firstSyncGates.has(strandId))) {
1492
- return 'needs-resume';
1493
- }
1494
- try {
1495
- await this.ensureFounderBootstrap(strandId);
1496
- } catch (error) {
1497
- this.withdrawFounderRequest(strandId);
1498
- throw error;
1499
- }
1500
- return 'bootstrapped';
1501
- }
1502
-
1503
- /**
1504
- * Give a tracked storage replica ({@link StartStrandConfig.sAppConfig} absent) the app's
1505
- * schema in place — the seam for an app on this machine claiming a strand this node was
1506
- * already hosting. No runtime rebuild: the libp2p node, its peer id and connections, the
1507
- * store and its warm cache all stay, and the new `App` tables read the blocks the replica
1508
- * already holds. Callers attach BEFORE {@link foundExistingStrand}: the founder bootstrap
1509
- * writes the sApp into `Strand.Header` and refuses a database that has none.
1510
- *
1511
- * A runtime build in flight (the replica's launch, or a hibernation wake) is waited out
1512
- * first, so the attach acts on the database that build produced. Then, in order: the
1513
- * schema signature is checked before anything changes; the retained launch config takes
1514
- * the sApp, so every later rebuild applies it through `composeStrand`; the live database —
1515
- * published, or still held by the first-sync gate — gets the schema; and only then does
1516
- * the instance record `sAppInfo` and the sApp's latency hint. A quiesced instance gets the
1517
- * config alone, and its next resume applies the schema.
1518
- *
1519
- * @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
1520
- * is logged and left as it is, as a second claim of a claimed strand always has been —
1521
- * else `'attached'`.
1522
- * @throws when the strand is not tracked (a caller bug, or a launch that failed while
1523
- * this waited on it); when the schema signature is refused, having changed nothing; and
1524
- * whatever the live apply throws (e.g. a quiesce closed the database mid-apply) — the
1525
- * retained config then still carries the sApp and `sAppInfo` stays unset, so the next
1526
- * claim retries the apply.
1527
- *
1528
- * NOTE: two claims of one replica racing each other both pass the `sAppInfo` check and
1529
- * apply concurrently; the loser of the declarative diff can reject, and its `addStrand`
1530
- * retry then resolves `'already-attached'`. Needs an app calling `addStrand` twice at once
1531
- * (or beside a watcher retry); if it is ever seen, chain attaches per strand id.
1532
- */
1533
- async attachSApp(
1534
- strandId: string,
1535
- sAppConfig: SAppConfig,
1536
- options: { requireSignedSchemas?: boolean } = {}
1537
- ): Promise<'attached' | 'already-attached'> {
1538
- await this.settleRuntimeBuilds(strandId);
1539
- const instance = this.instances.get(strandId);
1540
- const config = this.launchConfigs.get(strandId);
1541
- if (!instance || !config) {
1542
- throw new Error(`Cannot attach an sApp to strand ${strandId}: not tracked`);
1543
- }
1544
- if (instance.sAppInfo) {
1545
- if (instance.sAppInfo.id !== sAppConfig.id) {
1546
- log('attachSApp: strand %s already runs sApp %s — the claim for sApp %s is ignored',
1547
- strandId, instance.sAppInfo.id, sAppConfig.id);
1548
- }
1549
- return 'already-attached';
1550
- }
1551
- const sAppInfo = verifiedSAppInfo(strandId, sAppConfig, options.requireSignedSchemas);
1552
- // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1553
- // config object, which is not ours to rewrite.
1554
- this.launchConfigs.set(strandId, { ...config, sAppConfig });
1555
- const database = instance.database ?? this.firstSyncGates.get(strandId)?.database;
1556
- if (database) {
1557
- await database.attachAppSchema(sAppConfig);
1558
- } else {
1559
- log('attachSApp: strand %s is quiesced — its next resume applies the sApp schema', strandId);
1560
- }
1561
- instance.sAppInfo = sAppInfo;
1562
- if (sAppConfig.latencyHint) {
1563
- // HibernationManager reads the hint whenever it arms a timer, so the app's hint governs
1564
- // from the next one: a timer already armed runs once at the old duration, and a replica
1565
- // launched under a realtime default was never tracked, so it stays up until relaunched.
1566
- instance.latencyHint = sAppConfig.latencyHint;
1567
- }
1568
- log('Strand %s upgraded from storage replica to %s', strandId, describeSApp(sAppConfig));
1569
- return 'attached';
1570
- }
1571
-
1572
- /**
1573
- * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1574
- * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
1575
- * the hibernation manager's idle timer fired on it). `false` for a writable, quiesced,
1576
- * or untracked strand. The predicate {@link whenWritable}'s callers gate on.
1577
- */
1578
- isAwaitingFirstSync(strandId: string): boolean {
1579
- const instance = this.instances.get(strandId);
1580
- return instance !== undefined && isAwaitingFirstSync(instance);
1581
- }
1582
-
1583
- /**
1584
- * Resolve once the strand's database is published to the app — immediately for a
1585
- * writable strand — or reject with {@link StrandAwaitingFirstSyncError} after
1586
- * `timeoutMs` (default: the retained launch config's `firstSync.timeoutMs`, else
1587
- * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}). The rejection is RETRYABLE: nothing is
1588
- * torn down, the gate keeps probing, and a later call waits afresh. A quiesced
1589
- * (hibernating) strand is not woken here — the wait spans a resume, so a caller that
1590
- * wants one wakes it (`CadreNode.wakeStrand`) and waits. Rejects immediately for an
1591
- * untracked strand, and whenever the strand is stopped while a wait is pending.
1592
- */
1593
- whenWritable(strandId: string, options?: { timeoutMs?: number }): Promise<StrandInstance> {
1594
- const instance = this.instances.get(strandId);
1595
- if (!instance) {
1596
- return Promise.reject(new Error(`Cannot wait for strand ${strandId} to become writable: not tracked`));
1597
- }
1598
- if (instance.database) {
1599
- return Promise.resolve(instance);
1600
- }
1601
- const timeoutMs = options?.timeoutMs
1602
- ?? this.launchConfigs.get(strandId)?.firstSync?.timeoutMs
1603
- ?? DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS;
1604
- const startedAt = Date.now();
1605
- return new Promise<StrandInstance>((resolve, reject) => {
1606
- const waiters = this.writableWaiters.get(strandId) ?? new Set<WritableWaiter>();
1607
- this.writableWaiters.set(strandId, waiters);
1608
- const settle = (): void => {
1609
- clearTimeout(timer);
1610
- waiters.delete(waiter);
1611
- if (waiters.size === 0) {
1612
- this.writableWaiters.delete(strandId);
1613
- }
1614
- };
1615
- const waiter: WritableWaiter = {
1616
- resolve: () => { settle(); resolve(instance); },
1617
- reject: (error) => { settle(); reject(error); }
1618
- };
1619
- const timer = setTimeout(
1620
- () => waiter.reject(new StrandAwaitingFirstSyncError(strandId, Date.now() - startedAt)),
1621
- timeoutMs);
1622
- (timer as { unref?: () => void }).unref?.();
1623
- waiters.add(waiter);
1624
- });
1625
- }
1626
-
1627
- /**
1628
- * The tracked instance once no runtime build of it is in flight, or `undefined` when none
1629
- * is tracked by then — never launched, or the launch waited on failed and dropped its
1630
- * record. For a caller about to hand a tracked instance to an app: until its build settles
1631
- * the instance is `'starting'` with no database, and a failed launch leaves no instance.
1632
- * Never rejects; a failed build is reported by the call that started it.
1633
- */
1634
- async whenRuntimeBuilt(strandId: string): Promise<StrandInstance | undefined> {
1635
- await this.settleRuntimeBuilds(strandId);
1636
- return this.instances.get(strandId);
1637
- }
1638
-
1639
- /**
1640
- * Hand a launch's database to the app: set `instance.database`, retire the gate that
1641
- * held it, release every {@link whenWritable} waiter, and — when the launch had already
1642
- * been reported gated — flip it `'active'` and announce it through the retained config's
1643
- * `onWritable`. "Reported gated" is any status but `'starting'`: `'syncing'`, or the
1644
- * `'idle'` the hibernation manager's idle timer flips a long-gated joiner to (see
1645
- * {@link isAwaitingFirstSync}) — the Header arriving after that flip must announce too,
1646
- * or an app hanging on `strand:writable` never learns the strand opened. A publish that
1647
- * lands while the instance is still `'starting'` (the Header arrived during the rest of
1648
- * bring-up) announces nothing: `startStrand` / `resumeStrand` are about to report the
1649
- * strand `'active'`, and `strand:writable` is defined as the follow-up to a gated
1650
- * launch, never a duplicate of `strand:started`.
1651
- *
1652
- * It is also the one seam where a WITHHELD database becomes available, so it kicks the
1653
- * membership reconciler: on a gated launch that loop's own immediate pass ran while
1654
- * `instance.database` was still unset, found no database, and would otherwise not try
1655
- * again until its retry timer fired. A launch that was never gated publishes here BEFORE
1656
- * the reconciler is constructed, so there is nothing registered to kick and the loop's own
1657
- * immediate pass already sees the database — which is why the kick needs no `wasGated`
1658
- * test of its own.
1659
- */
1660
- private publishDatabase(instance: StrandInstance, database: StrandDatabase): void {
1661
- const strandId = instance.strandId;
1662
- const gate = this.firstSyncGates.get(strandId);
1663
- if (gate) {
1664
- gate.open();
1665
- this.firstSyncGates.delete(strandId);
1666
- }
1667
- instance.database = database;
1668
- const wasGated = instance.status !== 'starting';
1669
- if (wasGated) {
1670
- // A Header delivered by a peer is strand activity; a publish during bring-up is not
1671
- // (see the build's end in `buildStrandRuntime`).
1672
- instance.lastActivity = new Date();
1673
- instance.status = 'active';
1674
- log('Strand %s is now writable: Strand.Header held', strandId);
1675
- }
1676
- const waiters = this.writableWaiters.get(strandId);
1677
- if (waiters) {
1678
- this.writableWaiters.delete(strandId);
1679
- waiters.forEach((waiter) => waiter.resolve());
1680
- }
1681
- if (wasGated) {
1682
- this.launchConfigs.get(strandId)?.onWritable?.(strandId);
1683
- }
1684
- // Not awaited: publishing must stay synchronous, and the pass chains on the loop's own
1685
- // tail, so it cannot overlap a pass already in flight.
1686
- void this.membershipReconcilers.get(strandId)?.reconcile();
1687
- }
1688
-
1689
- /** Reject every {@link whenWritable} waiter for a strand that is going away. */
1690
- private rejectWritableWaiters(strandId: string, reason: string): void {
1691
- const waiters = this.writableWaiters.get(strandId);
1692
- if (!waiters) {
1693
- return;
1694
- }
1695
- this.writableWaiters.delete(strandId);
1696
- waiters.forEach((waiter) => waiter.reject(new Error(`Strand ${strandId} ${reason} before becoming writable`)));
1697
- }
1698
-
1699
- /**
1700
- * Undo {@link foundExistingStrand}'s founder flip after the founding it promised
1701
- * failed, so the retained config again says what this instance actually runs as (a
1702
- * joiner) and the next founder request re-attempts the bootstrap. The resolved party
1703
- * key stays retained — it is this party's identity for the strand either way. No-op
1704
- * when the strand is untracked or its config does not found.
1705
- */
1706
- withdrawFounderRequest(strandId: string): void {
1707
- const config = this.launchConfigs.get(strandId);
1708
- if (config?.founder === true) {
1709
- this.launchConfigs.set(strandId, { ...config, founder: false });
1710
- }
1711
- }
1712
-
1713
- /**
1714
- * Run the (idempotent) founder bootstrap against a tracked strand's LIVE
1715
- * database, independently of what the retained launch config says.
1716
- *
1717
- * Separate from {@link foundExistingStrand} because a caller that resolved
1718
- * `'needs-resume'` and woke the strand must not assume the wake's own rebuild
1719
- * founded it: `HibernationManager` COALESCES wakes, so a wake already in flight
1720
- * when the config flipped had already read the PRE-flip config and rebuilt as a
1721
- * joiner. Re-running the bootstrap costs one insert-if-absent probe per table
1722
- * and is the only thing that makes "founding resolves once the Header is
1723
- * written" true on that path.
1724
- *
1725
- * A strand still `'syncing'` (its database withheld behind the first-sync gate) is
1726
- * founded through the gate: the bootstrap writes the Header this machine was waiting
1727
- * to receive, so a successful run publishes the database and the strand goes
1728
- * `'active'` — the one legitimate way a joiner launch becomes writable without a peer.
1729
- *
1730
- * @throws when the strand is not tracked, or is still quiesced (no live
1731
- * database) — both mean the bootstrap did NOT run, which a founder request
1732
- * must never swallow.
1733
- */
1734
- async ensureFounderBootstrap(strandId: string): Promise<void> {
1735
- const instance = this.instances.get(strandId);
1736
- if (!instance) {
1737
- throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: not tracked`);
1738
- }
1739
- const gate = this.firstSyncGates.get(strandId);
1740
- const database = instance.database ?? gate?.database;
1741
- if (!database) {
1742
- throw new Error(
1743
- `Cannot run the founder bootstrap for strand ${strandId}: it is quiesced, so there ` +
1744
- 'is no live database to write to — resume it first.'
1745
- );
1746
- }
1747
- // Forward the RETAINED config's party key: a foundExistingStrand that flipped a
1748
- // joiner launch to founder may have resolved a key the live database's captured
1749
- // config (built at the original launch) never saw.
1750
- await database.ensureFounderBootstrap(this.launchConfigs.get(strandId)?.partyMemberPrivateKey);
1751
- if (!instance.database) {
1752
- this.publishDatabase(instance, database);
1753
- }
1754
- }
1755
-
1756
- /**
1757
- * Stop a strand instance
1758
- */
1759
- async stopStrand(strandId: string): Promise<void> {
1760
- const instance = this.instances.get(strandId);
1761
- if (!instance) {
1762
- log('Strand %s not found', strandId);
1763
- return;
1764
- }
1765
-
1766
- log('Stopping strand instance: %s', strandId);
1767
- instance.status = 'stopping';
1768
- this.rejectWritableWaiters(strandId, 'was stopped');
1769
-
1770
- try {
1771
- await this.releaseRuntime(instance);
1772
- instance.status = 'stopped';
1773
- this.instances.delete(strandId);
1774
- this.launchConfigs.delete(strandId);
1775
- // Storage is released only here — NOT in releaseRuntime, which a quiesce shares.
1776
- await this.disposeStrandStorage(strandId);
1777
- log('Strand %s stopped successfully', strandId);
1778
- } catch (error) {
1779
- instance.status = 'error';
1780
- instance.error = error instanceof Error ? error.message : String(error);
1781
- log('Error stopping strand %s: %s', strandId, instance.error);
1782
- throw error;
1783
- }
1784
- }
1785
-
1786
- /**
1787
- * Stop all strand instances
1788
- */
1789
- async stopAll(): Promise<void> {
1790
- this.stopping = true;
1791
- log('Stopping all strand instances (%d)', this.instances.size);
1792
-
1793
- const stopPromises = Array.from(this.instances.keys()).map(id =>
1794
- this.stopStrand(id).catch(err => {
1795
- log('Error stopping strand %s during shutdown: %s', id, err);
1796
- })
1797
- );
1798
-
1799
- await Promise.all(stopPromises);
1800
- this.stopping = false;
1801
- log('All strand instances stopped');
1802
- }
1803
- }
1804
-
1
+ import debug from 'debug';
2
+ import type { PrivateKey } from '@libp2p/interface';
3
+ import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
4
+ import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
5
+ import { StrandDatabase } from './strand-database.js';
6
+ import { PeerJoinBackfill, type PeerJoinBackfillConfig } from './peer-join-backfill.js';
7
+ import { strandNetworkStatePersistence, type StrandNetworkStateStore } from './strand-network-state.js';
8
+ import {
9
+ StrandRevocationEnforcer,
10
+ createRevocationConnectionGater,
11
+ readStrandRevocationRows,
12
+ type StrandRevocationEnforcementConfig
13
+ } from './strand-revocation-enforcer.js';
14
+ import {
15
+ StrandMembershipReconciler,
16
+ type PendingMembershipInviteSource,
17
+ type StrandMembershipReconciliationConfig
18
+ } from './strand-membership-reconciler.js';
19
+ import { removeMemberPeer } from './strand-membership-writer.js';
20
+ import { strandMemberKeyPair } from './strand-member-key.js';
21
+ import { assertSchemaSignature } from './schema-verification.js';
22
+ import { assertStrandScopeKey } from './storage-scope.js';
23
+ import {
24
+ StrandFirstSyncGate,
25
+ StrandAwaitingFirstSyncError,
26
+ strandFirstSyncComplete,
27
+ DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS,
28
+ type StrandFirstSyncConfig
29
+ } from './strand-first-sync-gate.js';
30
+ import type {
31
+ StrandInstance,
32
+ StrandRow,
33
+ StorageConfig,
34
+ NetworkConfig,
35
+ LatencyHint,
36
+ NodeProfile,
37
+ SAppConfig,
38
+ SAppInfo,
39
+ RawStorageProvider,
40
+ Libp2pNodeWithRepo
41
+ } from './types.js';
42
+ import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
43
+ import { strandNodeAddrs } from './strand-network-config.js';
44
+ import { resolveRelayServer } from './relay-server.js';
45
+ import { strandCohortTopicOption, type StrandReactivityConfig } from './strand-reactivity.js';
46
+ import { superviseRelayReservation, type RelayReservationSupervisor } from './relay-reservation.js';
47
+ import { isCohortUnreachableRead } from './control-read-retry.js';
48
+ import { declaredCohortReadDeadlineMs, optimysticDialLimits, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
49
+
50
+ const log = debug('sereus:cadre:strand-manager');
51
+
52
+ /**
53
+ * How often a joining launch that could not build its database, because no seed peer had
54
+ * answered, rebuilds its runtime (`StrandInstanceManager.deferUnreachableJoin`). Each attempt
55
+ * starts a libp2p node and dials the seed, so it is spaced well above the first-sync gate's
56
+ * probe cadence.
57
+ */
58
+ export const UNREACHABLE_JOIN_RETRY_MS = 5_000;
59
+ const timing = debug('sereus:cadre:timing');
60
+
61
+ /** One caller blocked in {@link StrandInstanceManager.whenWritable}. */
62
+ interface WritableWaiter {
63
+ resolve: () => void;
64
+ reject: (error: Error) => void;
65
+ }
66
+
67
+ /**
68
+ * Whether a tracked instance is launched but waiting for its first sync: the runtime is
69
+ * up (`libp2pNode` set) while the database is still withheld (`database` unset). That
70
+ * is the first-sync gate's state regardless of what `status` says — the hibernation
71
+ * manager's idle timer can flip a long-gated joiner to `'idle'` without changing it.
72
+ *
73
+ * NOTE: `status` is therefore not the source of truth for "gated": a joiner that reaches
74
+ * nobody for `idleTimeout` (5 min at the interactive hint) reads `'idle'`, then
75
+ * `'hibernating'`, while still holding no database. Harmless today — every decision the
76
+ * runtime makes goes through this predicate, {@link StrandInstanceManager.publishDatabase}
77
+ * announces on the idle-flipped instance exactly as on a `'syncing'` one, and an app that
78
+ * shows "waiting for the other member" on `'syncing'` merely sees "idle" instead after
79
+ * five minutes. If an app ever needs the gated LABEL to survive the idle flip, make the
80
+ * hibernation manager's idle transition preserve `'syncing'` rather than adding a flag.
81
+ */
82
+ export function isAwaitingFirstSync(instance: StrandInstance): boolean {
83
+ return instance.libp2pNode !== undefined && instance.database === undefined;
84
+ }
85
+
86
+ /**
87
+ * The status a LIVE instance (runtime rebuilt or woken) should report: `'active'` once its
88
+ * database is published, `'syncing'` while the first-sync gate still withholds it. Every
89
+ * site that would otherwise write `'active'` after a build or a wake goes through this,
90
+ * so a gated joiner never reads as writable.
91
+ */
92
+ export function liveStrandStatus(instance: StrandInstance): 'active' | 'syncing' {
93
+ return instance.database ? 'active' : 'syncing';
94
+ }
95
+
96
+ /**
97
+ * Configuration for starting a strand instance
98
+ */
99
+ export interface StartStrandConfig {
100
+ strandRow: StrandRow;
101
+ /**
102
+ * sApp configuration provided by the hosting application. Absent for a **storage
103
+ * replica** — a strand this node stores and serves without the app installed
104
+ * (`CadreNodeConfig.hostUnclaimedStrands`): no schema signature check, no `App`
105
+ * schema, no `sAppInfo` on the instance, and the default latency hint. A replica is
106
+ * always a joiner; `StrandDatabase` refuses to found one. An app that later claims the
107
+ * strand upgrades the running replica in place ({@link StrandInstanceManager.attachSApp}).
108
+ */
109
+ sAppConfig?: SAppConfig;
110
+ storage?: StorageConfig;
111
+ network?: NetworkConfig;
112
+ profile: NodeProfile;
113
+ defaultLatencyHint: LatencyHint;
114
+ privateKey?: PrivateKey;
115
+ /** Cohort-derived discovery seed (multiaddr strings). Defaults to [] when omitted. */
116
+ bootstrapNodes?: string[];
117
+ /**
118
+ * Require a valid author signature on the sApp schema before bring-up.
119
+ * Defaults to true (fail closed) when omitted; set false only for dev/test
120
+ * with unsigned demo schemas. Mirrors {@link CadreNodeConfig.requireSignedSchemas}.
121
+ */
122
+ requireSignedSchemas?: boolean;
123
+ /**
124
+ * Whether this node founds the strand (vs. joins it). Forwarded to the
125
+ * StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
126
+ * runs once at bring-up. Joiners leave this unset and write nothing. Callers
127
+ * resolve it BEFORE calling `startStrand` (`CadreNode.launchStrand` derives it
128
+ * from the row's `FounderOwnerKey` when no explicit flag is given — see
129
+ * {@link StrandConfig.founder}); a founder request that arrives while the
130
+ * instance is already tracked goes through {@link StrandInstanceManager.foundExistingStrand},
131
+ * never through a repeat `startStrand`.
132
+ */
133
+ founder?: boolean;
134
+ /**
135
+ * THIS party's own strand membership private key (base64 protobuf) for a closed
136
+ * strand — the identity the founder bootstrap derives `Member.Key` /
137
+ * `Manager.MemberKey` from. Resolved by the caller (`CadreNode.launchStrand`: an
138
+ * explicit attach-time key, else the control-layer `StrandPartyKey` row, healed by
139
+ * mint-on-launch for the founding machine) and retained with the launch config, so a
140
+ * hibernation wake rebuilds under the same identity. Deliberately NOT the strand
141
+ * row's shared `MemberPrivateKey` (see `StrandDatabaseConfig.partyMemberPrivateKey`).
142
+ * Absent for open strands, and for joiners with no persisted party key.
143
+ */
144
+ partyMemberPrivateKey?: string;
145
+ /**
146
+ * Number of nodes Optimystic is told this strand's replication cluster should
147
+ * have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
148
+ * forwards here: every node on the strand should use the same value, and it is
149
+ * frozen when the strand's libp2p node is created. Defaults to
150
+ * `DEFAULT_STRAND_CLUSTER_SIZE` (4); values below `MIN_CLUSTER_SIZE` (2) are
151
+ * rejected by `resolveStrandClusterSize`. Strand-only — the
152
+ * control network's breadth is the fixed `CONTROL_REPLICATION_BREADTH`.
153
+ */
154
+ clusterSize?: number;
155
+ /**
156
+ * Machines that SERVE THIS STRAND (this node included), for the strand node's
157
+ * block-repair corroboration yardstick — see `resolveRepairYardstick`. It must be
158
+ * an authenticated per-strand count: the machines that actually run a node for this
159
+ * strand, not the machines that exist.
160
+ *
161
+ * **The party's enrolled-machine count is NOT this number and must never be passed
162
+ * here.** A strand launches only on machines whose embedding app registered its sApp
163
+ * config (`CadreNode.addStrand`), so a closed strand shared by two machines of a
164
+ * three-machine party is served by two. Passing three over-declares, and
165
+ * over-declaring is the unsafe direction: at a declaration of 3 or more Optimystic
166
+ * pins the repair corroboration floor at two corroborating peers, which a cohort
167
+ * that can only ever field one peer can never reach — so the strand can never repair
168
+ * a block (`cluster-fetch:no-quorum`, surfacing as reads failing with `Missing
169
+ * block`). That regression is why this field was renamed off "enrolledMachines":
170
+ * `bug-strand-yardstick-counts-party-machines`.
171
+ *
172
+ * **Nothing feeds it in production today.** No authenticated per-strand serving count
173
+ * exists yet, so `CadreNode` passes nothing and the strand node runs the frozen
174
+ * `STRAND_CLUSTER_POLICY` — declaring no yardstick, which leaves the known,
175
+ * upstream-tracked single-voter exposure
176
+ * (`backlog/debt-read-repair-single-voter-corroboration`). Building the count is
177
+ * `backlog/feat-strand-yardstick-from-serving-machines`; this field and its
178
+ * threading are the seam it plugs into.
179
+ *
180
+ * Volatile when a source does exist: re-resolve it on every resume beside the cohort
181
+ * seed, since machines join and leave a strand while it hibernates. Omitting it means
182
+ * "this node does not know", which declares nothing.
183
+ */
184
+ servingMachines?: number;
185
+ /**
186
+ * Tuning for the strand peer-join block catch-up ({@link PeerJoinBackfill}),
187
+ * forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
188
+ * libp2p node connects to a peer this runtime has not yet caught up, every
189
+ * block in the strand's own raw store is pushed to it. Runs only on strands
190
+ * with per-strand storage; `{ enabled: false }` restores the pre-existing
191
+ * no-backfill behaviour.
192
+ */
193
+ backfill?: PeerJoinBackfillConfig;
194
+ /**
195
+ * Whether this strand's libp2p node is built with Optimystic's change notifications,
196
+ * forwarded from {@link CadreNodeConfig.strandReactivity} and resolved per strand by
197
+ * `strandCohortTopicOption`. Retained with the launch config, so a hibernation wake
198
+ * rebuilds the node with the same setting.
199
+ */
200
+ reactivity?: StrandReactivityConfig;
201
+ /**
202
+ * Tuning for the CLOSED-strand revoked-peer gate
203
+ * ({@link StrandRevocationEnforcer}), forwarded from
204
+ * {@link CadreNodeConfig.strandRevocationEnforcement}. Armed only for
205
+ * `strandRow.Type === 'c'` (an open strand has no membership rows to derive a
206
+ * deny set from); `{ enabled: false }` restores the pre-existing behaviour
207
+ * (a removed party's peers keep being served).
208
+ */
209
+ revocationEnforcement?: StrandRevocationEnforcementConfig;
210
+
211
+ /**
212
+ * The staged formation membership invitation for THIS strand — the seam the
213
+ * bring-up membership reconciler (`strand-membership-reconciler.ts`) reads to
214
+ * redeem the party's `Strand.Member` seat, and clears once the invitation is
215
+ * spent, burned, or dead. `CadreNode.launchStrand` wires it over its in-memory
216
+ * `pendingMembershipInvites` map; read lazily per pass, so a re-formation that
217
+ * replaces the entry between passes is picked up. Only consulted for closed
218
+ * strands launched with a {@link partyMemberPrivateKey}; harmless otherwise.
219
+ */
220
+ pendingMembershipInvite?: PendingMembershipInviteSource;
221
+
222
+ /**
223
+ * Tuning for the CLOSED-strand membership reconciler, forwarded from
224
+ * {@link CadreNodeConfig.strandMembershipReconciliation}. Armed only for
225
+ * closed strands launched with a {@link partyMemberPrivateKey};
226
+ * `{ enabled: false }` disarms the loop (test fixtures that hand-drive the
227
+ * membership writers). When it names no `pollIntervalMs` the reconciler
228
+ * mirrors {@link revocationEnforcement}'s cadence — as its IDLE cadence and
229
+ * the cap of its unfinished-join retry ladder, not as a flat retry interval.
230
+ */
231
+ membershipReconciliation?: StrandMembershipReconciliationConfig;
232
+
233
+ /**
234
+ * Called when THIS node's own peer id turns up in the strand's revoked set —
235
+ * this node's party was removed from the closed strand, or left it. Fires once
236
+ * per removal (re-armed if the party is re-admitted; a resume also rebuilds the
237
+ * enforcer and may legitimately re-fire), and only for closed strands with the
238
+ * gate armed. `CadreNode`
239
+ * wires it to its `strand:revoked` event; nothing is stopped or torn down on
240
+ * this node's behalf — what to do about it is the app's call.
241
+ */
242
+ onSelfRevoked?: (strandId: string) => void;
243
+
244
+ /**
245
+ * Called when the membership reconciler reports a blocked re-join — this node holds a
246
+ * staged membership invitation for the strand that its writes cannot redeem, which is
247
+ * what a REMOVED party handed a fresh invitation hits (see "Reporting a blocked
248
+ * re-join" in `strand-membership-reconciler.ts` for the two triggers, one of them a
249
+ * suspicion rather than a verdict). At most once per re-arm of the loop — a further
250
+ * invitation staged while the loop is still running does not reset the report — and
251
+ * only for closed strands with a party key. `CadreNode` wires it to its `strand:rejoin-blocked`
252
+ * event; nothing is stopped or torn down — a remaining manager admitting this party's
253
+ * key directly is the remedy, and the loop finishes the join by itself once that lands.
254
+ */
255
+ onRejoinBlocked?: (strandId: string) => void;
256
+
257
+ /**
258
+ * Tuning for the JOINING machine's first-sync write gate (`strand-first-sync-gate.ts`),
259
+ * forwarded from {@link CadreNodeConfig.strandFirstSync}: the Header probe cadence while
260
+ * a non-founder launch is `'syncing'`, and the default budget {@link whenWritable} waits
261
+ * before rejecting. Founders and machines that already hold the Header are never gated.
262
+ */
263
+ firstSync?: StrandFirstSyncConfig;
264
+
265
+ /**
266
+ * Called when a launch that came up `'syncing'` (see {@link firstSync}) becomes
267
+ * writable: the strand's `Strand.Header` arrived from a peer — or a founder request
268
+ * ran the bootstrap against the gated database — and `instance.database` is now set.
269
+ * Never called for a launch that was writable when `startStrand`/`resumeStrand`
270
+ * resolved. `CadreNode` wires it to its `strand:writable` event. Retained with the
271
+ * launch config, so a hibernation resume that comes up gated announces too.
272
+ */
273
+ onWritable?: (strandId: string) => void;
274
+
275
+ /**
276
+ * Re-announce this strand's delegate peer id (`delegatePeerId` — the strand
277
+ * node's own transport peer id) to ONE relay (`relayAddr`, a direct dial addr
278
+ * with a trailing `/p2p/<relayPeerId>`), unthrottled. Awaited before every
279
+ * relay-reservation RE-drive of that relay — never before the first attempt,
280
+ * which the launch/resume seed pass has already announced for.
281
+ *
282
+ * Why: a party control node running the relay server admits the strand node on
283
+ * an in-memory delegate grant that a relay restart drops, so a re-drive that did
284
+ * not re-announce first would be denied at the relay's gate. `CadreNode` wires
285
+ * its own control-mesh announce here; a throw or rejection is logged by the
286
+ * supervisor and the re-drive still runs. Retained with the launch config, so a
287
+ * hibernation wake's rebuilt supervisors carry it too.
288
+ */
289
+ announceDelegateToRelay?: (strandId: string, relayAddr: string, delegatePeerId: string) => Promise<void>;
290
+
291
+ /**
292
+ * The node-local strand network state (`strand-network-state.ts`): this strand's
293
+ * slice of it is handed to db-p2p as `persistence`, so the strand node saves its
294
+ * FRET routing table (with each peer's signed address record) on every connection
295
+ * and re-imports it when it is next built. Absent ⇒ nothing is saved and the node
296
+ * starts with an empty table. `CadreNode` passes its store. Retained with the launch
297
+ * config, so a hibernation wake rebuilds the node over the same state.
298
+ */
299
+ networkState?: StrandNetworkStateStore;
300
+ }
301
+
302
+ /**
303
+ * Volatile inputs re-resolved when resuming a quiesced strand — the ones that can
304
+ * have moved since it last ran, and that the rebuilt libp2p node freezes again. Each
305
+ * one omitted keeps the value the retained launch config already holds, so a resume
306
+ * that passes nothing rebuilds the strand exactly as it last ran.
307
+ */
308
+ export interface ResumeStrandOverrides {
309
+ /**
310
+ * Freshly-resolved cohort discovery seed (multiaddr strings). Grows as peers are
311
+ * learned since the strand first launched.
312
+ */
313
+ bootstrapNodes?: string[];
314
+ /**
315
+ * Freshly-read count of the machines serving this strand, for the repair yardstick
316
+ * (see {@link StartStrandConfig.servingMachines} — including why the party's
317
+ * enrolled-machine count is not it). Moves whenever a machine starts or stops
318
+ * serving the strand, which a hibernating strand does not otherwise notice. Nothing
319
+ * passes it today. Omitting it here does NOT mean "unknown" the way omitting it from
320
+ * {@link StartStrandConfig} does — it retains the last value; see `resumeStrand`.
321
+ */
322
+ servingMachines?: number;
323
+ }
324
+
325
+ /** A {@link StrandInstanceManager.resumeStrand} or {@link StrandInstanceManager.quiesceStrand} of one strand. */
326
+ type RuntimeTransition =
327
+ | { kind: 'resume'; operation: Promise<StrandInstance> }
328
+ | { kind: 'quiesce'; operation: Promise<void> };
329
+
330
+ /**
331
+ * Get the isolated storage path for a specific strand.
332
+ *
333
+ * @deprecated This helper is Node-only and throws in React Native (it assumes a
334
+ * filesystem layout). Use a storage provider factory function instead, which
335
+ * receives the strandId and can create strand-specific storage paths using
336
+ * platform-appropriate methods.
337
+ *
338
+ * @example
339
+ * // Instead of using getStrandStoragePath, use a storage provider factory:
340
+ * const storage = {
341
+ * provider: (strandId: string) => new FileRawStorage(`./data/strands/${strandId}`)
342
+ * };
343
+ */
344
+ export function getStrandStoragePath(basePath: string, strandId: string): string {
345
+ // Check if we're in a Node.js environment
346
+ if (typeof process === 'undefined' || !process.versions?.node) {
347
+ throw new Error(
348
+ 'getStrandStoragePath is not available in React Native. ' +
349
+ 'Use a storage provider factory function instead.'
350
+ );
351
+ }
352
+
353
+ // Sanitize strandId for filesystem safety (UUIDs should be safe, but just in case)
354
+ const safeId = strandId.replace(/[^a-zA-Z0-9-]/g, '_');
355
+
356
+ // Build the path with plain string joins rather than the Node `path` module.
357
+ // A static `require('path')` forces RN bundlers (e.g. Metro) to *resolve* the
358
+ // module at bundle time even though this Node-only helper throws above before
359
+ // ever reaching here — joining by hand keeps the module free of any Node
360
+ // built-in reference, so RN bundles need no `path` shim.
361
+ const trimmedBase = basePath.replace(/[\\/]+$/, '');
362
+ return `${trimmedBase}/strands/${safeId}`;
363
+ }
364
+
365
+ /**
366
+ * Resolve a storage provider for a specific strand.
367
+ * If the provider is a factory function, call it with the strandId.
368
+ *
369
+ * Called only from {@link StrandInstanceManager.startStrand}, which owns the result
370
+ * for the instance's lifetime — see `strandStorages`, and which has already asserted
371
+ * the id is usable as a scope key. Do not move that assertion here: this function
372
+ * returns early when no provider is configured, and the id becomes a libp2p protocol
373
+ * prefix on that path too.
374
+ *
375
+ * @param provider - Storage provider (instance or factory)
376
+ * @param strandId - The strand ID to create storage for
377
+ * @returns The resolved IRawStorage instance, or undefined if no provider
378
+ */
379
+ function resolveStrandStorage(
380
+ provider: RawStorageProvider | undefined,
381
+ strandId: string
382
+ ): IRawStorage | undefined {
383
+ if (!provider) {
384
+ return undefined;
385
+ }
386
+ const storage = typeof provider === 'function' ? provider(strandId) : provider;
387
+ // Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's cached-storage.ts).
388
+ // Called ONCE per strand launch — `startStrand` keeps the result for the instance's
389
+ // lifetime — so the wrap survives every runtime rebuild the instance goes through.
390
+ return wrapStorageWithCache(storage, strandId);
391
+ }
392
+
393
+ /**
394
+ * Verify an sApp's schema signature (fail-closed unless `requireSignedSchemas` is
395
+ * false) and project the config onto the instance's `sAppInfo`.
396
+ */
397
+ function verifiedSAppInfo(strandId: string, sAppConfig: SAppConfig, requireSignedSchemas: boolean | undefined): SAppInfo {
398
+ assertSchemaSignature(sAppConfig, { requireSignature: requireSignedSchemas ?? true });
399
+ log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
400
+ return {
401
+ id: sAppConfig.id,
402
+ version: sAppConfig.version,
403
+ schema: sAppConfig.schema,
404
+ signature: sAppConfig.signature
405
+ };
406
+ }
407
+
408
+ /** Log wording for what a launch runs: the sApp and its version, or a storage replica. */
409
+ function describeSApp(sAppConfig: SAppConfig | undefined): string {
410
+ return sAppConfig ? `sApp: ${sAppConfig.id} v${sAppConfig.version}` : 'storage replica';
411
+ }
412
+
413
+ /**
414
+ * Manages individual strand instances - creates and destroys isolated libp2p nodes
415
+ * for each strand the cadre participates in.
416
+ */
417
+ export class StrandInstanceManager {
418
+ private instances: Map<string, StrandInstance> = new Map();
419
+ /**
420
+ * Retained launch config per strand, captured in `startStrand` and cleared in
421
+ * `stopStrand`. `resumeStrand` reuses it to rebuild a quiesced strand's runtime
422
+ * without the caller re-threading storage/network/profile/key/sApp config.
423
+ */
424
+ private launchConfigs: Map<string, StartStrandConfig> = new Map();
425
+ /**
426
+ * The per-strand peer-join block catch-up, keyed by strand id. Private (not on
427
+ * the public {@link StrandInstance}) because it is runtime plumbing with the
428
+ * same lifetime as the strand's libp2p node: created in `buildStrandRuntime`,
429
+ * stopped and dropped in `releaseRuntime` — so quiesce → resume rebuilds it
430
+ * with a fresh caught-up-peer memo, which is intended (a resumed node may have
431
+ * missed writes).
432
+ */
433
+ private backfills: Map<string, PeerJoinBackfill> = new Map();
434
+ /**
435
+ * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
436
+ * id. Same lifecycle rationale as {@link backfills}: created in
437
+ * `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so quiesce →
438
+ * resume rebuilds it with a fresh (initially empty, fail-open) snapshot.
439
+ */
440
+ private revocationEnforcers: Map<string, StrandRevocationEnforcer> = new Map();
441
+ /**
442
+ * The per-strand membership reconciler (closed strands launched with a party
443
+ * key only), keyed by strand id. Same lifecycle rationale as {@link backfills}:
444
+ * created in `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so
445
+ * quiesce → resume rebuilds it and re-runs the idempotent join ladder (redeem a
446
+ * staged invitation, write this machine's own `MemberPeer` binding) from scratch.
447
+ */
448
+ private membershipReconcilers: Map<string, StrandMembershipReconciler> = new Map();
449
+ /**
450
+ * The per-strand first-sync gate (`strand-first-sync-gate.ts`), keyed by strand id:
451
+ * holds a launch's `StrandDatabase` while it is NOT yet published on the instance.
452
+ * Present from the moment the database object exists in `buildStrandRuntime` until
453
+ * {@link publishDatabase} hands it to the app — so it doubles as the rollback handle
454
+ * for a failed `initialize()` — and, for a joiner that holds no `Strand.Header` yet,
455
+ * for as long as the launch stays `'syncing'`. Same lifecycle as {@link backfills}:
456
+ * stopped (its database closed) and dropped in `releaseRuntime`, so a quiesce →
457
+ * resume rebuild re-probes over the same store. An entry and `instance.database`
458
+ * are mutually exclusive.
459
+ */
460
+ private firstSyncGates: Map<string, StrandFirstSyncGate> = new Map();
461
+ /**
462
+ * Callers blocked in {@link whenWritable}, keyed by strand id. Resolved by
463
+ * {@link publishDatabase}, rejected by {@link stopStrand}; each waiter also carries
464
+ * its own timeout. Deliberately NOT cleared by `releaseRuntime`: a waiter outlives a
465
+ * quiesce → resume cycle, since the rebuild is what may finally publish the database.
466
+ */
467
+ private writableWaiters: Map<string, Set<WritableWaiter>> = new Map();
468
+ /**
469
+ * The per-strand relay-reservation supervisors — ONE PER CONFIGURED RELAY, each
470
+ * over the node's own bare `/p2p-circuit` listener for that relay
471
+ * (`strand-network-config.ts`) — keyed by strand id. Same lifecycle rationale as
472
+ * {@link backfills}: started in `buildStrandRuntime` right after the libp2p node
473
+ * exists, stopped FIRST and dropped in `releaseRuntime` (before the database
474
+ * closes and the node stops, so no re-drive dials a node being torn down), so
475
+ * quiesce → resume rebuilds them over the new node. What makes a strand node's
476
+ * lost relay slot come back on its own — the control node has the same loop in
477
+ * `CadreNode.reserveRelays`.
478
+ */
479
+ private relaySupervisors: Map<string, RelayReservationSupervisor[]> = new Map();
480
+ /**
481
+ * The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
482
+ * store, resolved once in `startStrand` and held until `stopStrand` disposes it.
483
+ * Private for the same reason `backfills` is: runtime plumbing, not part of the
484
+ * public {@link StrandInstance}.
485
+ *
486
+ * Deliberately NOT touched by `releaseRuntime`: that is what lets a quiesce →
487
+ * resume cycle rebuild the libp2p node over the SAME store, keeping its
488
+ * write-through cache warm (and, on an in-memory backend, keeping the strand's
489
+ * blocks at all). An entry exists iff `instances` does AND the launch config
490
+ * supplied a storage provider.
491
+ *
492
+ * NOTE: a hibernating strand therefore keeps its store — and its share of the
493
+ * process-wide cache pool — resident for as long as the instance is tracked. That
494
+ * is the point (a warm wake), and the pool evicts under pressure, so the cost is
495
+ * one map entry per hibernating strand today. If a device ever hibernates strands
496
+ * by the hundred, revisit: dropping the store at quiesce and paying for a cold
497
+ * wake becomes the better trade.
498
+ */
499
+ private strandStorages: Map<string, IRawStorage> = new Map();
500
+ /**
501
+ * The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
502
+ * `resumeStrand`), present until it settles. The instance is tracked for the whole build,
503
+ * with its database not yet constructed or not yet initialized, so a caller that must act
504
+ * on the FINISHED runtime ({@link attachSApp}, {@link whenRuntimeBuilt}) waits on this instead. A failed build's
505
+ * rejection belongs to the call that started it.
506
+ */
507
+ private runtimeBuilds: Map<string, Promise<void>> = new Map();
508
+ /**
509
+ * The latest resume or quiesce requested per strand id, present until it settles. Each one
510
+ * starts only once the one requested before it has settled, so a wake that lands during a
511
+ * quiesce rebuilds after it, and a quiesce requested during a rebuild releases what that
512
+ * rebuild finished. The WHOLE operation, not its entry in {@link runtimeBuilds}: that settles
513
+ * before a resume's own catch records `'error'`, so a joiner awaiting it could read a stale
514
+ * status.
515
+ */
516
+ private transitions: Map<string, RuntimeTransition> = new Map();
517
+ /**
518
+ * The pending rebuild per strand id of a launch whose database could not be built
519
+ * because no seed peer had answered yet ({@link deferUnreachableJoin}). Cleared by
520
+ * {@link releaseRuntime}, so a stop, a quiesce or a resume cancels it.
521
+ */
522
+ private unreachableJoinRetries: Map<string, unknown> = new Map();
523
+ private stopping = false;
524
+
525
+ constructor() {
526
+ log('StrandInstanceManager created');
527
+ }
528
+
529
+ /**
530
+ * Get all current strand instances
531
+ */
532
+ getInstances(): Map<string, StrandInstance> {
533
+ return new Map(this.instances);
534
+ }
535
+
536
+ /**
537
+ * Get a specific strand instance
538
+ */
539
+ getInstance(strandId: string): StrandInstance | undefined {
540
+ return this.instances.get(strandId);
541
+ }
542
+
543
+ /**
544
+ * Check if a strand is currently running
545
+ */
546
+ hasStrand(strandId: string): boolean {
547
+ return this.instances.has(strandId);
548
+ }
549
+
550
+ /**
551
+ * Start a new strand instance.
552
+ *
553
+ * A failed launch leaves NOTHING tracked: the instance and its retained launch
554
+ * config are both dropped before the error is rethrown, so the strand id is
555
+ * free for a genuine retry. This matches the pre-registration failure path
556
+ * (a rejected schema signature, which throws before anything is recorded) —
557
+ * both failure modes of this call leave the same residue: none. Callers learn
558
+ * of the failure from the rejected promise (and, on the control-discovered
559
+ * path, from CadreNode's `strand:error` event), not from an error record left
560
+ * behind in `instances`.
561
+ *
562
+ * The strand's raw storage is resolved HERE, once, and held for the instance's
563
+ * lifetime (see `strandStorages`) — `buildStrandRuntime` only reads it, so a
564
+ * hibernation wake never re-enters the embedder's provider.
565
+ */
566
+ async startStrand(config: StartStrandConfig): Promise<StrandInstance> {
567
+ const { strandRow, sAppConfig } = config;
568
+ const strandId = strandRow.Id;
569
+
570
+ if (this.stopping) {
571
+ throw new Error('StrandInstanceManager is stopping');
572
+ }
573
+
574
+ if (this.instances.has(strandId)) {
575
+ // Callers resolve founder-ness BEFORE reaching here (CadreNode.launchStrand) and
576
+ // honor a founder request on a tracked instance via foundExistingStrand — so a
577
+ // founder flag arriving at this early return against a non-founder retained
578
+ // config is a dropped bootstrap, and must never again be silent.
579
+ if (config.founder === true && this.launchConfigs.get(strandId)?.founder !== true) {
580
+ log('startStrand: strand %s is already running but was NOT launched as founder — ' +
581
+ 'this early return DROPS the founder request; use foundExistingStrand', strandId);
582
+ }
583
+ log('Strand %s already running', strandId);
584
+ return this.instances.get(strandId)!;
585
+ }
586
+
587
+ log('Starting strand instance: %s (%s)', strandId, describeSApp(sAppConfig));
588
+ const tTotal = performance.now();
589
+
590
+ // The id becomes two names below: the storage scope key the embedder's provider
591
+ // turns into a directory or database name, and `networkName` in buildStrandRuntime,
592
+ // from which the libp2p protocol prefix `/optimystic/strand-<id>` is built. A
593
+ // replicated strand row carries whatever id the founding node wrote, so neither is
594
+ // safe unchecked. Asserted HERE rather than in `resolveStrandStorage` because that
595
+ // function returns early when no provider is configured — a node with no storage
596
+ // still reaches buildStrandRuntime and still mints the protocol prefix.
597
+ assertStrandScopeKey(strandId);
598
+
599
+ const sAppInfo = sAppConfig ? verifiedSAppInfo(strandId, sAppConfig, config.requireSignedSchemas) : undefined;
600
+
601
+ // Resolve this strand's storage ONCE, before anything is recorded. If a factory
602
+ // function is provided, it is called with the strandId to create strand-specific
603
+ // storage (e.g. strand-isolated directories). A provider that throws therefore
604
+ // fails the launch alongside the schema-signature check, leaving nothing tracked.
605
+ const strandStorage = resolveStrandStorage(config.storage?.provider, strandId);
606
+ if (strandStorage) {
607
+ log('Strand %s using provided storage provider', strandId);
608
+ }
609
+
610
+ // Determine latency hint: sApp config > default
611
+ const latencyHint = sAppConfig?.latencyHint ?? config.defaultLatencyHint;
612
+
613
+ const instance: StrandInstance = {
614
+ strandId,
615
+ status: 'starting',
616
+ sAppInfo,
617
+ memberPrivateKey: strandRow.MemberPrivateKey ?? undefined,
618
+ connectedPeers: 0,
619
+ lastActivity: new Date(),
620
+ latencyHint
621
+ };
622
+
623
+ this.instances.set(strandId, instance);
624
+ this.launchConfigs.set(strandId, config);
625
+ if (strandStorage) {
626
+ this.strandStorages.set(strandId, strandStorage);
627
+ }
628
+
629
+ try {
630
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, config));
631
+ timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
632
+ log('Strand %s started successfully (%s)', strandId, describeSApp(sAppConfig));
633
+ return instance;
634
+ } catch (error) {
635
+ // Status/error first — the (now discarded) record is still what `log` reports on.
636
+ instance.status = 'error';
637
+ instance.error = error instanceof Error ? error.message : String(error);
638
+ log('Failed to start strand %s: %s', strandId, instance.error);
639
+ // Drop the dead record so this strand id can be launched again. Keep the
640
+ // `launchConfigs` has an entry iff `instances` does invariant — resumeStrand
641
+ // reads both, and a config without an instance would strand the config.
642
+ this.instances.delete(strandId);
643
+ this.launchConfigs.delete(strandId);
644
+ // The store this launch resolved goes with it: nothing owns it any more, and a
645
+ // retained cache wrapper would be handed back (already retired) on a retry.
646
+ await this.disposeStrandStorage(strandId);
647
+ throw error;
648
+ }
649
+ }
650
+
651
+ /** Hold `build` in {@link runtimeBuilds} until it settles, and settle as it does. */
652
+ private async trackRuntimeBuild(strandId: string, build: Promise<void>): Promise<void> {
653
+ this.runtimeBuilds.set(strandId, build);
654
+ try {
655
+ await build;
656
+ } finally {
657
+ if (this.runtimeBuilds.get(strandId) === build) {
658
+ this.runtimeBuilds.delete(strandId);
659
+ }
660
+ }
661
+ }
662
+
663
+ /**
664
+ * Wait until no runtime build of `strandId` is in flight. Never rejects: a failed build
665
+ * is reported by the call that started it, and the caller re-reads the instance after.
666
+ *
667
+ * NOTE: after a failed launch the re-read sees the instance gone only because
668
+ * `startStrand`'s catch deletes it synchronously, and its continuation is queued before
669
+ * this waiter's. If that cleanup ever awaits before `instances.delete`, a waiter can act
670
+ * on the dying record; move the cleanup inside the tracked promise then.
671
+ */
672
+ private async settleRuntimeBuilds(strandId: string): Promise<void> {
673
+ for (let build = this.runtimeBuilds.get(strandId); build; build = this.runtimeBuilds.get(strandId)) {
674
+ await build.catch((error: unknown) => {
675
+ log('Strand %s: the runtime build waited on failed (reported by its launcher): %o', strandId, error);
676
+ });
677
+ }
678
+ }
679
+
680
+ /**
681
+ * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
682
+ * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
683
+ * launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
684
+ * volatile inputs (bootstrapNodes, servingMachines, network, profile,
685
+ * privateKey, sApp config) from `config`, so the caller controls the
686
+ * cohort-derived values. Storage is the
687
+ * one input it does NOT re-read from `config`: that belongs to the instance and
688
+ * comes from `strandStorages`.
689
+ */
690
+ private async buildStrandRuntime(instance: StrandInstance, config: StartStrandConfig): Promise<void> {
691
+ const strandId = instance.strandId;
692
+ const { sAppConfig } = config;
693
+
694
+ // Refuse a bad `linkRoundTripMs` before any strand bring-up, for the reason
695
+ // `CadreNode.start()` does: every budget derived from it here is behind a condition
696
+ // (backfill disabled, no relay addrs), so a zero or NaN declaration would otherwise
697
+ // surface later inside a path that logs and carries on. `link-budget.ts`.
698
+ resolveLinkRoundTripMs(config.network?.linkRoundTripMs);
699
+
700
+ // The store the instance OWNS (resolved once in `startStrand`), not a fresh
701
+ // resolution: a rebuild must reach the same backend through the same warm cache.
702
+ const strandStorage = this.strandStorages.get(strandId);
703
+
704
+ // db-p2p namespaces every one of the node's protocol ids by network name
705
+ // (`/optimystic/<networkName>/...`), so anything dialing this node's own
706
+ // services must derive its prefix from the SAME string the node was built
707
+ // with — hence one binding for both, not two literals that can drift.
708
+ const networkName = `strand-${strandId}`;
709
+ // Set when the database build failed only because no seed peer has answered yet; see
710
+ // {@link deferUnreachableJoin}.
711
+ let joinWaitsForSeed = false;
712
+ const protocolPrefix = `/optimystic/${networkName}`;
713
+
714
+ // Whether this node runs the circuit-relay server, and its init — the same
715
+ // resolution the control node takes from the same config (`relay-server.ts`).
716
+ const relayServer = resolveRelayServer(config.network, config.profile);
717
+ // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
718
+ // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
719
+ // bind one port), the announce config is dropped (it names the control node's
720
+ // address), and every configured relay becomes one bare `/p2p-circuit` SEARCH
721
+ // listener plus a dial addr in `relayAddrs` for the per-relay supervisor started
722
+ // below — the same route the control node takes, because the configured
723
+ // `<relay>/p2p-circuit` shape loses its address on a relay restart, a hangup, or
724
+ // libp2p's own reservation refresh and recovers from none of them. See
725
+ // `strand-network-config.ts` for all three, and `relay-reservation.ts`.
726
+ //
727
+ // `relayAddrs` is destructured OFF: it is not a `createLibp2pNode` option, and
728
+ // spreading it would hand db-p2p a key it does not know.
729
+ const { relayAddrs: relayDialAddrs = [], ...addrOptions } = strandNodeAddrs(config.network);
730
+
731
+ // The CLOSED-strand revoked-peer gate (see strand-revocation-enforcer.ts).
732
+ // Constructed BEFORE the libp2p node because the node's options embed its
733
+ // predicates; its deny set starts empty (admit everything — fail-open, same
734
+ // posture as bring-up everywhere here) and is first populated after the
735
+ // strand database initializes below. Registered in the map immediately so
736
+ // the failure rollback (releaseRuntime) tears it down like every other
737
+ // runtime component. Open strands skip it entirely: their Member/MemberPeer
738
+ // tables are empty by schema (`OnlyClosed`), so there is nothing to derive
739
+ // a deny set from — arming would be pointless polling.
740
+ const revocationEnforcer =
741
+ config.strandRow.Type === 'c' && config.revocationEnforcement?.enabled !== false
742
+ ? new StrandRevocationEnforcer({
743
+ label: strandId,
744
+ readRows: () => {
745
+ const database = instance.database;
746
+ if (!database) {
747
+ throw new Error(`strand ${strandId} has no live database`);
748
+ }
749
+ return readStrandRevocationRows(database.getDatabase());
750
+ },
751
+ // Read per sweep, never captured: this closure is built BEFORE the
752
+ // libp2p node exists (its options embed the enforcer's predicates),
753
+ // and `releaseRuntime` clears the field again on quiesce. Undefined
754
+ // therefore means "no transport right now", which the enforcer
755
+ // treats as "nothing to tear down".
756
+ getNetwork: () => instance.libp2pNode,
757
+ onSelfRevoked: () => config.onSelfRevoked?.(strandId)
758
+ }, config.revocationEnforcement)
759
+ : undefined;
760
+ if (revocationEnforcer) {
761
+ this.revocationEnforcers.set(strandId, revocationEnforcer);
762
+ }
763
+
764
+ // Closed strands compose revoked-peer denial onto the caller's gater and arm
765
+ // the fail-closed per-stream gate; open strands keep the raw configured gater
766
+ // (their peers are legitimately cross-party and nothing is ever revoked).
767
+ const revocationGateOptions = revocationEnforcer
768
+ ? {
769
+ connectionGater: createRevocationConnectionGater(revocationEnforcer, config.network?.connectionGater),
770
+ authorizeInboundStream: (remotePeerId: string, protocol: string) =>
771
+ revocationEnforcer.authorizeStream(remotePeerId, protocol)
772
+ }
773
+ : config.network?.connectionGater
774
+ ? { connectionGater: config.network.connectionGater }
775
+ : {};
776
+
777
+ const reactivityOption = strandCohortTopicOption(config.reactivity, strandId);
778
+ if (reactivityOption.cohortTopic) {
779
+ log('strand %s: change notifications on', strandId);
780
+ }
781
+
782
+ try {
783
+ // Bound once: the breadth is also the ceiling on the repair yardstick below, and the
784
+ // two must be derived from the same resolution. Inside the `try` deliberately — a
785
+ // rejected clusterSize is a build failure that runs the same cleanup as any other.
786
+ const strandClusterSize = resolveStrandClusterSize(config.clusterSize);
787
+
788
+ let t0 = performance.now();
789
+ const node = await createLibp2pNode({
790
+ port: 0, // Random port
791
+ bootstrapNodes: config.bootstrapNodes ?? [],
792
+ networkName,
793
+ storage: strandStorage,
794
+ fretProfile: config.profile === 'storage' ? 'core' : 'edge',
795
+ relay: relayServer.enabled,
796
+ // NOTE: unlike the control node, a strand node has no unauthorized-reservation budget,
797
+ // so with the per-connection cap off any peer that reaches this node can hold one of
798
+ // `maxReservations` slots and forward uncapped through it. If strand relays are ever
799
+ // abused for bandwidth, give strand nodes a budget or a capped init of their own.
800
+ ...(relayServer.enabled && { relayServerInit: relayServer.init }),
801
+ clusterSize: strandClusterSize,
802
+ // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
803
+ // breadth reasoning, and the shape match with the control policy is a coincidence.
804
+ //
805
+ // The builder declares this node's block-repair corroboration yardstick from the
806
+ // count of machines SERVING this strand, capped at the breadth above (a block
807
+ // never lives on more machines than the cohort is wide). Given no count — the
808
+ // production path today, since no per-strand serving count exists yet — it
809
+ // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
810
+ // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
811
+ // serving set that changed while the strand slept.
812
+ // The read deadline is settled by the same helper the control node uses, because the
813
+ // two networks ride one link: the host's own, else derived from its declared
814
+ // `linkRoundTripMs`, else nothing, and the base policy's COHORT_READ_DEADLINE_MS stands.
815
+ clusterPolicy: strandClusterPolicy(strandClusterSize, {
816
+ servingMachines: config.servingMachines,
817
+ cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(config.network)
818
+ }),
819
+ arachnode: {
820
+ enableRingZulu: config.profile === 'storage'
821
+ },
822
+ ...(config.privateKey && { privateKey: config.privateKey }),
823
+ // Built per runtime, not per launch: the adapter stops saving once the strand's
824
+ // state is forgotten, and a rebuilt runtime is what starts it saving again.
825
+ //
826
+ // NOTE: the saved state also restores which peers db-p2p saw serving the strand,
827
+ // and it counts them in the cohort until FRET finds them unreachable. A node built
828
+ // while every other member is offline therefore fails its first writes ("Failed to
829
+ // get super-majority") — for 4-6 s on loopback, the window a running node already
830
+ // sees when a member drops. If apps report failed sends after relaunching alone on
831
+ // a slow relayed link, retry the write in the app or raise it upstream (db-p2p's
832
+ // remembered-serving fallback); dropping `persistence` brings back the split
833
+ // restart.
834
+ ...(config.networkState && { persistence: strandNetworkStatePersistence(config.networkState, strandId) }),
835
+ ...(config.network?.transports && { transports: config.network.transports }),
836
+ ...(config.network?.noiseCrypto && { noiseCrypto: config.network.noiseCrypto }),
837
+ // Unconditional, and the same default the control node takes: every node of
838
+ // every party runs the monitor over its connection to a slow phone, so the
839
+ // widened ping deadline has to reach the strand nodes too (see
840
+ // DEFAULT_CONNECTION_MONITOR).
841
+ connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
842
+ // The same declared link the control node states, and for the same reason: a strand
843
+ // node is also the listener for every other member's strand node, and Optimystic
844
+ // derives that listener's limit from this (`link-budget.ts`).
845
+ linkRoundTripMs: resolveLinkRoundTripMs(config.network?.linkRoundTripMs),
846
+ // The same dial limits as the control node: a strand node dials the same machines,
847
+ // through the same relays and gates.
848
+ ...optimysticDialLimits(config.network?.linkRoundTripMs),
849
+ // Listen entries plus the WebSocket transport switch they imply — a strand node
850
+ // announces nothing the operator configured (`strand-network-config.ts`), and
851
+ // spreads AFTER `transports` above because the switch is a no-op whenever the
852
+ // embedder supplied its own factories. The bare `/p2p-circuit` entries (one per
853
+ // relay) open no connection here; each registers a pending reservation that the
854
+ // matching supervisor below fills, which is what gives a NAT'd strand node a
855
+ // reachable relay slot — and works against a party-run relay because the launch
856
+ // path announced this strand's derived peerId to it first (delegate admission;
857
+ // see cadre-node.ts).
858
+ ...addrOptions,
859
+ // The raw configured gater (open strands), or the revocation-composed
860
+ // gater plus the fail-closed per-stream revoked-peer gate (closed
861
+ // strands) — resolved above, before the try.
862
+ ...revocationGateOptions,
863
+ // `{}` unless the operator enabled change notifications for this strand (`strand-reactivity.ts`).
864
+ ...reactivityOption
865
+ }) as Libp2pNodeWithRepo;
866
+ timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
867
+
868
+ instance.libp2pNode = node;
869
+
870
+ // One reservation supervisor PER RELAY, started now so the first attempts
871
+ // overlap the database bring-up below (the relay is never in a strand's
872
+ // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
873
+ // — so an open relay connection cannot disturb that bring-up). Their first
874
+ // attempts are awaited before the strand goes `active`, below. Registered in the
875
+ // map immediately so the failure rollback (releaseRuntime) stops them like every
876
+ // other runtime component.
877
+ this.relaySupervisors.set(strandId, this.startRelaySupervisors(strandId, node, relayDialAddrs, config));
878
+
879
+ // Create and initialize the StrandDatabase.
880
+ //
881
+ // Held by the first-sync gate (registered before initialize) rather than attached
882
+ // to the instance: a failed init is then cleaned up by releaseRuntime below through
883
+ // the gate (close() is safe on a partially-initialized db), and the app only ever
884
+ // sees `instance.database` once this machine is allowed to write to the strand.
885
+ t0 = performance.now();
886
+ const strandDb = new StrandDatabase({
887
+ strandId,
888
+ sAppConfig,
889
+ libp2pNode: node,
890
+ coordinatedRepo: node.coordinatedRepo,
891
+ // Founder bootstrap inputs: the strand's type drives which membership rows
892
+ // are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
893
+ // caller from the control-layer StrandPartyKey row) derives the founding
894
+ // Member/Manager key. The row's shared MemberPrivateKey rides along as the
895
+ // read-gating secret only — it derives nobody's identity.
896
+ strandType: config.strandRow.Type,
897
+ memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
898
+ partyMemberPrivateKey: config.partyMemberPrivateKey,
899
+ founder: config.founder
900
+ });
901
+ const gate = new StrandFirstSyncGate({
902
+ label: strandId,
903
+ database: strandDb,
904
+ onHeaderHeld: () => this.publishDatabase(instance, strandDb)
905
+ }, config.firstSync);
906
+ this.firstSyncGates.set(strandId, gate);
907
+ try {
908
+ await strandDb.initialize();
909
+ } catch (error) {
910
+ joinWaitsForSeed = isCohortUnreachableRead(error);
911
+ throw error;
912
+ }
913
+ timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
914
+
915
+ // The first-sync write gate. A founder just wrote the Header in its bootstrap; any
916
+ // other machine may commit only once it holds the Header — which it can only have
917
+ // received from a peer (now, or on an earlier run over the same store) — and has read
918
+ // each App table once, so their collections are fetched too. Publishing the database
919
+ // is what makes the strand writable to the app; until then the instance stays
920
+ // `'syncing'` and the gate re-probes on its cadence. Everything armed below reads
921
+ // `instance.database` lazily, so the reconciler and enforcer wait with it — and
922
+ // `publishDatabase` kicks the reconciler when the wait ends, so a joiner's membership
923
+ // rows land with the strand rather than one retry later.
924
+ t0 = performance.now();
925
+ if (config.founder === true || await strandFirstSyncComplete(strandDb.getDatabase(), strandId)) {
926
+ this.publishDatabase(instance, strandDb);
927
+ } else {
928
+ gate.start();
929
+ }
930
+ timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
931
+
932
+ // The membership reconciler: finish this party's join on every machine —
933
+ // redeem a staged formation invitation if one is pending, then write the
934
+ // durable machine→party binding (Strand.MemberPeer). Armed here, the one
935
+ // seam where the strand's transport peer id and a live Database both exist
936
+ // for launch AND hibernation resume, for closed strands that carry the
937
+ // party's own membership key. start() kicks an immediate pass WITHOUT
938
+ // awaiting it — a joiner at this instant has not synced the founder's rows
939
+ // and may lack write quorum, so bring-up is never blocked; an unfinished join
940
+ // then retries on a short doubling ladder capped at the configured
941
+ // pollIntervalMs (which is also the flat cadence while nobody has admitted this
942
+ // party yet), and the loop stops once the member row and this machine's own
943
+ // binding are both in place. Armed AFTER the gate above, so a gated launch
944
+ // always has a reconciler registered by the time the gate can publish.
945
+ if (config.strandRow.Type === 'c' && config.partyMemberPrivateKey
946
+ && config.membershipReconciliation?.enabled !== false) {
947
+ const reconciler = new StrandMembershipReconciler({
948
+ label: strandId,
949
+ partyMemberPrivateKey: config.partyMemberPrivateKey,
950
+ // Read per pass, never captured — same lifecycle argument as the
951
+ // enforcer's getNetwork: quiesce drops both handles.
952
+ getDatabase: () => instance.database?.getDatabase(),
953
+ getOwnPeerId: () => instance.libp2pNode?.peerId.toString(),
954
+ pendingInvite: config.pendingMembershipInvite,
955
+ // The enforcer's CURRENT snapshot view of this node's own peer id — the
956
+ // reconciler stops rather than fight a self-revocation. Absent when the
957
+ // gate is disarmed (fail-open: the loop just runs).
958
+ isSelfRevoked: revocationEnforcer
959
+ ? () => {
960
+ const ownPeerId = instance.libp2pNode?.peerId.toString();
961
+ return ownPeerId !== undefined && revocationEnforcer.isRevoked(ownPeerId);
962
+ }
963
+ : undefined,
964
+ onRejoinBlocked: () => config.onRejoinBlocked?.(strandId)
965
+ }, {
966
+ pollIntervalMs: config.membershipReconciliation?.pollIntervalMs
967
+ ?? config.revocationEnforcement?.pollIntervalMs
968
+ });
969
+ this.membershipReconcilers.set(strandId, reconciler);
970
+ reconciler.start();
971
+ }
972
+
973
+ // Arm the revoked-peer deny-set poll now that the database it reads
974
+ // exists. start() kicks an immediate refresh WITHOUT awaiting it —
975
+ // bring-up and resume are never blocked on a membership read; until the
976
+ // first read lands the empty snapshot admits everyone (fail-open).
977
+ revocationEnforcer?.start();
978
+
979
+ // Peer-join block catch-up: push this strand's own blocks to each newly
980
+ // connected peer, so a machine that joined after blocks were committed
981
+ // still ends up physically holding them (without per-strand storage there
982
+ // is nothing to copy). Armed for EVERY stored strand: `PeerJoinBackfill`
983
+ // only does work when the strand's libp2p node reports a peer connection,
984
+ // so on a device that is genuinely alone it is inert, and arming it at
985
+ // launch is what closes the "founded alone, never replicates" hole — a
986
+ // peer that joins later gets the founder's blocks without any relaunch.
987
+ // No `authorizePeer` gate, deliberately — see the module comment in
988
+ // peer-join-backfill.ts for the strand-side argument (and why the control
989
+ // network, which DOES gate, is different).
990
+ //
991
+ // NOTE: cost is one PeerJoinBackfill object + one `peer:identify` listener
992
+ // per running strand — linear in strand count, negligible at the handful a
993
+ // device or host runs today. If a node ever hosts strands by the hundred,
994
+ // move to one shared listener that dispatches by strand id.
995
+ if (strandStorage && config.backfill?.enabled !== false) {
996
+ if (node.keyNetwork) {
997
+ const backfill = new PeerJoinBackfill({
998
+ label: strandId,
999
+ libp2p: node,
1000
+ peerNetwork: node.keyNetwork,
1001
+ storage: strandStorage,
1002
+ // The same prefix the receiver registered its block-transfer handler
1003
+ // under — derived from networkName above, never re-spelled here.
1004
+ protocolPrefix
1005
+ }, {
1006
+ // Dial and response deadlines counted in link round trips, not fixed milliseconds:
1007
+ // a relayed dial costs a fixed number of exchanges, so a fixed budget has a link
1008
+ // speed above which this catch-up can never reach the peer at all. See
1009
+ // `link-budget.ts`. Spread BEFORE the host's own config so an explicit
1010
+ // `strandBackfill.dialTimeoutMs` still wins.
1011
+ ...peerJoinPushBudget(config.network?.linkRoundTripMs),
1012
+ ...config.backfill
1013
+ });
1014
+ backfill.start();
1015
+ this.backfills.set(strandId, backfill);
1016
+ } else {
1017
+ log('Strand %s: libp2p node exposes no keyNetwork; peer-join block catch-up is inert', strandId);
1018
+ }
1019
+ }
1020
+
1021
+ // The relay supervisors' FIRST attempts, all of them, before `active`: the happy
1022
+ // path still publishes its circuit addr before `addStrand` resolves (the
1023
+ // strand-addr RPC answers read it, and the same-party circuit scenario asserts
1024
+ // it). Fail-SOFT — a first attempt that lands nothing does not fail the launch.
1025
+ // The strand's database is up, and the supervisor keeps trying on its backoff;
1026
+ // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
1027
+ //
1028
+ // NOTE: a relay that is down costs this launch one full drive — the reservation budget
1029
+ // counted from the declared link round trip, 18 s at its default (`link-budget.ts`) — and
1030
+ // `StrandWatcher` launches strands one at a time, so N strands cost N of those in
1031
+ // bring-up during a relay outage, and MORE on a host that declared a slower link. If that
1032
+ // ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
1033
+ // than shortening the drive.
1034
+ t0 = performance.now();
1035
+ await this.awaitFirstRelayAttempts(strandId);
1036
+ timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
1037
+
1038
+ // No `lastActivity` stamp: bringing a runtime up is not activity. A check-in marks
1039
+ // activity before its resume and must see only what landed while the strand was up.
1040
+ instance.status = liveStrandStatus(instance);
1041
+ if (instance.status === 'syncing') {
1042
+ log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
1043
+ 'a member of the strand is reached', strandId);
1044
+ }
1045
+ } catch (error) {
1046
+ // Roll back any partially-attached runtime so the instance is left with
1047
+ // NEITHER handle. Otherwise the `libp2pNode || database` "already live"
1048
+ // guard in resumeStrand/handleStrandWake would treat a half-built strand
1049
+ // as healthy — leaking the libp2p node and never retrying the rebuild.
1050
+ await this.releaseRuntime(instance).catch((cleanupErr) => {
1051
+ log('buildStrandRuntime cleanup for strand %s also failed: %o', strandId, cleanupErr);
1052
+ });
1053
+ if (joinWaitsForSeed) {
1054
+ this.deferUnreachableJoin(instance, error);
1055
+ return;
1056
+ }
1057
+ throw error;
1058
+ }
1059
+ }
1060
+
1061
+ /**
1062
+ * Keep a launch `'syncing'` when its database could not be built because no seed peer has
1063
+ * answered yet, and rebuild the runtime every {@link UNREACHABLE_JOIN_RETRY_MS}.
1064
+ *
1065
+ * Optimystic (since 1.11.0) will not rule a block never-created while any of a node's
1066
+ * `bootstrapNodes` is unheard, so a joiner alone with an unreachable seed cannot build even
1067
+ * the schema catalog: `strandDb.initialize()` fails with `cohort-unreachable`. A founder
1068
+ * given a seed is refused the same way (normally it has none: no sibling runs the strand yet). Refusing is
1069
+ * right (building it alone would fork the catalog), but failing the launch would drop the
1070
+ * strand until the watcher relaunched it, and `addStrand` would reject with a raw Optimystic
1071
+ * error. Deferring gives a joiner the same contract the first-sync gate gives one that did
1072
+ * build: the strand stays tracked, {@link whenWritable} waits and rejects with the retryable
1073
+ * `StrandAwaitingFirstSyncError`, and the launch proceeds once a seed peer answers. A failed
1074
+ * composition stops the injected libp2p node, so each retry rebuilds the whole runtime.
1075
+ */
1076
+ private deferUnreachableJoin(instance: StrandInstance, error: unknown): void {
1077
+ const strandId = instance.strandId;
1078
+ instance.status = 'syncing';
1079
+ log('Strand %s: no seed peer has answered yet, so this joiner cannot build its database (%s); ' +
1080
+ 'retrying in %dms', strandId, error instanceof Error ? error.message : String(error), UNREACHABLE_JOIN_RETRY_MS);
1081
+ const timer = setTimeout(() => {
1082
+ this.unreachableJoinRetries.delete(strandId);
1083
+ void this.retryUnreachableJoin(instance);
1084
+ }, UNREACHABLE_JOIN_RETRY_MS);
1085
+ (timer as { unref?: () => void }).unref?.();
1086
+ this.unreachableJoinRetries.set(strandId, timer);
1087
+ }
1088
+
1089
+ /** One retry scheduled by {@link deferUnreachableJoin}, unless the strand was stopped or rebuilt since. */
1090
+ private async retryUnreachableJoin(instance: StrandInstance): Promise<void> {
1091
+ const strandId = instance.strandId;
1092
+ const config = this.launchConfigs.get(strandId);
1093
+ if (this.stopping || this.instances.get(strandId) !== instance || !config
1094
+ || instance.libp2pNode || this.runtimeBuilds.has(strandId) || this.transitions.has(strandId)) {
1095
+ return;
1096
+ }
1097
+ try {
1098
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, config));
1099
+ } catch (error) {
1100
+ instance.status = 'error';
1101
+ instance.error = error instanceof Error ? error.message : String(error);
1102
+ log('Strand %s: rebuilding a joiner that was waiting for a seed peer failed: %s', strandId, instance.error);
1103
+ }
1104
+ }
1105
+
1106
+ /**
1107
+ * Release an instance's strand-network runtime: stop the relay supervisors and
1108
+ * the other background loops, close the StrandDatabase, then stop the libp2p
1109
+ * node (construction order in reverse), clearing both fields and zeroing
1110
+ * connectedPeers. Tolerant of partially-built state — either handle may be
1111
+ * absent — so it doubles as rollback for a failed `buildStrandRuntime`. Shared
1112
+ * by `quiesceStrand`, `stopStrand`, and that rollback path.
1113
+ *
1114
+ * Leaves `strandStorages` untouched by design — the store outlives the runtime it
1115
+ * was built into, which is what makes a resume warm. Only `stopStrand` disposes it.
1116
+ */
1117
+ private async releaseRuntime(instance: StrandInstance): Promise<void> {
1118
+ // Relay supervisors FIRST of all — before anything below is torn down — so no
1119
+ // re-drive is scheduled against a node being stopped. `stop()` is synchronous
1120
+ // and never awaits a drive, so a relay that is down cannot delay this teardown.
1121
+ // `stop()` also cancels the drive already in flight, so an attempt started
1122
+ // moments before this does not keep dialing and polling the node the lines
1123
+ // below are stopping; its result is discarded either way.
1124
+ const unreachableJoinRetry = this.unreachableJoinRetries.get(instance.strandId);
1125
+ if (unreachableJoinRetry !== undefined) {
1126
+ clearTimeout(unreachableJoinRetry as ReturnType<typeof setTimeout>);
1127
+ this.unreachableJoinRetries.delete(instance.strandId);
1128
+ }
1129
+ const relaySupervisors = this.relaySupervisors.get(instance.strandId);
1130
+ if (relaySupervisors) {
1131
+ relaySupervisors.forEach((supervisor) => supervisor.stop());
1132
+ this.relaySupervisors.delete(instance.strandId);
1133
+ }
1134
+ // Backfill next — before the database closes and the libp2p node stops — so
1135
+ // no NEW catch-up push is issued against a torn-down transport. A push already
1136
+ // in flight is not awaited; it fails into the module's own per-chunk catch.
1137
+ const backfill = this.backfills.get(instance.strandId);
1138
+ if (backfill) {
1139
+ backfill.stop();
1140
+ this.backfills.delete(instance.strandId);
1141
+ }
1142
+ // The revoked-peer enforcer goes with the runtime it gated: a resume
1143
+ // rebuilds it with a fresh (initially empty, fail-open) snapshot.
1144
+ const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
1145
+ if (revocationEnforcer) {
1146
+ revocationEnforcer.stop();
1147
+ this.revocationEnforcers.delete(instance.strandId);
1148
+ }
1149
+ // The membership reconciler likewise: a resume rebuilds it and re-runs the
1150
+ // idempotent ladder, which is what makes resume-after-partial-join heal.
1151
+ const membershipReconciler = this.membershipReconcilers.get(instance.strandId);
1152
+ if (membershipReconciler) {
1153
+ membershipReconciler.stop();
1154
+ this.membershipReconcilers.delete(instance.strandId);
1155
+ }
1156
+ // A still-gated database (a joiner that never received the Header, or a launch whose
1157
+ // initialize() failed) is closed through its gate; a published one through the
1158
+ // instance. Never both — publishing moves the handle from the gate to the instance.
1159
+ const gate = this.firstSyncGates.get(instance.strandId);
1160
+ if (gate) {
1161
+ gate.stop();
1162
+ this.firstSyncGates.delete(instance.strandId);
1163
+ await gate.database.close();
1164
+ }
1165
+ if (instance.database) {
1166
+ await instance.database.close();
1167
+ instance.database = undefined;
1168
+ }
1169
+ if (instance.libp2pNode) {
1170
+ await instance.libp2pNode.stop();
1171
+ instance.libp2pNode = undefined;
1172
+ }
1173
+ instance.connectedPeers = 0;
1174
+ }
1175
+
1176
+ /**
1177
+ * One {@link superviseRelayReservation} per relay dial addr, each over exactly
1178
+ * that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
1179
+ * one relay re-drives only that one. Default retry timings — the control node's (2 s
1180
+ * doubling to 60 s between failed attempts, a 5 s liveness check while held). Each DRIVE's
1181
+ * own deadline is counted in link round trips at this host's declared
1182
+ * `network.linkRoundTripMs` (`link-budget.ts`), so the same declaration that lengthens the
1183
+ * control node's drive lengthens these.
1184
+ *
1185
+ * Each supervisor's `beforeRedrive` is the caller's
1186
+ * {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
1187
+ * node's peer id, so a re-drive after a relay restart is preceded by a fresh
1188
+ * delegate grant on the relay about to be dialed. Returns `[]` when no relay is
1189
+ * configured — the strand then simply has no circuit listener to fill.
1190
+ */
1191
+ private startRelaySupervisors(
1192
+ strandId: string,
1193
+ node: Libp2pNodeWithRepo,
1194
+ relayDialAddrs: readonly string[],
1195
+ config: StartStrandConfig
1196
+ ): RelayReservationSupervisor[] {
1197
+ if (relayDialAddrs.length === 0) {
1198
+ return [];
1199
+ }
1200
+ const announce = config.announceDelegateToRelay;
1201
+ const delegatePeerId = node.peerId.toString();
1202
+ const timeoutMs = relayReservationBudgetMs(config.network?.linkRoundTripMs);
1203
+ return relayDialAddrs.map((relayAddr) => superviseRelayReservation(node, [relayAddr], {
1204
+ timeoutMs,
1205
+ ...(announce && { beforeRedrive: () => announce(strandId, relayAddr, delegatePeerId) })
1206
+ }));
1207
+ }
1208
+
1209
+ /**
1210
+ * Wait for every relay supervisor's first attempt to settle, concurrently, and log
1211
+ * the ones that landed nothing. Never throws — a strand whose relay is unreachable
1212
+ * at launch still comes up, with its supervisors retrying in the background.
1213
+ */
1214
+ private async awaitFirstRelayAttempts(strandId: string): Promise<void> {
1215
+ const supervisors = this.relaySupervisors.get(strandId) ?? [];
1216
+ if (supervisors.length === 0) {
1217
+ return;
1218
+ }
1219
+ await Promise.all(supervisors.map((supervisor) => supervisor.firstAttempt));
1220
+ supervisors.forEach((supervisor) => {
1221
+ if (supervisor.lastError !== null) {
1222
+ log('Strand %s: relay reservation not held after the first attempt (retrying in the background): %s',
1223
+ strandId, supervisor.lastError);
1224
+ }
1225
+ });
1226
+ }
1227
+
1228
+ /**
1229
+ * Drop the strand's owned store and release this strand's claim on its cache. The
1230
+ * wrapper counts holders, so the cache is emptied and unregistered from the shared
1231
+ * pool only if no other scope still holds it. Called only where the strand's whole
1232
+ * lifetime ends (`stopStrand`, and the failed-launch rollback) — never on a quiesce.
1233
+ *
1234
+ * A dispose failure is logged, not thrown: the store is already unreferenced here,
1235
+ * and failing the stop over a cache-bookkeeping error would leave the caller unable
1236
+ * to tear the strand down. `disposeStorageCache` no-ops for an unwrapped store
1237
+ * (e.g. `MemoryRawStorage`), so no instanceof test is needed.
1238
+ */
1239
+ private async disposeStrandStorage(strandId: string): Promise<void> {
1240
+ const storage = this.strandStorages.get(strandId);
1241
+ if (!storage) {
1242
+ return;
1243
+ }
1244
+ this.strandStorages.delete(strandId);
1245
+ try {
1246
+ await disposeStorageCache(storage);
1247
+ } catch (error) {
1248
+ log('Failed to dispose storage cache for strand %s: %o', strandId, error);
1249
+ }
1250
+ }
1251
+
1252
+ /**
1253
+ * Refresh a strand's revoked-peer deny set NOW — and, with it, run the
1254
+ * teardown sweep that hangs up every connected revoked peer.
1255
+ *
1256
+ * The gate is otherwise poll-driven (default
1257
+ * `DEFAULT_REVOCATION_POLL_INTERVAL_MS`), because neither a replicated
1258
+ * revocation nor a local `revokeMember`/`leaveStrand` raises anything this
1259
+ * runtime can hook. A caller that just wrote a revocation against the
1260
+ * strand's database should follow it with this call, which makes the cut
1261
+ * immediate instead of waiting out the interval.
1262
+ *
1263
+ * Quiet no-op — matching `quiesceStrand`'s posture — when the strand is not
1264
+ * tracked, is quiesced, is open, or has the gate disabled: in every one of
1265
+ * those cases there is no enforcer to refresh. Never rejects (the enforcer's
1266
+ * refresh contains its own failures), and resolves only once the sweep has
1267
+ * finished.
1268
+ */
1269
+ async refreshRevocationEnforcement(strandId: string): Promise<void> {
1270
+ const enforcer = this.revocationEnforcers.get(strandId);
1271
+ if (!enforcer) {
1272
+ log('refreshRevocationEnforcement: strand %s has no armed revocation enforcer', strandId);
1273
+ return;
1274
+ }
1275
+ await enforcer.refresh();
1276
+ }
1277
+
1278
+ /**
1279
+ * A fresh membership invitation was staged for `strandId` (`CadreNode`'s
1280
+ * `adoptFormationMembershipInvite`): re-arm the strand's membership reconciler so the
1281
+ * invitation is attempted now rather than never — the loop finished during the first
1282
+ * join and would otherwise stay stopped until a relaunch. Quiet no-op when no reconciler
1283
+ * is armed: a first formation stages before the strand is added, and bring-up then arms
1284
+ * a loop that finds the invitation by itself. Returns at once; the pass runs on the
1285
+ * loop's own serialized chain and never rejects.
1286
+ */
1287
+ notifyMembershipInviteStaged(strandId: string): void {
1288
+ const reconciler = this.membershipReconcilers.get(strandId);
1289
+ if (!reconciler) {
1290
+ log('notifyMembershipInviteStaged: strand %s has no armed membership reconciler — bring-up will find the invitation', strandId);
1291
+ return;
1292
+ }
1293
+ void reconciler.rearm();
1294
+ }
1295
+
1296
+ /**
1297
+ * Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
1298
+ * self arm of `removeMemberPeer`, signed with the retained launch config's party
1299
+ * key. Called by `CadreNode.unpublishStrand` BEFORE the local stop, while the
1300
+ * strand database and transport are still live: after the unpublish commits, the
1301
+ * party's `StrandPartyKey` control row is gone, so no restart could ever sign
1302
+ * this removal again — the in-memory retained key is the last chance.
1303
+ *
1304
+ * Never throws. Quiet no-op for a strand that is untracked, open, or launched
1305
+ * without a party key; a quiesced (hibernating) strand, or a strand whose write
1306
+ * quorum is already unreachable, leaves the stale binding behind with a log —
1307
+ * it grants nothing today (admission is not allowlist-gated) and only
1308
+ * mis-credits diversity. The strand's own membership reconciler is stopped —
1309
+ * and its in-flight pass awaited — FIRST, so a pass that was already past its
1310
+ * own stopped check cannot re-register the binding this is about to clear.
1311
+ *
1312
+ * NOTE: deliberately NOT handled here (or anywhere yet): clearing the bindings
1313
+ * of a machine removed from the CADRE at the control layer, or of a party's
1314
+ * OTHER machines when the party unpublishes — the party still holds the key on
1315
+ * every surviving machine and can clear from any of them, and the stale rows
1316
+ * grant nothing while admission is deny-list only. Revisit when
1317
+ * `feat-strand-member-allowlist-admission` lands and a stale binding starts
1318
+ * granting admission rather than merely mis-crediting diversity.
1319
+ */
1320
+ async clearOwnMemberPeerBinding(strandId: string): Promise<void> {
1321
+ const instance = this.instances.get(strandId);
1322
+ const config = this.launchConfigs.get(strandId);
1323
+ if (!instance || !config || config.strandRow.Type !== 'c' || !config.partyMemberPrivateKey) {
1324
+ return;
1325
+ }
1326
+ const reconciler = this.membershipReconcilers.get(strandId);
1327
+ if (reconciler) {
1328
+ reconciler.stop();
1329
+ await reconciler.settle();
1330
+ }
1331
+ const db = instance.database?.getDatabase();
1332
+ const peerId = instance.libp2pNode?.peerId.toString();
1333
+ if (!db || !peerId) {
1334
+ log('clearOwnMemberPeerBinding: strand %s has no live runtime — own MemberPeer binding left in place', strandId);
1335
+ return;
1336
+ }
1337
+ try {
1338
+ // Background write on the app's database: never join a transaction the app has open.
1339
+ await removeMemberPeer(db, {
1340
+ memberKeyPair: strandMemberKeyPair(config.partyMemberPrivateKey),
1341
+ peerId
1342
+ }, { joinOpenTransaction: false });
1343
+ log('clearOwnMemberPeerBinding: strand %s — own binding for peer %s cleared', strandId, peerId);
1344
+ } catch (error) {
1345
+ log('clearOwnMemberPeerBinding: strand %s — best-effort clear failed (strand may already be unreachable): %o',
1346
+ strandId, error);
1347
+ }
1348
+ }
1349
+
1350
+ /**
1351
+ * Quiesce a strand: release its strand-network resources (stop the libp2p node,
1352
+ * close the StrandDatabase) while RETAINING the instance record — identity,
1353
+ * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
1354
+ * resumed later, then mark it `hibernating`. Mechanically this is `stopStrand` minus
1355
+ * the instance/config deletion. A tracked strand with no runtime left (a failed
1356
+ * rebuild already released it) is only marked; an untracked one is left alone.
1357
+ *
1358
+ * Ordered with {@link resumeStrand} (see {@link transitions}): it starts once the resume
1359
+ * or launch build before it has settled, so it releases a finished runtime, and the
1360
+ * status write is its last step, so a resume requested meanwhile writes after it. A call
1361
+ * made while another quiesce is the latest request joins it. A failed release leaves the
1362
+ * status as it was and rejects.
1363
+ */
1364
+ async quiesceStrand(strandId: string): Promise<void> {
1365
+ const previous = this.transitions.get(strandId);
1366
+ if (previous?.kind === 'quiesce') {
1367
+ log('quiesceStrand: strand %s — joining the quiesce already requested', strandId);
1368
+ return previous.operation;
1369
+ }
1370
+ const operation = this.runQuiesce(strandId, previous);
1371
+ this.holdTransition(strandId, { kind: 'quiesce', operation });
1372
+ return operation;
1373
+ }
1374
+
1375
+ /** Body of {@link quiesceStrand}. */
1376
+ private async runQuiesce(strandId: string, previous: RuntimeTransition | undefined): Promise<void> {
1377
+ await this.settleTransition(strandId, previous);
1378
+ await this.settleRuntimeBuilds(strandId);
1379
+ const instance = this.instances.get(strandId);
1380
+ if (!instance) {
1381
+ log('quiesceStrand: strand %s not found', strandId);
1382
+ return;
1383
+ }
1384
+ if (instance.libp2pNode || instance.database) {
1385
+ log('Quiescing strand instance: %s', strandId);
1386
+ await this.releaseRuntime(instance);
1387
+ log('Strand %s quiesced (resources released, instance retained)', strandId);
1388
+ } else {
1389
+ log('quiesceStrand: strand %s already quiesced', strandId);
1390
+ }
1391
+ instance.status = 'hibernating';
1392
+ }
1393
+
1394
+ /**
1395
+ * Whether the latest runtime transition requested for `strandId` is a quiesce that has
1396
+ * not finished. Its handles may still be attached and its status still live, but it is on
1397
+ * its way to `hibernating`, so a caller that needs it up must {@link resumeStrand} it — the
1398
+ * resume runs after the quiesce — rather than treat it as live.
1399
+ */
1400
+ isQuiescing(strandId: string): boolean {
1401
+ return this.transitions.get(strandId)?.kind === 'quiesce';
1402
+ }
1403
+
1404
+ /**
1405
+ * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
1406
+ * from the retained launch config and re-attach them, transitioning it back to
1407
+ * `active`. `overrides` re-applies volatile inputs that may have changed since
1408
+ * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
1409
+ * count) and updates the retained config so a later resume reuses the latest
1410
+ * values. Returns the live instance unchanged if it is already running.
1411
+ *
1412
+ * Overlapping calls share one rebuild: a call made while a resume is the latest request
1413
+ * for the strand — a wake landing during a check-in — joins it and settles exactly as it
1414
+ * does, `'error'` status included. The joiner's own `overrides` are ignored; the first
1415
+ * resume's seed wins, and both callers resolved the cohort seed moments apart. A call made
1416
+ * while a quiesce is the latest request runs after that quiesce and rebuilds (see
1417
+ * {@link transitions}), and one made while `startStrand` is still building waits for that
1418
+ * build rather than starting a second.
1419
+ */
1420
+ async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
1421
+ if (this.stopping) {
1422
+ throw new Error('StrandInstanceManager is stopping');
1423
+ }
1424
+ // No `await` between this read and `holdTransition`, or a second caller slips through
1425
+ // and builds.
1426
+ const previous = this.transitions.get(strandId);
1427
+ if (previous?.kind === 'resume') {
1428
+ log('resumeStrand: strand %s — joining the resume already in flight', strandId);
1429
+ return previous.operation;
1430
+ }
1431
+ const operation = this.runResume(strandId, previous, overrides);
1432
+ this.holdTransition(strandId, { kind: 'resume', operation });
1433
+ return operation;
1434
+ }
1435
+
1436
+ /** Record `transition` as the latest for `strandId` until it settles. */
1437
+ private holdTransition(strandId: string, transition: RuntimeTransition): void {
1438
+ this.transitions.set(strandId, transition);
1439
+ const release = (): void => {
1440
+ if (this.transitions.get(strandId) === transition) {
1441
+ this.transitions.delete(strandId);
1442
+ }
1443
+ };
1444
+ // The rejection itself reaches the caller through the operation returned to it.
1445
+ void transition.operation.then(release, release);
1446
+ }
1447
+
1448
+ /** Wait for the transition requested before this one. Never rejects: its failure belongs to its caller. */
1449
+ private async settleTransition(strandId: string, previous: RuntimeTransition | undefined): Promise<void> {
1450
+ if (!previous) {
1451
+ return;
1452
+ }
1453
+ await (previous.operation as Promise<unknown>).catch((error: unknown) => {
1454
+ log('Strand %s: the %s queued ahead failed (reported by its caller): %o', strandId, previous.kind, error);
1455
+ });
1456
+ }
1457
+
1458
+ /** Body of {@link resumeStrand}; at most one runs per strand at a time. */
1459
+ private async runResume(
1460
+ strandId: string,
1461
+ previous: RuntimeTransition | undefined,
1462
+ overrides?: ResumeStrandOverrides
1463
+ ): Promise<StrandInstance> {
1464
+ // Before any "already live" test: a quiesce ahead still has the handles attached, and
1465
+ // mid-build (a launch still building) the libp2p node is attached while the database is
1466
+ // not. A failed launch has already dropped the instance, and the check below reports it
1467
+ // untracked.
1468
+ await this.settleTransition(strandId, previous);
1469
+ await this.settleRuntimeBuilds(strandId);
1470
+ const instance = this.instances.get(strandId);
1471
+ if (!instance) {
1472
+ throw new Error(`Cannot resume strand ${strandId}: not tracked`);
1473
+ }
1474
+ const launchConfig = this.launchConfigs.get(strandId);
1475
+ if (!launchConfig) {
1476
+ throw new Error(`Cannot resume strand ${strandId}: no retained launch config`);
1477
+ }
1478
+ if (instance.libp2pNode || instance.database) {
1479
+ log('resumeStrand: strand %s already live', strandId);
1480
+ return instance;
1481
+ }
1482
+
1483
+ log('Resuming strand instance: %s', strandId);
1484
+ const tTotal = performance.now();
1485
+
1486
+ // Re-apply volatile inputs and persist them so a subsequent resume reuses them.
1487
+ // Each `??` matters: a resume that passes no override must keep the retained value,
1488
+ // and a resume that passes one must leave it retained for the next resume — otherwise
1489
+ // a later no-override wake silently reverts to whatever launch time saw.
1490
+ const resumeConfig: StartStrandConfig = {
1491
+ ...launchConfig,
1492
+ bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes,
1493
+ // NOTE: `??` retains, so an override can raise or lower the count but cannot CLEAR it
1494
+ // back to "this node no longer knows" — the direction that would declare nothing. Moot
1495
+ // while nothing feeds `servingMachines` at all; if a source lands
1496
+ // (`backlog/feat-strand-yardstick-from-serving-machines`) that can legitimately lose the
1497
+ // count — a strand whose member rows became unreadable — this merge must gain an explicit
1498
+ // clear rather than silently declaring a stale number over a serving set it can no longer
1499
+ // see. Same applies to `bootstrapNodes` above, where a stale seed is harmless.
1500
+ servingMachines: overrides?.servingMachines ?? launchConfig.servingMachines
1501
+ };
1502
+ this.launchConfigs.set(strandId, resumeConfig);
1503
+
1504
+ instance.status = 'starting';
1505
+ try {
1506
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, resumeConfig));
1507
+ timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
1508
+ log('Strand %s resumed successfully', strandId);
1509
+ return instance;
1510
+ } catch (error) {
1511
+ instance.status = 'error';
1512
+ instance.error = error instanceof Error ? error.message : String(error);
1513
+ log('Failed to resume strand %s: %s', strandId, instance.error);
1514
+ throw error;
1515
+ }
1516
+ }
1517
+
1518
+ /**
1519
+ * Honor a founder request against an ALREADY-TRACKED strand — the seam that
1520
+ * closes the "whoever launches first decides whether the bootstrap runs" gap:
1521
+ * an instance first launched as a joiner (an app's own attach, or a watcher
1522
+ * poll winning the launch race) used to swallow a later founder request
1523
+ * silently, leaving the strand active with no `Strand.Header`.
1524
+ *
1525
+ * Flips the RETAINED launch config's `founder` to true, so every later
1526
+ * quiesce → resume rebuild founds as well (the bootstrap is insert-if-absent —
1527
+ * {@link StrandDatabase.ensureFounderBootstrap} — so re-running it per rebuild
1528
+ * writes nothing twice), and runs the bootstrap against the live database now.
1529
+ *
1530
+ * @returns how the request resolved:
1531
+ * - `'already-founder'` — the retained config already founds; nothing to do.
1532
+ * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1533
+ * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1534
+ * database) or being quiesced (its database is closing), so the bootstrap could
1535
+ * not run here: the CALLER must wake the
1536
+ * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1537
+ * manager does not) so the rebuild — which now founds — runs it.
1538
+ * @param resolvePartyKey - Asked for the party's own membership key when (and only
1539
+ * when) a CLOSED strand's retained config carries none — the instance was launched
1540
+ * as a joiner before its `StrandPartyKey` row existed or replicated. The resolved
1541
+ * key is retained alongside the founder flip so the bootstrap (now, or on the
1542
+ * caller's wake for `'needs-resume'`) can seat the founding Member/Manager.
1543
+ * @throws when the strand is not tracked — this seam exists only for the
1544
+ * tracked-instance launch path; an untracked id is a caller bug.
1545
+ * @throws whatever the live bootstrap throws (e.g. `PreSplitStrandIdentityError`),
1546
+ * after {@link withdrawFounderRequest} — so a retry re-attempts the founding rather
1547
+ * than resolving `'already-founder'` over an instance that never founded. A caller
1548
+ * whose `'needs-resume'` wake fails owes the same withdrawal.
1549
+ */
1550
+ async foundExistingStrand(
1551
+ strandId: string,
1552
+ resolvePartyKey?: () => Promise<string | undefined>
1553
+ ): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'> {
1554
+ const instance = this.instances.get(strandId);
1555
+ const config = this.launchConfigs.get(strandId);
1556
+ if (!instance || !config) {
1557
+ throw new Error(`Cannot found strand ${strandId}: not tracked`);
1558
+ }
1559
+ if (config.founder === true) {
1560
+ return 'already-founder';
1561
+ }
1562
+ const partyMemberPrivateKey =
1563
+ config.partyMemberPrivateKey
1564
+ ?? (config.strandRow.Type === 'c' ? await resolvePartyKey?.() : undefined);
1565
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1566
+ // config object, which is not ours to rewrite.
1567
+ this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1568
+ if (this.isQuiescing(strandId) || (!instance.database && !this.firstSyncGates.has(strandId))) {
1569
+ return 'needs-resume';
1570
+ }
1571
+ try {
1572
+ await this.ensureFounderBootstrap(strandId);
1573
+ } catch (error) {
1574
+ this.withdrawFounderRequest(strandId);
1575
+ throw error;
1576
+ }
1577
+ return 'bootstrapped';
1578
+ }
1579
+
1580
+ /**
1581
+ * Give a tracked storage replica ({@link StartStrandConfig.sAppConfig} absent) the app's
1582
+ * schema in place — the seam for an app on this machine claiming a strand this node was
1583
+ * already hosting. No runtime rebuild: the libp2p node, its peer id and connections, the
1584
+ * store and its warm cache all stay, and the new `App` tables read the blocks the replica
1585
+ * already holds. Callers attach BEFORE {@link foundExistingStrand}: the founder bootstrap
1586
+ * writes the sApp into `Strand.Header` and refuses a database that has none.
1587
+ *
1588
+ * A runtime build in flight (the replica's launch, or a hibernation wake) is waited out
1589
+ * first, so the attach acts on the database that build produced. Then, in order: the
1590
+ * schema signature is checked before anything changes; the retained launch config takes
1591
+ * the sApp, so every later rebuild applies it through `composeStrand`; the live database —
1592
+ * published, or still held by the first-sync gate — gets the schema; and only then does
1593
+ * the instance record `sAppInfo` and the sApp's latency hint. A quiesced instance gets the
1594
+ * config alone, and its next resume applies the schema.
1595
+ *
1596
+ * @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
1597
+ * is logged and left as it is, as a second claim of a claimed strand always has been —
1598
+ * else `'attached'`.
1599
+ * @throws when the strand is not tracked (a caller bug, or a launch that failed while
1600
+ * this waited on it); when the schema signature is refused, having changed nothing; and
1601
+ * whatever the live apply throws (e.g. a quiesce closed the database mid-apply) — the
1602
+ * retained config then still carries the sApp and `sAppInfo` stays unset, so the next
1603
+ * claim retries the apply.
1604
+ *
1605
+ * NOTE: two claims of one replica racing each other both pass the `sAppInfo` check and
1606
+ * apply concurrently; the loser of the declarative diff can reject, and its `addStrand`
1607
+ * retry then resolves `'already-attached'`. Needs an app calling `addStrand` twice at once
1608
+ * (or beside a watcher retry); if it is ever seen, chain attaches per strand id.
1609
+ */
1610
+ async attachSApp(
1611
+ strandId: string,
1612
+ sAppConfig: SAppConfig,
1613
+ options: { requireSignedSchemas?: boolean } = {}
1614
+ ): Promise<'attached' | 'already-attached'> {
1615
+ await this.settleRuntimeBuilds(strandId);
1616
+ const instance = this.instances.get(strandId);
1617
+ const config = this.launchConfigs.get(strandId);
1618
+ if (!instance || !config) {
1619
+ throw new Error(`Cannot attach an sApp to strand ${strandId}: not tracked`);
1620
+ }
1621
+ if (instance.sAppInfo) {
1622
+ if (instance.sAppInfo.id !== sAppConfig.id) {
1623
+ log('attachSApp: strand %s already runs sApp %s — the claim for sApp %s is ignored',
1624
+ strandId, instance.sAppInfo.id, sAppConfig.id);
1625
+ }
1626
+ return 'already-attached';
1627
+ }
1628
+ const sAppInfo = verifiedSAppInfo(strandId, sAppConfig, options.requireSignedSchemas);
1629
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1630
+ // config object, which is not ours to rewrite.
1631
+ this.launchConfigs.set(strandId, { ...config, sAppConfig });
1632
+ const database = instance.database ?? this.firstSyncGates.get(strandId)?.database;
1633
+ if (database) {
1634
+ await database.attachAppSchema(sAppConfig);
1635
+ } else {
1636
+ log('attachSApp: strand %s is quiesced — its next resume applies the sApp schema', strandId);
1637
+ }
1638
+ instance.sAppInfo = sAppInfo;
1639
+ if (sAppConfig.latencyHint) {
1640
+ // HibernationManager reads the hint whenever it arms a timer, so the app's hint governs
1641
+ // from the next one: a timer already armed runs once at the old duration, and a replica
1642
+ // launched under a realtime default was never tracked, so it stays up until relaunched.
1643
+ instance.latencyHint = sAppConfig.latencyHint;
1644
+ }
1645
+ log('Strand %s upgraded from storage replica to %s', strandId, describeSApp(sAppConfig));
1646
+ return 'attached';
1647
+ }
1648
+
1649
+ /**
1650
+ * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1651
+ * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
1652
+ * the hibernation manager's idle timer fired on it), or one waiting to rebuild because no
1653
+ * seed peer had answered ({@link deferUnreachableJoin}). `false` for a writable, quiesced,
1654
+ * or untracked strand. The predicate {@link whenWritable}'s callers gate on.
1655
+ */
1656
+ isAwaitingFirstSync(strandId: string): boolean {
1657
+ const instance = this.instances.get(strandId);
1658
+ return instance !== undefined
1659
+ && (isAwaitingFirstSync(instance) || this.unreachableJoinRetries.has(strandId));
1660
+ }
1661
+
1662
+ /**
1663
+ * Resolve once the strand's database is published to the app — immediately for a
1664
+ * writable strand — or reject with {@link StrandAwaitingFirstSyncError} after
1665
+ * `timeoutMs` (default: the retained launch config's `firstSync.timeoutMs`, else
1666
+ * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}). The rejection is RETRYABLE: nothing is
1667
+ * torn down, the gate keeps probing, and a later call waits afresh. A quiesced
1668
+ * (hibernating) strand is not woken here — the wait spans a resume, so a caller that
1669
+ * wants one wakes it (`CadreNode.wakeStrand`) and waits. Rejects immediately for an
1670
+ * untracked strand, and whenever the strand is stopped while a wait is pending.
1671
+ */
1672
+ whenWritable(strandId: string, options?: { timeoutMs?: number }): Promise<StrandInstance> {
1673
+ const instance = this.instances.get(strandId);
1674
+ if (!instance) {
1675
+ return Promise.reject(new Error(`Cannot wait for strand ${strandId} to become writable: not tracked`));
1676
+ }
1677
+ if (instance.database) {
1678
+ return Promise.resolve(instance);
1679
+ }
1680
+ const timeoutMs = options?.timeoutMs
1681
+ ?? this.launchConfigs.get(strandId)?.firstSync?.timeoutMs
1682
+ ?? DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS;
1683
+ const startedAt = Date.now();
1684
+ return new Promise<StrandInstance>((resolve, reject) => {
1685
+ const waiters = this.writableWaiters.get(strandId) ?? new Set<WritableWaiter>();
1686
+ this.writableWaiters.set(strandId, waiters);
1687
+ const settle = (): void => {
1688
+ clearTimeout(timer);
1689
+ waiters.delete(waiter);
1690
+ if (waiters.size === 0) {
1691
+ this.writableWaiters.delete(strandId);
1692
+ }
1693
+ };
1694
+ const waiter: WritableWaiter = {
1695
+ resolve: () => { settle(); resolve(instance); },
1696
+ reject: (error) => { settle(); reject(error); }
1697
+ };
1698
+ const timer = setTimeout(
1699
+ () => waiter.reject(new StrandAwaitingFirstSyncError(strandId, Date.now() - startedAt)),
1700
+ timeoutMs);
1701
+ (timer as { unref?: () => void }).unref?.();
1702
+ waiters.add(waiter);
1703
+ });
1704
+ }
1705
+
1706
+ /**
1707
+ * The tracked instance once no runtime build of it is in flight, or `undefined` when none
1708
+ * is tracked by then — never launched, or the launch waited on failed and dropped its
1709
+ * record. For a caller about to hand a tracked instance to an app: until its build settles
1710
+ * the instance is `'starting'` with no database, and a failed launch leaves no instance.
1711
+ * Never rejects; a failed build is reported by the call that started it.
1712
+ */
1713
+ async whenRuntimeBuilt(strandId: string): Promise<StrandInstance | undefined> {
1714
+ await this.settleRuntimeBuilds(strandId);
1715
+ return this.instances.get(strandId);
1716
+ }
1717
+
1718
+ /**
1719
+ * Hand a launch's database to the app: set `instance.database`, retire the gate that
1720
+ * held it, release every {@link whenWritable} waiter, and — when the launch had already
1721
+ * been reported gated — flip it `'active'` and announce it through the retained config's
1722
+ * `onWritable`. "Reported gated" is any status but `'starting'`: `'syncing'`, or the
1723
+ * `'idle'` the hibernation manager's idle timer flips a long-gated joiner to (see
1724
+ * {@link isAwaitingFirstSync}) — the Header arriving after that flip must announce too,
1725
+ * or an app hanging on `strand:writable` never learns the strand opened. A publish that
1726
+ * lands while the instance is still `'starting'` (the Header arrived during the rest of
1727
+ * bring-up) announces nothing: `startStrand` / `resumeStrand` are about to report the
1728
+ * strand `'active'`, and `strand:writable` is defined as the follow-up to a gated
1729
+ * launch, never a duplicate of `strand:started`.
1730
+ *
1731
+ * It is also the one seam where a WITHHELD database becomes available, so it kicks the
1732
+ * membership reconciler: on a gated launch that loop's own immediate pass ran while
1733
+ * `instance.database` was still unset, found no database, and would otherwise not try
1734
+ * again until its retry timer fired. A launch that was never gated publishes here BEFORE
1735
+ * the reconciler is constructed, so there is nothing registered to kick and the loop's own
1736
+ * immediate pass already sees the database — which is why the kick needs no `wasGated`
1737
+ * test of its own.
1738
+ */
1739
+ private publishDatabase(instance: StrandInstance, database: StrandDatabase): void {
1740
+ const strandId = instance.strandId;
1741
+ const gate = this.firstSyncGates.get(strandId);
1742
+ if (gate) {
1743
+ gate.open();
1744
+ this.firstSyncGates.delete(strandId);
1745
+ }
1746
+ instance.database = database;
1747
+ const wasGated = instance.status !== 'starting';
1748
+ if (wasGated) {
1749
+ // A Header delivered by a peer is strand activity; a publish during bring-up is not
1750
+ // (see the build's end in `buildStrandRuntime`).
1751
+ instance.lastActivity = new Date();
1752
+ instance.status = 'active';
1753
+ log('Strand %s is now writable: Strand.Header held', strandId);
1754
+ }
1755
+ const waiters = this.writableWaiters.get(strandId);
1756
+ if (waiters) {
1757
+ this.writableWaiters.delete(strandId);
1758
+ waiters.forEach((waiter) => waiter.resolve());
1759
+ }
1760
+ if (wasGated) {
1761
+ this.launchConfigs.get(strandId)?.onWritable?.(strandId);
1762
+ }
1763
+ // Not awaited: publishing must stay synchronous, and the pass chains on the loop's own
1764
+ // tail, so it cannot overlap a pass already in flight.
1765
+ void this.membershipReconcilers.get(strandId)?.reconcile();
1766
+ }
1767
+
1768
+ /** Reject every {@link whenWritable} waiter for a strand that is going away. */
1769
+ private rejectWritableWaiters(strandId: string, reason: string): void {
1770
+ const waiters = this.writableWaiters.get(strandId);
1771
+ if (!waiters) {
1772
+ return;
1773
+ }
1774
+ this.writableWaiters.delete(strandId);
1775
+ waiters.forEach((waiter) => waiter.reject(new Error(`Strand ${strandId} ${reason} before becoming writable`)));
1776
+ }
1777
+
1778
+ /**
1779
+ * Undo {@link foundExistingStrand}'s founder flip after the founding it promised
1780
+ * failed, so the retained config again says what this instance actually runs as (a
1781
+ * joiner) and the next founder request re-attempts the bootstrap. The resolved party
1782
+ * key stays retained — it is this party's identity for the strand either way. No-op
1783
+ * when the strand is untracked or its config does not found.
1784
+ */
1785
+ withdrawFounderRequest(strandId: string): void {
1786
+ const config = this.launchConfigs.get(strandId);
1787
+ if (config?.founder === true) {
1788
+ this.launchConfigs.set(strandId, { ...config, founder: false });
1789
+ }
1790
+ }
1791
+
1792
+ /**
1793
+ * Run the (idempotent) founder bootstrap against a tracked strand's LIVE
1794
+ * database, independently of what the retained launch config says.
1795
+ *
1796
+ * Separate from {@link foundExistingStrand} because a caller that resolved
1797
+ * `'needs-resume'` and woke the strand must not assume the wake's own rebuild
1798
+ * founded it: `HibernationManager` COALESCES wakes, so a wake already in flight
1799
+ * when the config flipped had already read the PRE-flip config and rebuilt as a
1800
+ * joiner. Re-running the bootstrap costs one insert-if-absent probe per table
1801
+ * and is the only thing that makes "founding resolves once the Header is
1802
+ * written" true on that path.
1803
+ *
1804
+ * A strand still `'syncing'` (its database withheld behind the first-sync gate) is
1805
+ * founded through the gate: the bootstrap writes the Header this machine was waiting
1806
+ * to receive, so a successful run publishes the database and the strand goes
1807
+ * `'active'` — the one legitimate way a joiner launch becomes writable without a peer.
1808
+ *
1809
+ * @throws when the strand is not tracked, or is still quiesced (no live
1810
+ * database) — both mean the bootstrap did NOT run, which a founder request
1811
+ * must never swallow.
1812
+ */
1813
+ async ensureFounderBootstrap(strandId: string): Promise<void> {
1814
+ const instance = this.instances.get(strandId);
1815
+ if (!instance) {
1816
+ throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: not tracked`);
1817
+ }
1818
+ const gate = this.firstSyncGates.get(strandId);
1819
+ const database = instance.database ?? gate?.database;
1820
+ if (!database) {
1821
+ throw new Error(
1822
+ `Cannot run the founder bootstrap for strand ${strandId}: it is quiesced, so there ` +
1823
+ 'is no live database to write to — resume it first.'
1824
+ );
1825
+ }
1826
+ // Forward the RETAINED config's party key: a foundExistingStrand that flipped a
1827
+ // joiner launch to founder may have resolved a key the live database's captured
1828
+ // config (built at the original launch) never saw.
1829
+ await database.ensureFounderBootstrap(this.launchConfigs.get(strandId)?.partyMemberPrivateKey);
1830
+ if (!instance.database) {
1831
+ this.publishDatabase(instance, database);
1832
+ }
1833
+ }
1834
+
1835
+ /**
1836
+ * Stop a strand instance
1837
+ */
1838
+ async stopStrand(strandId: string): Promise<void> {
1839
+ const instance = this.instances.get(strandId);
1840
+ if (!instance) {
1841
+ log('Strand %s not found', strandId);
1842
+ return;
1843
+ }
1844
+
1845
+ log('Stopping strand instance: %s', strandId);
1846
+ instance.status = 'stopping';
1847
+ this.rejectWritableWaiters(strandId, 'was stopped');
1848
+
1849
+ try {
1850
+ await this.releaseRuntime(instance);
1851
+ instance.status = 'stopped';
1852
+ this.instances.delete(strandId);
1853
+ this.launchConfigs.delete(strandId);
1854
+ // Storage is released only here — NOT in releaseRuntime, which a quiesce shares.
1855
+ await this.disposeStrandStorage(strandId);
1856
+ log('Strand %s stopped successfully', strandId);
1857
+ } catch (error) {
1858
+ instance.status = 'error';
1859
+ instance.error = error instanceof Error ? error.message : String(error);
1860
+ log('Error stopping strand %s: %s', strandId, instance.error);
1861
+ throw error;
1862
+ }
1863
+ }
1864
+
1865
+ /**
1866
+ * Stop all strand instances
1867
+ */
1868
+ async stopAll(): Promise<void> {
1869
+ this.stopping = true;
1870
+ log('Stopping all strand instances (%d)', this.instances.size);
1871
+
1872
+ const stopPromises = Array.from(this.instances.keys()).map(id =>
1873
+ this.stopStrand(id).catch(err => {
1874
+ log('Error stopping strand %s during shutdown: %s', id, err);
1875
+ })
1876
+ );
1877
+
1878
+ await Promise.all(stopPromises);
1879
+ this.stopping = false;
1880
+ log('All strand instances stopped');
1881
+ }
1882
+ }
1883
+