@serfab/cadre-core 0.12.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +23 -8
  2. package/dist/bootstrap-peer-store-file.js +2 -2
  3. package/dist/bootstrap-peer-store.d.ts +32 -12
  4. package/dist/bootstrap-peer-store.js +41 -15
  5. package/dist/bootstrap-peer-store.js.map +1 -1
  6. package/dist/cadre-node.d.ts +883 -102
  7. package/dist/cadre-node.js +1800 -209
  8. package/dist/cadre-node.js.map +1 -1
  9. package/dist/control-authorization.d.ts +2 -2
  10. package/dist/control-authorization.js +1 -0
  11. package/dist/control-authorization.js.map +1 -1
  12. package/dist/control-cohort.d.ts +13 -26
  13. package/dist/control-cohort.js +0 -26
  14. package/dist/control-cohort.js.map +1 -1
  15. package/dist/control-database.d.ts +318 -44
  16. package/dist/control-database.js +516 -74
  17. package/dist/control-database.js.map +1 -1
  18. package/dist/control-read-retry.d.ts +40 -11
  19. package/dist/control-read-retry.js +48 -15
  20. package/dist/control-read-retry.js.map +1 -1
  21. package/dist/control-retry.d.ts +58 -3
  22. package/dist/control-retry.js +58 -3
  23. package/dist/control-retry.js.map +1 -1
  24. package/dist/control-schema.d.ts +1 -1
  25. package/dist/control-schema.js +133 -21
  26. package/dist/control-schema.js.map +1 -1
  27. package/dist/control-write-retry.d.ts +80 -11
  28. package/dist/control-write-retry.js +168 -29
  29. package/dist/control-write-retry.js.map +1 -1
  30. package/dist/enrolled-machine-store-file.d.ts +13 -0
  31. package/dist/enrolled-machine-store-file.js +41 -0
  32. package/dist/enrolled-machine-store-file.js.map +1 -0
  33. package/dist/enrolled-machine-store.d.ts +81 -0
  34. package/dist/enrolled-machine-store.js +225 -0
  35. package/dist/enrolled-machine-store.js.map +1 -0
  36. package/dist/file-durable-slot.js +5 -3
  37. package/dist/file-durable-slot.js.map +1 -1
  38. package/dist/index.d.ts +17 -8
  39. package/dist/index.js +54 -9
  40. package/dist/index.js.map +1 -1
  41. package/dist/membership-connection-gater.d.ts +15 -1
  42. package/dist/membership-connection-gater.js +9 -3
  43. package/dist/membership-connection-gater.js.map +1 -1
  44. package/dist/node-local-snapshot.d.ts +21 -7
  45. package/dist/node-local-snapshot.js +30 -9
  46. package/dist/node-local-snapshot.js.map +1 -1
  47. package/dist/peer-dial.d.ts +143 -0
  48. package/dist/peer-dial.js +176 -0
  49. package/dist/peer-dial.js.map +1 -0
  50. package/dist/{strand-backfill.d.ts → peer-join-backfill.d.ts} +49 -20
  51. package/dist/{strand-backfill.js → peer-join-backfill.js} +133 -52
  52. package/dist/peer-join-backfill.js.map +1 -0
  53. package/dist/relay-addrs.d.ts +160 -49
  54. package/dist/relay-addrs.js +287 -60
  55. package/dist/relay-addrs.js.map +1 -1
  56. package/dist/relay-reservation.d.ts +71 -3
  57. package/dist/relay-reservation.js +116 -8
  58. package/dist/relay-reservation.js.map +1 -1
  59. package/dist/seed-bootstrap.d.ts +30 -1
  60. package/dist/seed-bootstrap.js +62 -17
  61. package/dist/seed-bootstrap.js.map +1 -1
  62. package/dist/storage-scope.d.ts +62 -0
  63. package/dist/storage-scope.js +75 -0
  64. package/dist/storage-scope.js.map +1 -0
  65. package/dist/strand-database.d.ts +41 -11
  66. package/dist/strand-database.js +48 -18
  67. package/dist/strand-database.js.map +1 -1
  68. package/dist/strand-first-sync-gate.d.ts +152 -0
  69. package/dist/strand-first-sync-gate.js +208 -0
  70. package/dist/strand-first-sync-gate.js.map +1 -0
  71. package/dist/strand-formation-manager.d.ts +99 -5
  72. package/dist/strand-formation-manager.js +119 -8
  73. package/dist/strand-formation-manager.js.map +1 -1
  74. package/dist/strand-formation-protocol.d.ts +97 -3
  75. package/dist/strand-formation-protocol.js +108 -2
  76. package/dist/strand-formation-protocol.js.map +1 -1
  77. package/dist/strand-instance-manager.d.ts +389 -19
  78. package/dist/strand-instance-manager.js +678 -65
  79. package/dist/strand-instance-manager.js.map +1 -1
  80. package/dist/strand-member-key.d.ts +20 -12
  81. package/dist/strand-member-key.js +20 -12
  82. package/dist/strand-member-key.js.map +1 -1
  83. package/dist/strand-member-registry.js +3 -0
  84. package/dist/strand-member-registry.js.map +1 -1
  85. package/dist/strand-membership-reconciler.d.ts +361 -0
  86. package/dist/strand-membership-reconciler.js +548 -0
  87. package/dist/strand-membership-reconciler.js.map +1 -0
  88. package/dist/strand-membership-writer.d.ts +143 -11
  89. package/dist/strand-membership-writer.js +419 -135
  90. package/dist/strand-membership-writer.js.map +1 -1
  91. package/dist/strand-network-config.d.ts +116 -0
  92. package/dist/strand-network-config.js +192 -0
  93. package/dist/strand-network-config.js.map +1 -0
  94. package/dist/strand-revocation-enforcer.d.ts +392 -0
  95. package/dist/strand-revocation-enforcer.js +533 -0
  96. package/dist/strand-revocation-enforcer.js.map +1 -0
  97. package/dist/strand-solicitation.d.ts +17 -1
  98. package/dist/strand-solicitation.js +13 -2
  99. package/dist/strand-solicitation.js.map +1 -1
  100. package/dist/strand-wake-protocol.d.ts +11 -7
  101. package/dist/strand-wake-protocol.js +20 -52
  102. package/dist/strand-wake-protocol.js.map +1 -1
  103. package/dist/strand-watcher.js +12 -1
  104. package/dist/strand-watcher.js.map +1 -1
  105. package/dist/timed-step.d.ts +12 -0
  106. package/dist/timed-step.js +28 -0
  107. package/dist/timed-step.js.map +1 -0
  108. package/dist/timeout-scheduler.d.ts +19 -0
  109. package/dist/timeout-scheduler.js +22 -0
  110. package/dist/timeout-scheduler.js.map +1 -0
  111. package/dist/types.d.ts +472 -46
  112. package/dist/types.js +7 -1
  113. package/dist/types.js.map +1 -1
  114. package/package.json +16 -8
  115. package/src/bootstrap-peer-store-file.ts +2 -2
  116. package/src/bootstrap-peer-store.ts +69 -27
  117. package/src/cadre-node.ts +1926 -228
  118. package/src/control-authorization.ts +2 -1
  119. package/src/control-cohort.ts +122 -130
  120. package/src/control-database.ts +586 -74
  121. package/src/control-read-retry.ts +202 -167
  122. package/src/control-retry.ts +297 -190
  123. package/src/control-schema.ts +133 -21
  124. package/src/control-write-retry.ts +525 -383
  125. package/src/enrolled-machine-store-file.ts +43 -0
  126. package/src/enrolled-machine-store.ts +280 -0
  127. package/src/file-durable-slot.ts +5 -3
  128. package/src/index.ts +134 -15
  129. package/src/membership-connection-gater.ts +9 -3
  130. package/src/node-local-snapshot.ts +38 -9
  131. package/src/peer-addr-book.ts +251 -250
  132. package/src/peer-dial.ts +215 -0
  133. package/src/{strand-backfill.ts → peer-join-backfill.ts} +478 -389
  134. package/src/relay-addrs.ts +458 -218
  135. package/src/relay-reservation.ts +951 -828
  136. package/src/seed-bootstrap.ts +76 -18
  137. package/src/storage-scope.ts +78 -0
  138. package/src/strand-database.ts +61 -21
  139. package/src/strand-first-sync-gate.ts +258 -0
  140. package/src/strand-formation-manager.ts +162 -8
  141. package/src/strand-formation-protocol.ts +850 -692
  142. package/src/strand-instance-manager.ts +877 -74
  143. package/src/strand-member-key.ts +20 -12
  144. package/src/strand-member-registry.ts +3 -0
  145. package/src/strand-membership-reconciler.ts +680 -0
  146. package/src/strand-membership-writer.ts +550 -190
  147. package/src/strand-network-config.ts +250 -0
  148. package/src/strand-revocation-enforcer.ts +653 -0
  149. package/src/strand-solicitation.ts +499 -471
  150. package/src/strand-wake-protocol.ts +21 -66
  151. package/src/strand-watcher.ts +325 -314
  152. package/src/timed-step.ts +28 -0
  153. package/src/timeout-scheduler.ts +28 -0
  154. package/src/types.ts +487 -52
  155. package/dist/strand-backfill.js.map +0 -1
@@ -3,8 +3,28 @@ import type { PrivateKey } from '@libp2p/interface';
3
3
  import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
4
4
  import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
5
5
  import { StrandDatabase } from './strand-database.js';
6
- import { StrandBackfill, type StrandBackfillConfig } from './strand-backfill.js';
6
+ import { PeerJoinBackfill, type PeerJoinBackfillConfig } from './peer-join-backfill.js';
7
+ import {
8
+ StrandRevocationEnforcer,
9
+ createRevocationConnectionGater,
10
+ readStrandRevocationRows,
11
+ type StrandRevocationEnforcementConfig
12
+ } from './strand-revocation-enforcer.js';
13
+ import {
14
+ StrandMembershipReconciler,
15
+ type PendingMembershipInviteSource,
16
+ type StrandMembershipReconciliationConfig
17
+ } from './strand-membership-reconciler.js';
18
+ import { removeMemberPeer } from './strand-membership-writer.js';
19
+ import { strandMemberKeyPair } from './strand-member-key.js';
7
20
  import { assertSchemaSignature } from './schema-verification.js';
21
+ import {
22
+ StrandFirstSyncGate,
23
+ StrandAwaitingFirstSyncError,
24
+ strandFirstSyncComplete,
25
+ DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS,
26
+ type StrandFirstSyncConfig
27
+ } from './strand-first-sync-gate.js';
8
28
  import type {
9
29
  StrandInstance,
10
30
  StrandRow,
@@ -17,13 +37,48 @@ import type {
17
37
  RawStorageProvider,
18
38
  Libp2pNodeWithRepo
19
39
  } from './types.js';
20
- import { resolveStrandClusterSize, STRAND_CLUSTER_POLICY } from './types.js';
21
- import { resolveListenAddrs } from './relay-addrs.js';
22
- import { resolveAnnounceAddrs } from './announce-addrs.js';
40
+ import { resolveStrandClusterSize, strandClusterPolicy } from './types.js';
41
+ import { strandNodeAddrs } from './strand-network-config.js';
42
+ import { superviseRelayReservation, type RelayReservationSupervisor } from './relay-reservation.js';
23
43
 
24
44
  const log = debug('sereus:cadre:strand-manager');
25
45
  const timing = debug('sereus:cadre:timing');
26
46
 
47
+ /** One caller blocked in {@link StrandInstanceManager.whenWritable}. */
48
+ interface WritableWaiter {
49
+ resolve: () => void;
50
+ reject: (error: Error) => void;
51
+ }
52
+
53
+ /**
54
+ * Whether a tracked instance is launched but waiting for its first sync: the runtime is
55
+ * up (`libp2pNode` set) while the database is still withheld (`database` unset). That
56
+ * is the first-sync gate's state regardless of what `status` says — the hibernation
57
+ * manager's idle timer can flip a long-gated joiner to `'idle'` without changing it.
58
+ *
59
+ * NOTE: `status` is therefore not the source of truth for "gated": a joiner that reaches
60
+ * nobody for `idleTimeout` (5 min at the interactive hint) reads `'idle'`, then
61
+ * `'hibernating'`, while still holding no database. Harmless today — every decision the
62
+ * runtime makes goes through this predicate, {@link StrandInstanceManager.publishDatabase}
63
+ * announces on the idle-flipped instance exactly as on a `'syncing'` one, and an app that
64
+ * shows "waiting for the other member" on `'syncing'` merely sees "idle" instead after
65
+ * five minutes. If an app ever needs the gated LABEL to survive the idle flip, make the
66
+ * hibernation manager's idle transition preserve `'syncing'` rather than adding a flag.
67
+ */
68
+ export function isAwaitingFirstSync(instance: StrandInstance): boolean {
69
+ return instance.libp2pNode !== undefined && instance.database === undefined;
70
+ }
71
+
72
+ /**
73
+ * The status a LIVE instance (runtime rebuilt or woken) should report: `'active'` once its
74
+ * database is published, `'syncing'` while the first-sync gate still withholds it. Every
75
+ * site that would otherwise write `'active'` after a build or a wake goes through this,
76
+ * so a gated joiner never reads as writable.
77
+ */
78
+ export function liveStrandStatus(instance: StrandInstance): 'active' | 'syncing' {
79
+ return instance.database ? 'active' : 'syncing';
80
+ }
81
+
27
82
  /**
28
83
  * Configuration for starting a strand instance
29
84
  */
@@ -47,10 +102,25 @@ export interface StartStrandConfig {
47
102
  /**
48
103
  * Whether this node founds the strand (vs. joins it). Forwarded to the
49
104
  * StrandDatabase so the founder bootstrap (Header, founding Member/Manager)
50
- * runs once at bring-up. Joiners leave this unset and write nothing. See
51
- * {@link StrandConfig.founder}.
105
+ * runs once at bring-up. Joiners leave this unset and write nothing. Callers
106
+ * resolve it BEFORE calling `startStrand` (`CadreNode.launchStrand` derives it
107
+ * from the row's `FounderOwnerKey` when no explicit flag is given — see
108
+ * {@link StrandConfig.founder}); a founder request that arrives while the
109
+ * instance is already tracked goes through {@link StrandInstanceManager.foundExistingStrand},
110
+ * never through a repeat `startStrand`.
52
111
  */
53
112
  founder?: boolean;
113
+ /**
114
+ * THIS party's own strand membership private key (base64 protobuf) for a closed
115
+ * strand — the identity the founder bootstrap derives `Member.Key` /
116
+ * `Manager.MemberKey` from. Resolved by the caller (`CadreNode.launchStrand`: an
117
+ * explicit attach-time key, else the control-layer `StrandPartyKey` row, healed by
118
+ * mint-on-launch for the founding machine) and retained with the launch config, so a
119
+ * hibernation wake rebuilds under the same identity. Deliberately NOT the strand
120
+ * row's shared `MemberPrivateKey` (see `StrandDatabaseConfig.partyMemberPrivateKey`).
121
+ * Absent for open strands, and for joiners with no persisted party key.
122
+ */
123
+ partyMemberPrivateKey?: string;
54
124
  /**
55
125
  * Number of nodes Optimystic is told this strand's replication cluster should
56
126
  * have. Same rule as {@link CadreNodeConfig.strandClusterSize}, which CadreNode
@@ -62,24 +132,143 @@ export interface StartStrandConfig {
62
132
  */
63
133
  clusterSize?: number;
64
134
  /**
65
- * Tuning for the strand peer-join block catch-up ({@link StrandBackfill}),
135
+ * Machines that SERVE THIS STRAND (this node included), for the strand node's
136
+ * block-repair corroboration yardstick — see `resolveRepairYardstick`. It must be
137
+ * an authenticated per-strand count: the machines that actually run a node for this
138
+ * strand, not the machines that exist.
139
+ *
140
+ * **The party's enrolled-machine count is NOT this number and must never be passed
141
+ * here.** A strand launches only on machines whose embedding app registered its sApp
142
+ * config (`CadreNode.addStrand`), so a closed strand shared by two machines of a
143
+ * three-machine party is served by two. Passing three over-declares, and
144
+ * over-declaring is the unsafe direction: at a declaration of 3 or more Optimystic
145
+ * pins the repair corroboration floor at two corroborating peers, which a cohort
146
+ * that can only ever field one peer can never reach — so the strand can never repair
147
+ * a block (`cluster-fetch:no-quorum`, surfacing as reads failing with `Missing
148
+ * block`). That regression is why this field was renamed off "enrolledMachines":
149
+ * `bug-strand-yardstick-counts-party-machines`.
150
+ *
151
+ * **Nothing feeds it in production today.** No authenticated per-strand serving count
152
+ * exists yet, so `CadreNode` passes nothing and the strand node runs the frozen
153
+ * `STRAND_CLUSTER_POLICY` — declaring no yardstick, which leaves the known,
154
+ * upstream-tracked single-voter exposure
155
+ * (`backlog/debt-read-repair-single-voter-corroboration`). Building the count is
156
+ * `backlog/feat-strand-yardstick-from-serving-machines`; this field and its
157
+ * threading are the seam it plugs into.
158
+ *
159
+ * Volatile when a source does exist: re-resolve it on every resume beside the cohort
160
+ * seed, since machines join and leave a strand while it hibernates. Omitting it means
161
+ * "this node does not know", which declares nothing.
162
+ */
163
+ servingMachines?: number;
164
+ /**
165
+ * Tuning for the strand peer-join block catch-up ({@link PeerJoinBackfill}),
66
166
  * forwarded from {@link CadreNodeConfig.strandBackfill}. When the strand's
67
167
  * libp2p node connects to a peer this runtime has not yet caught up, every
68
168
  * block in the strand's own raw store is pushed to it. Runs only on strands
69
169
  * with per-strand storage; `{ enabled: false }` restores the pre-existing
70
170
  * no-backfill behaviour.
71
171
  */
72
- backfill?: StrandBackfillConfig;
172
+ backfill?: PeerJoinBackfillConfig;
173
+ /**
174
+ * Tuning for the CLOSED-strand revoked-peer gate
175
+ * ({@link StrandRevocationEnforcer}), forwarded from
176
+ * {@link CadreNodeConfig.strandRevocationEnforcement}. Armed only for
177
+ * `strandRow.Type === 'c'` (an open strand has no membership rows to derive a
178
+ * deny set from); `{ enabled: false }` restores the pre-existing behaviour
179
+ * (a removed party's peers keep being served).
180
+ */
181
+ revocationEnforcement?: StrandRevocationEnforcementConfig;
182
+
183
+ /**
184
+ * The staged formation membership invitation for THIS strand — the seam the
185
+ * bring-up membership reconciler (`strand-membership-reconciler.ts`) reads to
186
+ * redeem the party's `Strand.Member` seat, and clears once the invitation is
187
+ * spent, burned, or dead. `CadreNode.launchStrand` wires it over its in-memory
188
+ * `pendingMembershipInvites` map; read lazily per pass, so a re-formation that
189
+ * replaces the entry between passes is picked up. Only consulted for closed
190
+ * strands launched with a {@link partyMemberPrivateKey}; harmless otherwise.
191
+ */
192
+ pendingMembershipInvite?: PendingMembershipInviteSource;
193
+
194
+ /**
195
+ * Tuning for the CLOSED-strand membership reconciler, forwarded from
196
+ * {@link CadreNodeConfig.strandMembershipReconciliation}. Armed only for
197
+ * closed strands launched with a {@link partyMemberPrivateKey};
198
+ * `{ enabled: false }` disarms the loop (test fixtures that hand-drive the
199
+ * membership writers). When it names no `pollIntervalMs` the reconciler
200
+ * mirrors {@link revocationEnforcement}'s cadence — as its IDLE cadence and
201
+ * the cap of its unfinished-join retry ladder, not as a flat retry interval.
202
+ */
203
+ membershipReconciliation?: StrandMembershipReconciliationConfig;
204
+
205
+ /**
206
+ * Called when THIS node's own peer id turns up in the strand's revoked set —
207
+ * this node's party was removed from the closed strand, or left it. Fires once
208
+ * per removal (re-armed if the party is re-admitted; a resume also rebuilds the
209
+ * enforcer and may legitimately re-fire), and only for closed strands with the
210
+ * gate armed. `CadreNode`
211
+ * wires it to its `strand:revoked` event; nothing is stopped or torn down on
212
+ * this node's behalf — what to do about it is the app's call.
213
+ */
214
+ onSelfRevoked?: (strandId: string) => void;
215
+
216
+ /**
217
+ * Tuning for the JOINING machine's first-sync write gate (`strand-first-sync-gate.ts`),
218
+ * forwarded from {@link CadreNodeConfig.strandFirstSync}: the Header probe cadence while
219
+ * a non-founder launch is `'syncing'`, and the default budget {@link whenWritable} waits
220
+ * before rejecting. Founders and machines that already hold the Header are never gated.
221
+ */
222
+ firstSync?: StrandFirstSyncConfig;
223
+
224
+ /**
225
+ * Called when a launch that came up `'syncing'` (see {@link firstSync}) becomes
226
+ * writable: the strand's `Strand.Header` arrived from a peer — or a founder request
227
+ * ran the bootstrap against the gated database — and `instance.database` is now set.
228
+ * Never called for a launch that was writable when `startStrand`/`resumeStrand`
229
+ * resolved. `CadreNode` wires it to its `strand:writable` event. Retained with the
230
+ * launch config, so a hibernation resume that comes up gated announces too.
231
+ */
232
+ onWritable?: (strandId: string) => void;
233
+
234
+ /**
235
+ * Re-announce this strand's delegate peer id (`delegatePeerId` — the strand
236
+ * node's own transport peer id) to ONE relay (`relayAddr`, a direct dial addr
237
+ * with a trailing `/p2p/<relayPeerId>`), unthrottled. Awaited before every
238
+ * relay-reservation RE-drive of that relay — never before the first attempt,
239
+ * which the launch/resume seed pass has already announced for.
240
+ *
241
+ * Why: a party control node running the relay server admits the strand node on
242
+ * an in-memory delegate grant that a relay restart drops, so a re-drive that did
243
+ * not re-announce first would be denied at the relay's gate. `CadreNode` wires
244
+ * its own control-mesh announce here; a throw or rejection is logged by the
245
+ * supervisor and the re-drive still runs. Retained with the launch config, so a
246
+ * hibernation wake's rebuilt supervisors carry it too.
247
+ */
248
+ announceDelegateToRelay?: (strandId: string, relayAddr: string, delegatePeerId: string) => Promise<void>;
73
249
  }
74
250
 
75
251
  /**
76
- * Volatile inputs re-resolved when resuming a quiesced strand. The cohort
77
- * discovery seed is the only one — it grows as peers are learned since the
78
- * strand first launched.
252
+ * Volatile inputs re-resolved when resuming a quiesced strand — the ones that can
253
+ * have moved since it last ran, and that the rebuilt libp2p node freezes again. Each
254
+ * one omitted keeps the value the retained launch config already holds, so a resume
255
+ * that passes nothing rebuilds the strand exactly as it last ran.
79
256
  */
80
257
  export interface ResumeStrandOverrides {
81
- /** Freshly-resolved cohort discovery seed (multiaddr strings). */
258
+ /**
259
+ * Freshly-resolved cohort discovery seed (multiaddr strings). Grows as peers are
260
+ * learned since the strand first launched.
261
+ */
82
262
  bootstrapNodes?: string[];
263
+ /**
264
+ * Freshly-read count of the machines serving this strand, for the repair yardstick
265
+ * (see {@link StartStrandConfig.servingMachines} — including why the party's
266
+ * enrolled-machine count is not it). Moves whenever a machine starts or stops
267
+ * serving the strand, which a hibernating strand does not otherwise notice. Nothing
268
+ * passes it today. Omitting it here does NOT mean "unknown" the way omitting it from
269
+ * {@link StartStrandConfig} does — it retains the last value; see `resumeStrand`.
270
+ */
271
+ servingMachines?: number;
83
272
  }
84
273
 
85
274
  /**
@@ -162,7 +351,53 @@ export class StrandInstanceManager {
162
351
  * with a fresh caught-up-peer memo, which is intended (a resumed node may have
163
352
  * missed writes).
164
353
  */
165
- private backfills: Map<string, StrandBackfill> = new Map();
354
+ private backfills: Map<string, PeerJoinBackfill> = new Map();
355
+ /**
356
+ * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
357
+ * id. Same lifecycle rationale as {@link backfills}: created in
358
+ * `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so quiesce →
359
+ * resume rebuilds it with a fresh (initially empty, fail-open) snapshot.
360
+ */
361
+ private revocationEnforcers: Map<string, StrandRevocationEnforcer> = new Map();
362
+ /**
363
+ * The per-strand membership reconciler (closed strands launched with a party
364
+ * key only), keyed by strand id. Same lifecycle rationale as {@link backfills}:
365
+ * created in `buildStrandRuntime`, stopped and dropped in `releaseRuntime`, so
366
+ * quiesce → resume rebuilds it and re-runs the idempotent join ladder (redeem a
367
+ * staged invitation, write this machine's own `MemberPeer` binding) from scratch.
368
+ */
369
+ private membershipReconcilers: Map<string, StrandMembershipReconciler> = new Map();
370
+ /**
371
+ * The per-strand first-sync gate (`strand-first-sync-gate.ts`), keyed by strand id:
372
+ * holds a launch's `StrandDatabase` while it is NOT yet published on the instance.
373
+ * Present from the moment the database object exists in `buildStrandRuntime` until
374
+ * {@link publishDatabase} hands it to the app — so it doubles as the rollback handle
375
+ * for a failed `initialize()` — and, for a joiner that holds no `Strand.Header` yet,
376
+ * for as long as the launch stays `'syncing'`. Same lifecycle as {@link backfills}:
377
+ * stopped (its database closed) and dropped in `releaseRuntime`, so a quiesce →
378
+ * resume rebuild re-probes over the same store. An entry and `instance.database`
379
+ * are mutually exclusive.
380
+ */
381
+ private firstSyncGates: Map<string, StrandFirstSyncGate> = new Map();
382
+ /**
383
+ * Callers blocked in {@link whenWritable}, keyed by strand id. Resolved by
384
+ * {@link publishDatabase}, rejected by {@link stopStrand}; each waiter also carries
385
+ * its own timeout. Deliberately NOT cleared by `releaseRuntime`: a waiter outlives a
386
+ * quiesce → resume cycle, since the rebuild is what may finally publish the database.
387
+ */
388
+ private writableWaiters: Map<string, Set<WritableWaiter>> = new Map();
389
+ /**
390
+ * The per-strand relay-reservation supervisors — ONE PER CONFIGURED RELAY, each
391
+ * over the node's own bare `/p2p-circuit` listener for that relay
392
+ * (`strand-network-config.ts`) — keyed by strand id. Same lifecycle rationale as
393
+ * {@link backfills}: started in `buildStrandRuntime` right after the libp2p node
394
+ * exists, stopped FIRST and dropped in `releaseRuntime` (before the database
395
+ * closes and the node stops, so no re-drive dials a node being torn down), so
396
+ * quiesce → resume rebuilds them over the new node. What makes a strand node's
397
+ * lost relay slot come back on its own — the control node has the same loop in
398
+ * `CadreNode.reserveRelays`.
399
+ */
400
+ private relaySupervisors: Map<string, RelayReservationSupervisor[]> = new Map();
166
401
  /**
167
402
  * The resolved (cache-wrapped) raw storage per strand id — the instance's OWN
168
403
  * store, resolved once in `startStrand` and held until `stopStrand` disposes it.
@@ -235,6 +470,14 @@ export class StrandInstanceManager {
235
470
  }
236
471
 
237
472
  if (this.instances.has(strandId)) {
473
+ // Callers resolve founder-ness BEFORE reaching here (CadreNode.launchStrand) and
474
+ // honor a founder request on a tracked instance via foundExistingStrand — so a
475
+ // founder flag arriving at this early return against a non-founder retained
476
+ // config is a dropped bootstrap, and must never again be silent.
477
+ if (config.founder === true && this.launchConfigs.get(strandId)?.founder !== true) {
478
+ log('startStrand: strand %s is already running but was NOT launched as founder — ' +
479
+ 'this early return DROPS the founder request; use foundExistingStrand', strandId);
480
+ }
238
481
  log('Strand %s already running', strandId);
239
482
  return this.instances.get(strandId)!;
240
483
  }
@@ -309,8 +552,9 @@ export class StrandInstanceManager {
309
552
  * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
310
553
  * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
311
554
  * launch) and `resumeStrand` (rehydrating a quiesced instance). Reads all
312
- * volatile inputs (bootstrapNodes, network, profile, privateKey, sApp config)
313
- * from `config`, so the caller controls the cohort-derived values. Storage is the
555
+ * volatile inputs (bootstrapNodes, servingMachines, network, profile,
556
+ * privateKey, sApp config) from `config`, so the caller controls the
557
+ * cohort-derived values. Storage is the
314
558
  * one input it does NOT re-read from `config`: that belongs to the instance and
315
559
  * comes from `strandStorages`.
316
560
  */
@@ -332,17 +576,72 @@ export class StrandInstanceManager {
332
576
  // Determine relay mode: if explicitly set in config, use that;
333
577
  // otherwise default to true for storage profile nodes.
334
578
  const enableRelay = config.network?.enableRelay ?? (config.profile === 'storage');
335
- // The CONFIGURED relay route (the default), deliberately NOT the control node's
336
- // `'search'` route: nothing drives an explicit reservation for a strand node, so
337
- // a bare `/p2p-circuit` search entry would register a pending reservation that
338
- // never gets filled and leave the strand node undialable. The ordering hazard
339
- // that pushed the control node onto the search route does not exist here — a
340
- // strand node's protocol ids are namespaced `/optimystic/strand-<id>/…`, so a
341
- // relay dialed from inside `libp2p.start()` is never in the strand's cohort and
342
- // cannot refuse its database bring-up. See `relay-addrs.ts`.
343
- const listenAddrs = resolveListenAddrs(config.network);
579
+ // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
580
+ // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
581
+ // bind one port), the announce config is dropped (it names the control node's
582
+ // address), and every configured relay becomes one bare `/p2p-circuit` SEARCH
583
+ // listener plus a dial addr in `relayAddrs` for the per-relay supervisor started
584
+ // below — the same route the control node takes, because the configured
585
+ // `<relay>/p2p-circuit` shape loses its address on a relay restart, a hangup, or
586
+ // libp2p's own reservation refresh and recovers from none of them. See
587
+ // `strand-network-config.ts` for all three, and `relay-reservation.ts`.
588
+ //
589
+ // `relayAddrs` is destructured OFF: it is not a `createLibp2pNode` option, and
590
+ // spreading it would hand db-p2p a key it does not know.
591
+ const { relayAddrs: relayDialAddrs = [], ...addrOptions } = strandNodeAddrs(config.network);
592
+
593
+ // The CLOSED-strand revoked-peer gate (see strand-revocation-enforcer.ts).
594
+ // Constructed BEFORE the libp2p node because the node's options embed its
595
+ // predicates; its deny set starts empty (admit everything — fail-open, same
596
+ // posture as bring-up everywhere here) and is first populated after the
597
+ // strand database initializes below. Registered in the map immediately so
598
+ // the failure rollback (releaseRuntime) tears it down like every other
599
+ // runtime component. Open strands skip it entirely: their Member/MemberPeer
600
+ // tables are empty by schema (`OnlyClosed`), so there is nothing to derive
601
+ // a deny set from — arming would be pointless polling.
602
+ const revocationEnforcer =
603
+ config.strandRow.Type === 'c' && config.revocationEnforcement?.enabled !== false
604
+ ? new StrandRevocationEnforcer({
605
+ label: strandId,
606
+ readRows: () => {
607
+ const database = instance.database;
608
+ if (!database) {
609
+ throw new Error(`strand ${strandId} has no live database`);
610
+ }
611
+ return readStrandRevocationRows(database.getDatabase());
612
+ },
613
+ // Read per sweep, never captured: this closure is built BEFORE the
614
+ // libp2p node exists (its options embed the enforcer's predicates),
615
+ // and `releaseRuntime` clears the field again on quiesce. Undefined
616
+ // therefore means "no transport right now", which the enforcer
617
+ // treats as "nothing to tear down".
618
+ getNetwork: () => instance.libp2pNode,
619
+ onSelfRevoked: () => config.onSelfRevoked?.(strandId)
620
+ }, config.revocationEnforcement)
621
+ : undefined;
622
+ if (revocationEnforcer) {
623
+ this.revocationEnforcers.set(strandId, revocationEnforcer);
624
+ }
625
+
626
+ // Closed strands compose revoked-peer denial onto the caller's gater and arm
627
+ // the fail-closed per-stream gate; open strands keep the raw configured gater
628
+ // (their peers are legitimately cross-party and nothing is ever revoked).
629
+ const revocationGateOptions = revocationEnforcer
630
+ ? {
631
+ connectionGater: createRevocationConnectionGater(revocationEnforcer, config.network?.connectionGater),
632
+ authorizeInboundStream: (remotePeerId: string, protocol: string) =>
633
+ revocationEnforcer.authorizeStream(remotePeerId, protocol)
634
+ }
635
+ : config.network?.connectionGater
636
+ ? { connectionGater: config.network.connectionGater }
637
+ : {};
344
638
 
345
639
  try {
640
+ // Bound once: the breadth is also the ceiling on the repair yardstick below, and the
641
+ // two must be derived from the same resolution. Inside the `try` deliberately — a
642
+ // rejected clusterSize is a build failure that runs the same cleanup as any other.
643
+ const strandClusterSize = resolveStrandClusterSize(config.clusterSize);
644
+
346
645
  let t0 = performance.now();
347
646
  const node = await createLibp2pNode({
348
647
  port: 0, // Random port
@@ -351,52 +650,57 @@ export class StrandInstanceManager {
351
650
  storage: strandStorage,
352
651
  fretProfile: config.profile === 'storage' ? 'core' : 'edge',
353
652
  relay: enableRelay,
354
- clusterSize: resolveStrandClusterSize(config.clusterSize),
653
+ clusterSize: strandClusterSize,
355
654
  // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
356
655
  // breadth reasoning, and the shape match with the control policy is a coincidence.
357
- clusterPolicy: STRAND_CLUSTER_POLICY,
656
+ //
657
+ // The builder declares this node's block-repair corroboration yardstick from the
658
+ // count of machines SERVING this strand, capped at the breadth above (a block
659
+ // never lives on more machines than the cohort is wide). Given no count — the
660
+ // production path today, since no per-strand serving count exists yet — it
661
+ // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
662
+ // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
663
+ // serving set that changed while the strand slept.
664
+ clusterPolicy: strandClusterPolicy(strandClusterSize, config.servingMachines),
358
665
  arachnode: {
359
666
  enableRingZulu: config.profile === 'storage'
360
667
  },
361
668
  ...(config.privateKey && { privateKey: config.privateKey }),
362
669
  ...(config.network?.transports && { transports: config.network.transports }),
363
- // NOTE: strand nodes receive the same RESOLVED listen addrs as the
364
- // control node. A fixed-port listen addr (e.g. cadre-cli's example
365
- // `/ip4/0.0.0.0/tcp/4001`) would have control + strand nodes racing to
366
- // bind one port — EADDRINUSE. Unverified; if a deployment configures a
367
- // fixed port and strands fail to start, rewrite the port per node here.
368
- // An inherited configured `/p2p-circuit` addr, by contrast, is deliberate —
369
- // it is what gives a NAT'd strand node a reachable relay slot — and works
370
- // because the launch path announces this strand's derived peerId to
371
- // the relay first (delegate admission; see cadre-node.ts). Those circuit
372
- // entries come either from a hand-written `network.listenAddrs` or from
373
- // `network.relayAddrs`, which `resolveListenAddrs` folds into the same list
374
- // on the configured route this call takes (see the comment above it).
375
- ...(listenAddrs && { listenAddrs }),
376
- // A strand node behind a reverse proxy has the same advertising need as the
377
- // control node, and inherits the same `NetworkConfig` — so it announces the same
378
- // addresses.
379
- //
380
- // NOTE: an announce addr naming a PORT therefore names the control node's port,
381
- // not this strand node's — the same second-node-on-one-host problem the listen
382
- // NOTE above describes, and the reason both belong to one fix. It is currently
383
- // unreachable: naming a fixed public port means configuring a fixed
384
- // `listenAddrs` port, and that collides at bind time before any strand node gets
385
- // far enough to advertise anything. Whatever gives strand nodes their own listen
386
- // port has to give them their own announce addrs in the same pass, or a strand
387
- // will advertise an address that reaches the control node. Tracked as an arm of
388
- // `tickets/backlog/strand-network-nat-relay-reachability.md`.
389
- ...resolveAnnounceAddrs(config.network),
390
- ...(config.network?.connectionGater && { connectionGater: config.network.connectionGater })
670
+ // Listen entries plus the WebSocket transport switch they imply — a strand node
671
+ // announces nothing the operator configured (`strand-network-config.ts`), and
672
+ // spreads AFTER `transports` above because the switch is a no-op whenever the
673
+ // embedder supplied its own factories. The bare `/p2p-circuit` entries (one per
674
+ // relay) open no connection here; each registers a pending reservation that the
675
+ // matching supervisor below fills, which is what gives a NAT'd strand node a
676
+ // reachable relay slot — and works against a party-run relay because the launch
677
+ // path announced this strand's derived peerId to it first (delegate admission;
678
+ // see cadre-node.ts).
679
+ ...addrOptions,
680
+ // The raw configured gater (open strands), or the revocation-composed
681
+ // gater plus the fail-closed per-stream revoked-peer gate (closed
682
+ // strands) — resolved above, before the try.
683
+ ...revocationGateOptions
391
684
  }) as Libp2pNodeWithRepo;
392
685
  timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
393
686
 
394
687
  instance.libp2pNode = node;
395
688
 
689
+ // One reservation supervisor PER RELAY, started now so the first attempts
690
+ // overlap the database bring-up below (the relay is never in a strand's
691
+ // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
692
+ // — so an open relay connection cannot disturb that bring-up). Their first
693
+ // attempts are awaited before the strand goes `active`, below. Registered in the
694
+ // map immediately so the failure rollback (releaseRuntime) stops them like every
695
+ // other runtime component.
696
+ this.relaySupervisors.set(strandId, this.startRelaySupervisors(strandId, node, relayDialAddrs, config));
697
+
396
698
  // Create and initialize the StrandDatabase.
397
699
  //
398
- // Attach before initialize so a failed init is cleaned up by
399
- // releaseRuntime below (close() is safe on a partially-initialized db).
700
+ // Held by the first-sync gate (registered before initialize) rather than attached
701
+ // to the instance: a failed init is then cleaned up by releaseRuntime below through
702
+ // the gate (close() is safe on a partially-initialized db), and the app only ever
703
+ // sees `instance.database` once this machine is allowed to write to the strand.
400
704
  t0 = performance.now();
401
705
  const strandDb = new StrandDatabase({
402
706
  strandId,
@@ -404,33 +708,107 @@ export class StrandInstanceManager {
404
708
  libp2pNode: node,
405
709
  coordinatedRepo: node.coordinatedRepo,
406
710
  // Founder bootstrap inputs: the strand's type drives which membership rows
407
- // are written, and the closed-strand MemberPrivateKey derives the founding
408
- // Member/Manager key. Both come off the control-network strand row.
711
+ // are written, and the PARTY's own key (partyMemberPrivateKey, resolved by the
712
+ // caller from the control-layer StrandPartyKey row) derives the founding
713
+ // Member/Manager key. The row's shared MemberPrivateKey rides along as the
714
+ // read-gating secret only — it derives nobody's identity.
409
715
  strandType: config.strandRow.Type,
410
716
  memberPrivateKey: config.strandRow.MemberPrivateKey ?? undefined,
717
+ partyMemberPrivateKey: config.partyMemberPrivateKey,
411
718
  founder: config.founder
412
719
  });
413
- instance.database = strandDb;
720
+ const gate = new StrandFirstSyncGate({
721
+ label: strandId,
722
+ database: strandDb,
723
+ onHeaderHeld: () => this.publishDatabase(instance, strandDb)
724
+ }, config.firstSync);
725
+ this.firstSyncGates.set(strandId, gate);
414
726
  await strandDb.initialize();
415
727
  timing('[buildStrandRuntime:%s] strandDatabase.initialize: %dms', strandId, Math.round(performance.now() - t0));
416
728
 
729
+ // The first-sync write gate. A founder just wrote the Header in its bootstrap; any
730
+ // other machine may commit only once it holds the Header — which it can only have
731
+ // received from a peer (now, or on an earlier run over the same store) — and has read
732
+ // each App table once, so their collections are fetched too. Publishing the database
733
+ // is what makes the strand writable to the app; until then the instance stays
734
+ // `'syncing'` and the gate re-probes on its cadence. Everything armed below reads
735
+ // `instance.database` lazily, so the reconciler and enforcer wait with it — and
736
+ // `publishDatabase` kicks the reconciler when the wait ends, so a joiner's membership
737
+ // rows land with the strand rather than one retry later.
738
+ t0 = performance.now();
739
+ if (config.founder === true || await strandFirstSyncComplete(strandDb.getDatabase(), strandId)) {
740
+ this.publishDatabase(instance, strandDb);
741
+ } else {
742
+ gate.start();
743
+ }
744
+ timing('[buildStrandRuntime:%s] first-sync probe: %dms', strandId, Math.round(performance.now() - t0));
745
+
746
+ // The membership reconciler: finish this party's join on every machine —
747
+ // redeem a staged formation invitation if one is pending, then write the
748
+ // durable machine→party binding (Strand.MemberPeer). Armed here, the one
749
+ // seam where the strand's transport peer id and a live Database both exist
750
+ // for launch AND hibernation resume, for closed strands that carry the
751
+ // party's own membership key. start() kicks an immediate pass WITHOUT
752
+ // awaiting it — a joiner at this instant has not synced the founder's rows
753
+ // and may lack write quorum, so bring-up is never blocked; an unfinished join
754
+ // then retries on a short doubling ladder capped at the configured
755
+ // pollIntervalMs (which is also the flat cadence while nobody has admitted this
756
+ // party yet), and the loop stops once the member row and this machine's own
757
+ // binding are both in place. Armed AFTER the gate above, so a gated launch
758
+ // always has a reconciler registered by the time the gate can publish.
759
+ if (config.strandRow.Type === 'c' && config.partyMemberPrivateKey
760
+ && config.membershipReconciliation?.enabled !== false) {
761
+ const reconciler = new StrandMembershipReconciler({
762
+ label: strandId,
763
+ partyMemberPrivateKey: config.partyMemberPrivateKey,
764
+ // Read per pass, never captured — same lifecycle argument as the
765
+ // enforcer's getNetwork: quiesce drops both handles.
766
+ getDatabase: () => instance.database?.getDatabase(),
767
+ getOwnPeerId: () => instance.libp2pNode?.peerId.toString(),
768
+ pendingInvite: config.pendingMembershipInvite,
769
+ // The enforcer's CURRENT snapshot view of this node's own peer id — the
770
+ // reconciler stops rather than fight a self-revocation. Absent when the
771
+ // gate is disarmed (fail-open: the loop just runs).
772
+ isSelfRevoked: revocationEnforcer
773
+ ? () => {
774
+ const ownPeerId = instance.libp2pNode?.peerId.toString();
775
+ return ownPeerId !== undefined && revocationEnforcer.isRevoked(ownPeerId);
776
+ }
777
+ : undefined
778
+ }, {
779
+ pollIntervalMs: config.membershipReconciliation?.pollIntervalMs
780
+ ?? config.revocationEnforcement?.pollIntervalMs
781
+ });
782
+ this.membershipReconcilers.set(strandId, reconciler);
783
+ reconciler.start();
784
+ }
785
+
786
+ // Arm the revoked-peer deny-set poll now that the database it reads
787
+ // exists. start() kicks an immediate refresh WITHOUT awaiting it —
788
+ // bring-up and resume are never blocked on a membership read; until the
789
+ // first read lands the empty snapshot admits everyone (fail-open).
790
+ revocationEnforcer?.start();
791
+
417
792
  // Peer-join block catch-up: push this strand's own blocks to each newly
418
793
  // connected peer, so a machine that joined after blocks were committed
419
794
  // still ends up physically holding them (without per-strand storage there
420
- // is nothing to copy). Armed for EVERY stored strand: `StrandBackfill`
795
+ // is nothing to copy). Armed for EVERY stored strand: `PeerJoinBackfill`
421
796
  // only does work when the strand's libp2p node reports a peer connection,
422
797
  // so on a device that is genuinely alone it is inert, and arming it at
423
798
  // launch is what closes the "founded alone, never replicates" hole — a
424
799
  // peer that joins later gets the founder's blocks without any relaunch.
800
+ // No `authorizePeer` gate, deliberately — see the module comment in
801
+ // peer-join-backfill.ts for the strand-side argument (and why the control
802
+ // network, which DOES gate, is different).
425
803
  //
426
- // NOTE: cost is one StrandBackfill object + one `connection:open` listener
804
+ // NOTE: cost is one PeerJoinBackfill object + one `connection:open` listener
427
805
  // per running strand — linear in strand count, negligible at the handful a
428
806
  // device or host runs today. If a node ever hosts strands by the hundred,
429
807
  // move to one shared listener that dispatches by strand id.
430
808
  if (strandStorage && config.backfill?.enabled !== false) {
431
809
  if (node.keyNetwork) {
432
- const backfill = new StrandBackfill({
433
- strandId,
810
+ const backfill = new PeerJoinBackfill({
811
+ label: strandId,
434
812
  libp2p: node,
435
813
  peerNetwork: node.keyNetwork,
436
814
  storage: strandStorage,
@@ -445,8 +823,27 @@ export class StrandInstanceManager {
445
823
  }
446
824
  }
447
825
 
448
- instance.status = 'active';
826
+ // The relay supervisors' FIRST attempts, all of them, before `active`: the happy
827
+ // path still publishes its circuit addr before `addStrand` resolves (the
828
+ // strand-addr RPC answers read it, and the same-party circuit scenario asserts
829
+ // it). Fail-SOFT — a first attempt that lands nothing does not fail the launch.
830
+ // The strand's database is up, and the supervisor keeps trying on its backoff;
831
+ // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
832
+ //
833
+ // NOTE: a relay that is down costs this launch one full drive (10 s), and
834
+ // `StrandWatcher` launches strands one at a time — so N strands cost N × 10 s
835
+ // of bring-up during a relay outage. If that ever matters, stop awaiting here
836
+ // (the circuit addr then lands after `active`) rather than shortening the drive.
837
+ t0 = performance.now();
838
+ await this.awaitFirstRelayAttempts(strandId);
839
+ timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
840
+
841
+ instance.status = liveStrandStatus(instance);
449
842
  instance.lastActivity = new Date();
843
+ if (instance.status === 'syncing') {
844
+ log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
845
+ 'a member of the strand is reached', strandId);
846
+ }
450
847
  } catch (error) {
451
848
  // Roll back any partially-attached runtime so the instance is left with
452
849
  // NEITHER handle. Otherwise the `libp2pNode || database` "already live"
@@ -460,17 +857,30 @@ export class StrandInstanceManager {
460
857
  }
461
858
 
462
859
  /**
463
- * Release an instance's strand-network runtime: close the StrandDatabase, then
464
- * stop the libp2p node (construction order in reverse), clearing both fields
465
- * and zeroing connectedPeers. Tolerant of partially-built state — either handle
466
- * may be absent — so it doubles as rollback for a failed `buildStrandRuntime`.
467
- * Shared by `quiesceStrand`, `stopStrand`, and that rollback path.
860
+ * Release an instance's strand-network runtime: stop the relay supervisors and
861
+ * the other background loops, close the StrandDatabase, then stop the libp2p
862
+ * node (construction order in reverse), clearing both fields and zeroing
863
+ * connectedPeers. Tolerant of partially-built state — either handle may be
864
+ * absent — so it doubles as rollback for a failed `buildStrandRuntime`. Shared
865
+ * by `quiesceStrand`, `stopStrand`, and that rollback path.
468
866
  *
469
867
  * Leaves `strandStorages` untouched by design — the store outlives the runtime it
470
868
  * was built into, which is what makes a resume warm. Only `stopStrand` disposes it.
471
869
  */
472
870
  private async releaseRuntime(instance: StrandInstance): Promise<void> {
473
- // Backfill first — before the database closes and the libp2p node stops — so
871
+ // Relay supervisors FIRST of all — before anything below is torn down — so no
872
+ // re-drive is scheduled against a node being stopped. `stop()` is synchronous
873
+ // and never awaits a drive, so a relay that is down cannot delay this teardown.
874
+ // NOTE: a drive ALREADY in flight cannot be aborted (the drive takes no
875
+ // AbortSignal — `backlog/bug-relay-drive-not-cancellable`); it fails soft
876
+ // against the stopped node within its own 10 s deadline and its result is
877
+ // discarded.
878
+ const relaySupervisors = this.relaySupervisors.get(instance.strandId);
879
+ if (relaySupervisors) {
880
+ relaySupervisors.forEach((supervisor) => supervisor.stop());
881
+ this.relaySupervisors.delete(instance.strandId);
882
+ }
883
+ // Backfill next — before the database closes and the libp2p node stops — so
474
884
  // no NEW catch-up push is issued against a torn-down transport. A push already
475
885
  // in flight is not awaited; it fails into the module's own per-chunk catch.
476
886
  const backfill = this.backfills.get(instance.strandId);
@@ -478,6 +888,29 @@ export class StrandInstanceManager {
478
888
  backfill.stop();
479
889
  this.backfills.delete(instance.strandId);
480
890
  }
891
+ // The revoked-peer enforcer goes with the runtime it gated: a resume
892
+ // rebuilds it with a fresh (initially empty, fail-open) snapshot.
893
+ const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
894
+ if (revocationEnforcer) {
895
+ revocationEnforcer.stop();
896
+ this.revocationEnforcers.delete(instance.strandId);
897
+ }
898
+ // The membership reconciler likewise: a resume rebuilds it and re-runs the
899
+ // idempotent ladder, which is what makes resume-after-partial-join heal.
900
+ const membershipReconciler = this.membershipReconcilers.get(instance.strandId);
901
+ if (membershipReconciler) {
902
+ membershipReconciler.stop();
903
+ this.membershipReconcilers.delete(instance.strandId);
904
+ }
905
+ // A still-gated database (a joiner that never received the Header, or a launch whose
906
+ // initialize() failed) is closed through its gate; a published one through the
907
+ // instance. Never both — publishing moves the handle from the gate to the instance.
908
+ const gate = this.firstSyncGates.get(instance.strandId);
909
+ if (gate) {
910
+ gate.stop();
911
+ this.firstSyncGates.delete(instance.strandId);
912
+ await gate.database.close();
913
+ }
481
914
  if (instance.database) {
482
915
  await instance.database.close();
483
916
  instance.database = undefined;
@@ -489,6 +922,53 @@ export class StrandInstanceManager {
489
922
  instance.connectedPeers = 0;
490
923
  }
491
924
 
925
+ /**
926
+ * One {@link superviseRelayReservation} per relay dial addr, each over exactly
927
+ * that relay so "held" is judged per relay (`circuitMultiaddrsVia`) and losing
928
+ * one relay re-drives only that one. Default timings — the control node's (2 s
929
+ * doubling to 60 s between failed attempts, a 5 s liveness check while held).
930
+ *
931
+ * Each supervisor's `beforeRedrive` is the caller's
932
+ * {@link StartStrandConfig.announceDelegateToRelay} for THIS relay and THIS
933
+ * node's peer id, so a re-drive after a relay restart is preceded by a fresh
934
+ * delegate grant on the relay about to be dialed. Returns `[]` when no relay is
935
+ * configured — the strand then simply has no circuit listener to fill.
936
+ */
937
+ private startRelaySupervisors(
938
+ strandId: string,
939
+ node: Libp2pNodeWithRepo,
940
+ relayDialAddrs: readonly string[],
941
+ config: StartStrandConfig
942
+ ): RelayReservationSupervisor[] {
943
+ if (relayDialAddrs.length === 0) {
944
+ return [];
945
+ }
946
+ const announce = config.announceDelegateToRelay;
947
+ const delegatePeerId = node.peerId.toString();
948
+ return relayDialAddrs.map((relayAddr) => superviseRelayReservation(node, [relayAddr], {
949
+ ...(announce && { beforeRedrive: () => announce(strandId, relayAddr, delegatePeerId) })
950
+ }));
951
+ }
952
+
953
+ /**
954
+ * Wait for every relay supervisor's first attempt to settle, concurrently, and log
955
+ * the ones that landed nothing. Never throws — a strand whose relay is unreachable
956
+ * at launch still comes up, with its supervisors retrying in the background.
957
+ */
958
+ private async awaitFirstRelayAttempts(strandId: string): Promise<void> {
959
+ const supervisors = this.relaySupervisors.get(strandId) ?? [];
960
+ if (supervisors.length === 0) {
961
+ return;
962
+ }
963
+ await Promise.all(supervisors.map((supervisor) => supervisor.firstAttempt));
964
+ supervisors.forEach((supervisor) => {
965
+ if (supervisor.lastError !== null) {
966
+ log('Strand %s: relay reservation not held after the first attempt (retrying in the background): %s',
967
+ strandId, supervisor.lastError);
968
+ }
969
+ });
970
+ }
971
+
492
972
  /**
493
973
  * Drop the strand's owned store and release this strand's claim on its cache. The
494
974
  * wrapper counts holders, so the cache is emptied and unregistered from the shared
@@ -513,6 +993,86 @@ export class StrandInstanceManager {
513
993
  }
514
994
  }
515
995
 
996
+ /**
997
+ * Refresh a strand's revoked-peer deny set NOW — and, with it, run the
998
+ * teardown sweep that hangs up every connected revoked peer.
999
+ *
1000
+ * The gate is otherwise poll-driven (default
1001
+ * `DEFAULT_REVOCATION_POLL_INTERVAL_MS`), because neither a replicated
1002
+ * revocation nor a local `revokeMember`/`leaveStrand` raises anything this
1003
+ * runtime can hook. A caller that just wrote a revocation against the
1004
+ * strand's database should follow it with this call, which makes the cut
1005
+ * immediate instead of waiting out the interval.
1006
+ *
1007
+ * Quiet no-op — matching `quiesceStrand`'s posture — when the strand is not
1008
+ * tracked, is quiesced, is open, or has the gate disabled: in every one of
1009
+ * those cases there is no enforcer to refresh. Never rejects (the enforcer's
1010
+ * refresh contains its own failures), and resolves only once the sweep has
1011
+ * finished.
1012
+ */
1013
+ async refreshRevocationEnforcement(strandId: string): Promise<void> {
1014
+ const enforcer = this.revocationEnforcers.get(strandId);
1015
+ if (!enforcer) {
1016
+ log('refreshRevocationEnforcement: strand %s has no armed revocation enforcer', strandId);
1017
+ return;
1018
+ }
1019
+ await enforcer.refresh();
1020
+ }
1021
+
1022
+ /**
1023
+ * Best-effort removal of THIS machine's own `Strand.MemberPeer` binding — the
1024
+ * self arm of `removeMemberPeer`, signed with the retained launch config's party
1025
+ * key. Called by `CadreNode.unpublishStrand` BEFORE the local stop, while the
1026
+ * strand database and transport are still live: after the unpublish commits, the
1027
+ * party's `StrandPartyKey` control row is gone, so no restart could ever sign
1028
+ * this removal again — the in-memory retained key is the last chance.
1029
+ *
1030
+ * Never throws. Quiet no-op for a strand that is untracked, open, or launched
1031
+ * without a party key; a quiesced (hibernating) strand, or a strand whose write
1032
+ * quorum is already unreachable, leaves the stale binding behind with a log —
1033
+ * it grants nothing today (admission is not allowlist-gated) and only
1034
+ * mis-credits diversity. The strand's own membership reconciler is stopped —
1035
+ * and its in-flight pass awaited — FIRST, so a pass that was already past its
1036
+ * own stopped check cannot re-register the binding this is about to clear.
1037
+ *
1038
+ * NOTE: deliberately NOT handled here (or anywhere yet): clearing the bindings
1039
+ * of a machine removed from the CADRE at the control layer, or of a party's
1040
+ * OTHER machines when the party unpublishes — the party still holds the key on
1041
+ * every surviving machine and can clear from any of them, and the stale rows
1042
+ * grant nothing while admission is deny-list only. Revisit when
1043
+ * `feat-strand-member-allowlist-admission` lands and a stale binding starts
1044
+ * granting admission rather than merely mis-crediting diversity.
1045
+ */
1046
+ async clearOwnMemberPeerBinding(strandId: string): Promise<void> {
1047
+ const instance = this.instances.get(strandId);
1048
+ const config = this.launchConfigs.get(strandId);
1049
+ if (!instance || !config || config.strandRow.Type !== 'c' || !config.partyMemberPrivateKey) {
1050
+ return;
1051
+ }
1052
+ const reconciler = this.membershipReconcilers.get(strandId);
1053
+ if (reconciler) {
1054
+ reconciler.stop();
1055
+ await reconciler.settle();
1056
+ }
1057
+ const db = instance.database?.getDatabase();
1058
+ const peerId = instance.libp2pNode?.peerId.toString();
1059
+ if (!db || !peerId) {
1060
+ log('clearOwnMemberPeerBinding: strand %s has no live runtime — own MemberPeer binding left in place', strandId);
1061
+ return;
1062
+ }
1063
+ try {
1064
+ // Background write on the app's database: never join a transaction the app has open.
1065
+ await removeMemberPeer(db, {
1066
+ memberKeyPair: strandMemberKeyPair(config.partyMemberPrivateKey),
1067
+ peerId
1068
+ }, { joinOpenTransaction: false });
1069
+ log('clearOwnMemberPeerBinding: strand %s — own binding for peer %s cleared', strandId, peerId);
1070
+ } catch (error) {
1071
+ log('clearOwnMemberPeerBinding: strand %s — best-effort clear failed (strand may already be unreachable): %o',
1072
+ strandId, error);
1073
+ }
1074
+ }
1075
+
516
1076
  /**
517
1077
  * Quiesce a strand: release its strand-network resources (stop the libp2p node,
518
1078
  * close the StrandDatabase) while RETAINING the instance record — identity,
@@ -541,9 +1101,9 @@ export class StrandInstanceManager {
541
1101
  * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
542
1102
  * from the retained launch config and re-attach them, transitioning it back to
543
1103
  * `active`. `overrides` re-applies volatile inputs that may have changed since
544
- * launch (the cohort `bootstrapNodes` seed) and updates the retained config so
545
- * a later resume reuses the latest values. Returns the live instance unchanged
546
- * if it is already running.
1104
+ * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
1105
+ * count) and updates the retained config so a later resume reuses the latest
1106
+ * values. Returns the live instance unchanged if it is already running.
547
1107
  */
548
1108
  async resumeStrand(strandId: string, overrides?: ResumeStrandOverrides): Promise<StrandInstance> {
549
1109
  if (this.stopping) {
@@ -566,9 +1126,20 @@ export class StrandInstanceManager {
566
1126
  const tTotal = performance.now();
567
1127
 
568
1128
  // Re-apply volatile inputs and persist them so a subsequent resume reuses them.
1129
+ // Each `??` matters: a resume that passes no override must keep the retained value,
1130
+ // and a resume that passes one must leave it retained for the next resume — otherwise
1131
+ // a later no-override wake silently reverts to whatever launch time saw.
569
1132
  const resumeConfig: StartStrandConfig = {
570
1133
  ...launchConfig,
571
- bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes
1134
+ bootstrapNodes: overrides?.bootstrapNodes ?? launchConfig.bootstrapNodes,
1135
+ // NOTE: `??` retains, so an override can raise or lower the count but cannot CLEAR it
1136
+ // back to "this node no longer knows" — the direction that would declare nothing. Moot
1137
+ // while nothing feeds `servingMachines` at all; if a source lands
1138
+ // (`backlog/feat-strand-yardstick-from-serving-machines`) that can legitimately lose the
1139
+ // count — a strand whose member rows became unreadable — this merge must gain an explicit
1140
+ // clear rather than silently declaring a stale number over a serving set it can no longer
1141
+ // see. Same applies to `bootstrapNodes` above, where a stale seed is harmless.
1142
+ servingMachines: overrides?.servingMachines ?? launchConfig.servingMachines
572
1143
  };
573
1144
  this.launchConfigs.set(strandId, resumeConfig);
574
1145
 
@@ -586,6 +1157,237 @@ export class StrandInstanceManager {
586
1157
  }
587
1158
  }
588
1159
 
1160
+ /**
1161
+ * Honor a founder request against an ALREADY-TRACKED strand — the seam that
1162
+ * closes the "whoever launches first decides whether the bootstrap runs" gap:
1163
+ * an instance first launched as a joiner (an app's own attach, or a watcher
1164
+ * poll winning the launch race) used to swallow a later founder request
1165
+ * silently, leaving the strand active with no `Strand.Header`.
1166
+ *
1167
+ * Flips the RETAINED launch config's `founder` to true, so every later
1168
+ * quiesce → resume rebuild founds as well (the bootstrap is insert-if-absent —
1169
+ * {@link StrandDatabase.ensureFounderBootstrap} — so re-running it per rebuild
1170
+ * writes nothing twice), and runs the bootstrap against the live database now.
1171
+ *
1172
+ * @returns how the request resolved:
1173
+ * - `'already-founder'` — the retained config already founds; nothing to do.
1174
+ * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1175
+ * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1176
+ * database), so the bootstrap could not run here: the CALLER must wake the
1177
+ * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1178
+ * manager does not) so the rebuild — which now founds — runs it.
1179
+ * @param resolvePartyKey - Asked for the party's own membership key when (and only
1180
+ * when) a CLOSED strand's retained config carries none — the instance was launched
1181
+ * as a joiner before its `StrandPartyKey` row existed or replicated. The resolved
1182
+ * key is retained alongside the founder flip so the bootstrap (now, or on the
1183
+ * caller's wake for `'needs-resume'`) can seat the founding Member/Manager.
1184
+ * @throws when the strand is not tracked — this seam exists only for the
1185
+ * tracked-instance launch path; an untracked id is a caller bug.
1186
+ * @throws whatever the live bootstrap throws (e.g. `PreSplitStrandIdentityError`),
1187
+ * after {@link withdrawFounderRequest} — so a retry re-attempts the founding rather
1188
+ * than resolving `'already-founder'` over an instance that never founded. A caller
1189
+ * whose `'needs-resume'` wake fails owes the same withdrawal.
1190
+ */
1191
+ async foundExistingStrand(
1192
+ strandId: string,
1193
+ resolvePartyKey?: () => Promise<string | undefined>
1194
+ ): Promise<'already-founder' | 'bootstrapped' | 'needs-resume'> {
1195
+ const instance = this.instances.get(strandId);
1196
+ const config = this.launchConfigs.get(strandId);
1197
+ if (!instance || !config) {
1198
+ throw new Error(`Cannot found strand ${strandId}: not tracked`);
1199
+ }
1200
+ if (config.founder === true) {
1201
+ return 'already-founder';
1202
+ }
1203
+ const partyMemberPrivateKey =
1204
+ config.partyMemberPrivateKey
1205
+ ?? (config.strandRow.Type === 'c' ? await resolvePartyKey?.() : undefined);
1206
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1207
+ // config object, which is not ours to rewrite.
1208
+ this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1209
+ if (!instance.database && !this.firstSyncGates.has(strandId)) {
1210
+ return 'needs-resume';
1211
+ }
1212
+ try {
1213
+ await this.ensureFounderBootstrap(strandId);
1214
+ } catch (error) {
1215
+ this.withdrawFounderRequest(strandId);
1216
+ throw error;
1217
+ }
1218
+ return 'bootstrapped';
1219
+ }
1220
+
1221
+ /**
1222
+ * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1223
+ * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
1224
+ * the hibernation manager's idle timer fired on it). `false` for a writable, quiesced,
1225
+ * or untracked strand. The predicate {@link whenWritable}'s callers gate on.
1226
+ */
1227
+ isAwaitingFirstSync(strandId: string): boolean {
1228
+ const instance = this.instances.get(strandId);
1229
+ return instance !== undefined && isAwaitingFirstSync(instance);
1230
+ }
1231
+
1232
+ /**
1233
+ * Resolve once the strand's database is published to the app — immediately for a
1234
+ * writable strand — or reject with {@link StrandAwaitingFirstSyncError} after
1235
+ * `timeoutMs` (default: the retained launch config's `firstSync.timeoutMs`, else
1236
+ * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}). The rejection is RETRYABLE: nothing is
1237
+ * torn down, the gate keeps probing, and a later call waits afresh. A quiesced
1238
+ * (hibernating) strand is not woken here — the wait spans a resume, so a caller that
1239
+ * wants one wakes it (`CadreNode.wakeStrand`) and waits. Rejects immediately for an
1240
+ * untracked strand, and whenever the strand is stopped while a wait is pending.
1241
+ */
1242
+ whenWritable(strandId: string, options?: { timeoutMs?: number }): Promise<StrandInstance> {
1243
+ const instance = this.instances.get(strandId);
1244
+ if (!instance) {
1245
+ return Promise.reject(new Error(`Cannot wait for strand ${strandId} to become writable: not tracked`));
1246
+ }
1247
+ if (instance.database) {
1248
+ return Promise.resolve(instance);
1249
+ }
1250
+ const timeoutMs = options?.timeoutMs
1251
+ ?? this.launchConfigs.get(strandId)?.firstSync?.timeoutMs
1252
+ ?? DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS;
1253
+ const startedAt = Date.now();
1254
+ return new Promise<StrandInstance>((resolve, reject) => {
1255
+ const waiters = this.writableWaiters.get(strandId) ?? new Set<WritableWaiter>();
1256
+ this.writableWaiters.set(strandId, waiters);
1257
+ const settle = (): void => {
1258
+ clearTimeout(timer);
1259
+ waiters.delete(waiter);
1260
+ if (waiters.size === 0) {
1261
+ this.writableWaiters.delete(strandId);
1262
+ }
1263
+ };
1264
+ const waiter: WritableWaiter = {
1265
+ resolve: () => { settle(); resolve(instance); },
1266
+ reject: (error) => { settle(); reject(error); }
1267
+ };
1268
+ const timer = setTimeout(
1269
+ () => waiter.reject(new StrandAwaitingFirstSyncError(strandId, Date.now() - startedAt)),
1270
+ timeoutMs);
1271
+ (timer as { unref?: () => void }).unref?.();
1272
+ waiters.add(waiter);
1273
+ });
1274
+ }
1275
+
1276
+ /**
1277
+ * Hand a launch's database to the app: set `instance.database`, retire the gate that
1278
+ * held it, release every {@link whenWritable} waiter, and — when the launch had already
1279
+ * been reported gated — flip it `'active'` and announce it through the retained config's
1280
+ * `onWritable`. "Reported gated" is any status but `'starting'`: `'syncing'`, or the
1281
+ * `'idle'` the hibernation manager's idle timer flips a long-gated joiner to (see
1282
+ * {@link isAwaitingFirstSync}) — the Header arriving after that flip must announce too,
1283
+ * or an app hanging on `strand:writable` never learns the strand opened. A publish that
1284
+ * lands while the instance is still `'starting'` (the Header arrived during the rest of
1285
+ * bring-up) announces nothing: `startStrand` / `resumeStrand` are about to report the
1286
+ * strand `'active'`, and `strand:writable` is defined as the follow-up to a gated
1287
+ * launch, never a duplicate of `strand:started`.
1288
+ *
1289
+ * It is also the one seam where a WITHHELD database becomes available, so it kicks the
1290
+ * membership reconciler: on a gated launch that loop's own immediate pass ran while
1291
+ * `instance.database` was still unset, found no database, and would otherwise not try
1292
+ * again until its retry timer fired. A launch that was never gated publishes here BEFORE
1293
+ * the reconciler is constructed, so there is nothing registered to kick and the loop's own
1294
+ * immediate pass already sees the database — which is why the kick needs no `wasGated`
1295
+ * test of its own.
1296
+ */
1297
+ private publishDatabase(instance: StrandInstance, database: StrandDatabase): void {
1298
+ const strandId = instance.strandId;
1299
+ const gate = this.firstSyncGates.get(strandId);
1300
+ if (gate) {
1301
+ gate.open();
1302
+ this.firstSyncGates.delete(strandId);
1303
+ }
1304
+ instance.database = database;
1305
+ instance.lastActivity = new Date();
1306
+ const wasGated = instance.status !== 'starting';
1307
+ if (wasGated) {
1308
+ instance.status = 'active';
1309
+ log('Strand %s is now writable: Strand.Header held', strandId);
1310
+ }
1311
+ const waiters = this.writableWaiters.get(strandId);
1312
+ if (waiters) {
1313
+ this.writableWaiters.delete(strandId);
1314
+ waiters.forEach((waiter) => waiter.resolve());
1315
+ }
1316
+ if (wasGated) {
1317
+ this.launchConfigs.get(strandId)?.onWritable?.(strandId);
1318
+ }
1319
+ // Not awaited: publishing must stay synchronous, and the pass chains on the loop's own
1320
+ // tail, so it cannot overlap a pass already in flight.
1321
+ void this.membershipReconcilers.get(strandId)?.reconcile();
1322
+ }
1323
+
1324
+ /** Reject every {@link whenWritable} waiter for a strand that is going away. */
1325
+ private rejectWritableWaiters(strandId: string, reason: string): void {
1326
+ const waiters = this.writableWaiters.get(strandId);
1327
+ if (!waiters) {
1328
+ return;
1329
+ }
1330
+ this.writableWaiters.delete(strandId);
1331
+ waiters.forEach((waiter) => waiter.reject(new Error(`Strand ${strandId} ${reason} before becoming writable`)));
1332
+ }
1333
+
1334
+ /**
1335
+ * Undo {@link foundExistingStrand}'s founder flip after the founding it promised
1336
+ * failed, so the retained config again says what this instance actually runs as (a
1337
+ * joiner) and the next founder request re-attempts the bootstrap. The resolved party
1338
+ * key stays retained — it is this party's identity for the strand either way. No-op
1339
+ * when the strand is untracked or its config does not found.
1340
+ */
1341
+ withdrawFounderRequest(strandId: string): void {
1342
+ const config = this.launchConfigs.get(strandId);
1343
+ if (config?.founder === true) {
1344
+ this.launchConfigs.set(strandId, { ...config, founder: false });
1345
+ }
1346
+ }
1347
+
1348
+ /**
1349
+ * Run the (idempotent) founder bootstrap against a tracked strand's LIVE
1350
+ * database, independently of what the retained launch config says.
1351
+ *
1352
+ * Separate from {@link foundExistingStrand} because a caller that resolved
1353
+ * `'needs-resume'` and woke the strand must not assume the wake's own rebuild
1354
+ * founded it: `HibernationManager` COALESCES wakes, so a wake already in flight
1355
+ * when the config flipped had already read the PRE-flip config and rebuilt as a
1356
+ * joiner. Re-running the bootstrap costs one insert-if-absent probe per table
1357
+ * and is the only thing that makes "founding resolves once the Header is
1358
+ * written" true on that path.
1359
+ *
1360
+ * A strand still `'syncing'` (its database withheld behind the first-sync gate) is
1361
+ * founded through the gate: the bootstrap writes the Header this machine was waiting
1362
+ * to receive, so a successful run publishes the database and the strand goes
1363
+ * `'active'` — the one legitimate way a joiner launch becomes writable without a peer.
1364
+ *
1365
+ * @throws when the strand is not tracked, or is still quiesced (no live
1366
+ * database) — both mean the bootstrap did NOT run, which a founder request
1367
+ * must never swallow.
1368
+ */
1369
+ async ensureFounderBootstrap(strandId: string): Promise<void> {
1370
+ const instance = this.instances.get(strandId);
1371
+ if (!instance) {
1372
+ throw new Error(`Cannot run the founder bootstrap for strand ${strandId}: not tracked`);
1373
+ }
1374
+ const gate = this.firstSyncGates.get(strandId);
1375
+ const database = instance.database ?? gate?.database;
1376
+ if (!database) {
1377
+ throw new Error(
1378
+ `Cannot run the founder bootstrap for strand ${strandId}: it is quiesced, so there ` +
1379
+ 'is no live database to write to — resume it first.'
1380
+ );
1381
+ }
1382
+ // Forward the RETAINED config's party key: a foundExistingStrand that flipped a
1383
+ // joiner launch to founder may have resolved a key the live database's captured
1384
+ // config (built at the original launch) never saw.
1385
+ await database.ensureFounderBootstrap(this.launchConfigs.get(strandId)?.partyMemberPrivateKey);
1386
+ if (!instance.database) {
1387
+ this.publishDatabase(instance, database);
1388
+ }
1389
+ }
1390
+
589
1391
  /**
590
1392
  * Stop a strand instance
591
1393
  */
@@ -598,6 +1400,7 @@ export class StrandInstanceManager {
598
1400
 
599
1401
  log('Stopping strand instance: %s', strandId);
600
1402
  instance.status = 'stopping';
1403
+ this.rejectWritableWaiters(strandId, 'was stopped');
601
1404
 
602
1405
  try {
603
1406
  await this.releaseRuntime(instance);