@serfab/cadre-core 1.4.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/cadre-node.d.ts +38 -15
  2. package/dist/cadre-node.js +81 -29
  3. package/dist/cadre-node.js.map +1 -1
  4. package/dist/control-database.d.ts +24 -10
  5. package/dist/control-database.js +48 -16
  6. package/dist/control-database.js.map +1 -1
  7. package/dist/control-read-retry.d.ts +12 -0
  8. package/dist/control-read-retry.js +12 -0
  9. package/dist/control-read-retry.js.map +1 -1
  10. package/dist/control-write-retry.d.ts +5 -4
  11. package/dist/control-write-retry.js +24 -7
  12. package/dist/control-write-retry.js.map +1 -1
  13. package/dist/index.d.ts +3 -2
  14. package/dist/index.js +5 -2
  15. package/dist/index.js.map +1 -1
  16. package/dist/link-budget.d.ts +224 -0
  17. package/dist/link-budget.js +239 -0
  18. package/dist/link-budget.js.map +1 -0
  19. package/dist/peer-dial.d.ts +36 -17
  20. package/dist/peer-dial.js +37 -17
  21. package/dist/peer-dial.js.map +1 -1
  22. package/dist/peer-join-backfill.d.ts +58 -4
  23. package/dist/peer-join-backfill.js +166 -20
  24. package/dist/peer-join-backfill.js.map +1 -1
  25. package/dist/relay-reservation.d.ts +59 -11
  26. package/dist/relay-reservation.js +169 -43
  27. package/dist/relay-reservation.js.map +1 -1
  28. package/dist/seed-bootstrap.js +2 -1
  29. package/dist/seed-bootstrap.js.map +1 -1
  30. package/dist/strand-first-sync-gate.d.ts +105 -22
  31. package/dist/strand-first-sync-gate.js +121 -27
  32. package/dist/strand-first-sync-gate.js.map +1 -1
  33. package/dist/strand-instance-manager.d.ts +27 -2
  34. package/dist/strand-instance-manager.js +60 -13
  35. package/dist/strand-instance-manager.js.map +1 -1
  36. package/dist/strand-membership-reconciler.d.ts +200 -45
  37. package/dist/strand-membership-reconciler.js +302 -86
  38. package/dist/strand-membership-reconciler.js.map +1 -1
  39. package/dist/strand-membership-writer.d.ts +3 -2
  40. package/dist/strand-membership-writer.js +67 -90
  41. package/dist/strand-membership-writer.js.map +1 -1
  42. package/dist/strand-network-config.d.ts +2 -2
  43. package/dist/strand-network-config.js +2 -2
  44. package/dist/types.d.ts +105 -9
  45. package/dist/types.js +20 -4
  46. package/dist/types.js.map +1 -1
  47. package/package.json +8 -8
  48. package/src/cadre-node.ts +7449 -7397
  49. package/src/control-database.ts +48 -16
  50. package/src/control-read-retry.ts +214 -202
  51. package/src/control-write-retry.ts +24 -7
  52. package/src/index.ts +582 -563
  53. package/src/link-budget.ts +264 -0
  54. package/src/peer-dial.ts +39 -17
  55. package/src/peer-join-backfill.ts +648 -478
  56. package/src/relay-reservation.ts +1119 -951
  57. package/src/seed-bootstrap.ts +2 -1
  58. package/src/strand-first-sync-gate.ts +352 -258
  59. package/src/strand-instance-manager.ts +1520 -1458
  60. package/src/strand-membership-reconciler.ts +334 -92
  61. package/src/strand-membership-writer.ts +1962 -1987
  62. package/src/strand-network-config.ts +2 -2
  63. package/src/types.ts +104 -8
@@ -1,258 +1,352 @@
1
- /**
2
- * The first-sync write gate for a JOINING machine.
3
- *
4
- * A machine that has never received anything from another member of a strand must not
5
- * commit to it. With no strand peer connected, Optimystic's cohort for every block is the
6
- * machine itself, so the first write to a table finds no collection locally and INVENTS
7
- * one — exactly what a founder writing alone legitimately does. When the connection comes
8
- * up, two independently created histories share one collection id: Optimystic keeps one and
9
- * drops the other's commits, or never reconciles them at all. The joiner's own rows silently
10
- * vanish (`joining-machine-writes-before-first-sync-fork-tables`). Optimystic cannot tell a
11
- * founder from a joiner; sereus can, because the launch knows whether it is founding.
12
- *
13
- * The invariant this module enforces: **a machine that has never held this strand's
14
- * `Strand.Header` row, and has not read every `App` table once, must not commit to it.**
15
- * The founder writes the Header in its bootstrap; every other machine can only receive it
16
- * from a peer, so "no local Header" means "never synced". The gate therefore holds the
17
- * freshly initialized `StrandDatabase` back from the app — `StrandInstance.database` stays
18
- * unset and the instance reports `'syncing'` — and probes on a cadence until the Header is
19
- * readable AND a read of each app table settles, at which point the database is published
20
- * and the instance goes `'active'`. A machine that already holds the Header (a restart, a
21
- * hibernation resume, a founder) passes the probe on the first try and is never gated, so
22
- * offline-first writes on a machine that has synced before keep working.
23
- *
24
- * Why the app tables too: the Header is ONE collection. It reaches a joiner ahead of the
25
- * founder's app-table collections (peer-join backfill and pull-on-read deliver those
26
- * separately), and a write to a table whose collection this machine has not yet fetched
27
- * invents one exactly as a lone joiner did. Measured on the reference chat schema
28
- * (2026-09-16, direct connections, Header-only probe): the joiner's participant + message
29
- * written the instant `addStrand` resolved diverged in 3 of 4 runs. Reading each table once
30
- * pulls its collection while the founder is reachable, so the write that follows appends.
31
- *
32
- * Every probe is a read (`Strand.Header`, then `select count(1)` on each `App` table). Reads
33
- * never invent a collection (Optimystic's `Collection.open` resolves undefined on an
34
- * authoritatively absent header, and throws on an unreachable one), so probing is safe to
35
- * repeat and a throw is just "not yet". A table nobody has written yet reads as absent on
36
- * every machine; the first write to it still creates its collection — see `docs/strands.md`
37
- * ("Joining") for that residual.
38
- *
39
- * Owned by `StrandInstanceManager`: created in `buildStrandRuntime` for a non-founder launch
40
- * whose first probe finds no Header, stopped and dropped in `releaseRuntime` (so a quiesce →
41
- * resume rebuild re-probes over the same store), and force-opened when a founder request
42
- * runs the bootstrap against the still-gated database (`foundExistingStrand`).
43
- *
44
- * NOTE: the gate covers app and membership tables, not the schema catalog. Every launch —
45
- * joiner or founder — writes the catalog collection (`optimystic/schema` plus two
46
- * hash-named blocks, measured 2026-09-16) alone, during `connectToStrand`'s schema apply,
47
- * before any peer contact. That is the same "two histories under one id" shape this gate
48
- * exists to prevent, and it is only safe because both sides derive byte-identical content
49
- * from the same schema text. If the catalog ever carries per-machine or ordering-dependent
50
- * content (optimystic's own `0.5-same-named-tables-in-two-schemas-share-storage` re-keys
51
- * it), a joiner's catalog will fork against the founder's — gate the schema apply on the
52
- * Header too, or have optimystic open the catalog read-only when a peer already holds it.
53
- */
54
-
55
- import debug from 'debug';
56
- import type { Database } from '@quereus/quereus';
57
- import type { StrandDatabase } from './strand-database.js';
58
- import { defaultTimeoutScheduler, type TimeoutScheduler } from './timeout-scheduler.js';
59
-
60
- const log = debug('sereus:cadre:strand-first-sync');
61
-
62
- /** How often a gated launch re-probes (`Strand.Header`, then every `App` table) while waiting for its first sync. */
63
- export const DEFAULT_STRAND_FIRST_SYNC_POLL_MS = 500;
64
-
65
- /**
66
- * How long `CadreNode.addStrand` waits for a joining machine's first sync before rejecting
67
- * with {@link StrandAwaitingFirstSyncError}. Measured: over a direct connection the
68
- * joiner reads the founder's rows about 1.3 s after `addStrand` would previously have
69
- * resolved; a relayed path is slower, and a phone that just redeemed an invitation has the
70
- * host reachable moments ago, so 30 s is generous without leaving an app hanging.
71
- */
72
- export const DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS = 30_000;
73
-
74
- /** Embedder-facing tuning for the gate, threaded from `CadreNodeConfig.strandFirstSync`. */
75
- export interface StrandFirstSyncConfig {
76
- /**
77
- * Default wait for `CadreNode.addStrand` (and `whenStrandWritable`) before a joining
78
- * machine that has reached no other member is reported as not yet writable. Default
79
- * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}.
80
- */
81
- timeoutMs?: number;
82
- /** Probe cadence while gated. Default {@link DEFAULT_STRAND_FIRST_SYNC_POLL_MS}. */
83
- pollIntervalMs?: number;
84
- }
85
-
86
- /**
87
- * Thrown by `CadreNode.addStrand` / `whenStrandWritable` when a joining machine has not
88
- * received the strand's data from any other member within the wait budget. RETRYABLE: the
89
- * strand stays launched and keeps probing, so a later `addStrand` (or `whenStrandWritable`)
90
- * for the same strand completes the attach once another member is reachable; the
91
- * `strand:writable` event fires at that moment too.
92
- */
93
- export class StrandAwaitingFirstSyncError extends Error {
94
- readonly strandId: string;
95
- readonly waitedMs: number;
96
-
97
- constructor(strandId: string, waitedMs: number) {
98
- super(
99
- `Strand ${strandId} is not writable yet: no member of this strand has been reachable ` +
100
- `since this machine joined, so it has not received the strand's data (waited ${waitedMs} ms). ` +
101
- 'The strand stays launched and keeps trying — call addStrand again, or wait for the ' +
102
- "'strand:writable' event, once another member is reachable."
103
- );
104
- this.name = 'StrandAwaitingFirstSyncError';
105
- this.strandId = strandId;
106
- this.waitedMs = waitedMs;
107
- }
108
- }
109
-
110
- /**
111
- * Whether this machine holds the strand's `Strand.Header` row — the "has synced at least
112
- * once (or founded)" signal. A read, never a write: it cannot invent the collection. A
113
- * throwing read (an unreachable cohort mid-sync, a store still hydrating) reports `false`
114
- * with a log, since the caller's only response to either answer is "probe again".
115
- */
116
- export async function strandHeaderHeld(db: Database, label: string): Promise<boolean> {
117
- try {
118
- for await (const row of db.eval('select count(1) as Count from Strand.Header')) {
119
- return ((row.Count as number) ?? 0) > 0;
120
- }
121
- return false;
122
- } catch (error) {
123
- log('[%s] Header probe failed (treated as not yet held): %s', label,
124
- error instanceof Error ? error.message : String(error));
125
- return false;
126
- }
127
- }
128
-
129
- /** The tables the sApp declared in the `App` schema, by name; views excluded. */
130
- function appTableNames(db: Database): string[] {
131
- const app = db.schemaManager.getSchema('App');
132
- return app ? Array.from(app.getAllTables()).filter((table) => !table.isView).map((table) => table.name) : [];
133
- }
134
-
135
- /**
136
- * Whether a read of every `App` table settles on this machine — the "each app collection
137
- * has been fetched from the cohort, or is authoritatively absent" signal. Only a throwing
138
- * read (an unreachable cohort) reports `false`; an empty table is a settled read.
139
- */
140
- export async function appTablesReadable(db: Database, label: string): Promise<boolean> {
141
- for (const table of appTableNames(db)) {
142
- try {
143
- for await (const _row of db.eval(`select count(1) as Count from App."${table}"`)) {
144
- break;
145
- }
146
- } catch (error) {
147
- log('[%s] App.%s probe failed (treated as not yet synced): %s', label, table,
148
- error instanceof Error ? error.message : String(error));
149
- return false;
150
- }
151
- }
152
- return true;
153
- }
154
-
155
- /**
156
- * The gate's whole probe: the Header is held AND every app table has been read once. The
157
- * app tables are read only once the Header is — a joiner with no peer would otherwise pay
158
- * one failing network read per table per probe for nothing.
159
- */
160
- export async function strandFirstSyncComplete(db: Database, label: string): Promise<boolean> {
161
- return await strandHeaderHeld(db, label) && await appTablesReadable(db, label);
162
- }
163
-
164
- export interface StrandFirstSyncGateDeps {
165
- /** Log tag naming which strand this gate holds (the strand id). */
166
- label: string;
167
- /** The initialized database being held back from the app until the Header is held. */
168
- database: StrandDatabase;
169
- /**
170
- * Called exactly once, on the gate's own probe loop, when the first sync completes.
171
- * NOT called by {@link StrandFirstSyncGate.open} — a caller that force-opens the gate
172
- * (a founder bootstrap it just ran) publishes the database itself.
173
- */
174
- onHeaderHeld: () => void;
175
- /** Timer seam for the probe loop; omit for real (unref'd) timeouts. */
176
- scheduler?: TimeoutScheduler;
177
- }
178
-
179
- /**
180
- * Holds one launch's `StrandDatabase` until {@link strandFirstSyncComplete}. Probes are
181
- * sequential (the next is scheduled only after the previous read settles), so a slow
182
- * network read never stacks probes.
183
- */
184
- export class StrandFirstSyncGate {
185
- private readonly scheduler: TimeoutScheduler;
186
- private readonly pollIntervalMs: number;
187
- private timer: unknown;
188
- private stopped = false;
189
- private opened = false;
190
- private probing = false;
191
-
192
- constructor(private readonly deps: StrandFirstSyncGateDeps, config?: StrandFirstSyncConfig) {
193
- this.scheduler = deps.scheduler ?? defaultTimeoutScheduler;
194
- this.pollIntervalMs = config?.pollIntervalMs ?? DEFAULT_STRAND_FIRST_SYNC_POLL_MS;
195
- }
196
-
197
- /** The database this gate is holding back. */
198
- get database(): StrandDatabase {
199
- return this.deps.database;
200
- }
201
-
202
- /** True once the Header was seen (or the gate was force-opened); the loop is then done. */
203
- get isOpen(): boolean {
204
- return this.opened;
205
- }
206
-
207
- /** Arm the probe loop. The first probe runs after one interval — the caller already probed once. */
208
- start(): void {
209
- if (this.stopped || this.opened || this.timer !== undefined) return;
210
- this.schedule();
211
- log('[%s] first-sync gate armed: no Strand.Header held yet; probing every %dms',
212
- this.deps.label, this.pollIntervalMs);
213
- }
214
-
215
- /**
216
- * Force-open: the caller has just written the Header itself (a founder bootstrap run
217
- * against the gated database). Stops the loop; `onHeaderHeld` is NOT invoked.
218
- */
219
- open(): void {
220
- if (this.opened) return;
221
- this.opened = true;
222
- this.stop();
223
- log('[%s] first-sync gate opened by the caller (Header written locally)', this.deps.label);
224
- }
225
-
226
- /** Disarm the loop; a probe already in flight completes and is then ignored. */
227
- stop(): void {
228
- this.stopped = true;
229
- if (this.timer !== undefined) {
230
- this.scheduler.clearTimeout(this.timer);
231
- this.timer = undefined;
232
- }
233
- }
234
-
235
- private schedule(): void {
236
- this.timer = this.scheduler.setTimeout(() => {
237
- this.timer = undefined;
238
- void this.probe();
239
- }, this.pollIntervalMs);
240
- }
241
-
242
- private async probe(): Promise<void> {
243
- if (this.stopped || this.opened || this.probing) return;
244
- this.probing = true;
245
- const held = await strandFirstSyncComplete(this.deps.database.getDatabase(), this.deps.label)
246
- .finally(() => { this.probing = false; });
247
- // Re-check after the await: a release or a force-open may have landed mid-read.
248
- if (this.stopped || this.opened) return;
249
- if (!held) {
250
- this.schedule();
251
- return;
252
- }
253
- this.opened = true;
254
- this.stop();
255
- log('[%s] first-sync gate opened: Strand.Header and every App table read from a peer', this.deps.label);
256
- this.deps.onHeaderHeld();
257
- }
258
- }
1
+ /**
2
+ * The first-sync write gate for a JOINING machine.
3
+ *
4
+ * A machine that has never received anything from another member of a strand must not
5
+ * commit to it. With no strand peer connected, Optimystic's cohort for every block is the
6
+ * machine itself, so the first write to a table finds no collection locally and INVENTS
7
+ * one — exactly what a founder writing alone legitimately does. When the connection comes
8
+ * up, two independently created histories share one collection id: Optimystic keeps one and
9
+ * drops the other's commits, or never reconciles them at all. The joiner's own rows silently
10
+ * vanish (`joining-machine-writes-before-first-sync-fork-tables`). Optimystic cannot tell a
11
+ * founder from a joiner; sereus can, because the launch knows whether it is founding.
12
+ *
13
+ * The invariant this module enforces: **a machine that has never held this strand's
14
+ * `Strand.Header` row, and has not read every `App` table once, must not commit to it.**
15
+ * The founder writes the Header in its bootstrap; every other machine can only receive it
16
+ * from a peer, so "no local Header" means "never synced". The gate therefore holds the
17
+ * freshly initialized `StrandDatabase` back from the app — `StrandInstance.database` stays
18
+ * unset and the instance reports `'syncing'` — and probes on a cadence until the Header is
19
+ * readable AND a read of each app table settles, at which point the database is published
20
+ * and the instance goes `'active'`. A machine whose store already holds the Header and every
21
+ * app table (a founder, and normally a restart or a hibernation resume) passes the probe on
22
+ * the first try and is never gated, so offline-first writes on a machine that has synced
23
+ * before keep working.
24
+ *
25
+ * "Has synced before" is decided by what the local store actually holds, not by whether this
26
+ * machine once read the strand: a machine that left soon after its first sync can hold less
27
+ * than it read, and is then gated like a joiner. The re-attach measurement on
28
+ * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS} saw exactly that — a machine back over the store
29
+ * it kept, without the `Strand.Header` collection it had read before it left. Gating it is
30
+ * right: writing into a collection this machine does not hold would fork it.
31
+ *
32
+ * Why the app tables too: the Header is ONE collection. It reaches a joiner ahead of the
33
+ * founder's app-table collections (peer-join backfill and pull-on-read deliver those
34
+ * separately), and a write to a table whose collection this machine has not yet fetched
35
+ * invents one exactly as a lone joiner did. Measured on the reference chat schema
36
+ * (2026-09-16, direct connections, Header-only probe): the joiner's participant + message
37
+ * written the instant `addStrand` resolved diverged in 3 of 4 runs. Reading each table once
38
+ * pulls its collection while the founder is reachable, so the write that follows appends.
39
+ *
40
+ * Every probe is a read (`Strand.Header`, then `select count(1)` on each `App` table). Reads
41
+ * never invent a collection (Optimystic's `Collection.open` resolves undefined on an
42
+ * authoritatively absent header, and throws on an unreachable one), so probing is safe to
43
+ * repeat and a throw is just "not yet". A table nobody has written yet reads as absent on
44
+ * every machine; the first write to it still creates its collection — see `docs/strands.md`
45
+ * ("Joining") for that residual.
46
+ *
47
+ * Owned by `StrandInstanceManager`: created in `buildStrandRuntime` for every launch, armed
48
+ * only for a non-founder launch whose first probe fails, stopped and dropped in
49
+ * `releaseRuntime` (so a quiesce → resume rebuild re-probes over the same store), and opened
50
+ * by `publishDatabase` whenever the database is published without its loop — a launch that
51
+ * needed no gate, or a founder request that ran the bootstrap against the still-gated
52
+ * database (`foundExistingStrand`).
53
+ *
54
+ * NOTE: the gate covers app and membership tables, not the schema catalog. Every launch —
55
+ * joiner or founder — writes the catalog collection (`optimystic/schema` plus two
56
+ * hash-named blocks, measured 2026-09-16) alone, during `connectToStrand`'s schema apply,
57
+ * before any peer contact. That is the same "two histories under one id" shape this gate
58
+ * exists to prevent, and it is only safe because both sides derive byte-identical content
59
+ * from the same schema text. If the catalog ever carries per-machine or ordering-dependent
60
+ * content (optimystic's own `0.5-same-named-tables-in-two-schemas-share-storage` re-keys
61
+ * it), a joiner's catalog will fork against the founder's — gate the schema apply on the
62
+ * Header too, or have optimystic open the catalog read-only when a peer already holds it.
63
+ */
64
+
65
+ import debug from 'debug';
66
+ import type { Database } from '@quereus/quereus';
67
+ import type { StrandDatabase } from './strand-database.js';
68
+ import { defaultTimeoutScheduler, type TimeoutScheduler } from './timeout-scheduler.js';
69
+
70
+ const log = debug('sereus:cadre:strand-first-sync');
71
+
72
+ /** How often a gated launch re-probes (`Strand.Header`, then every `App` table) while waiting for its first sync. */
73
+ export const DEFAULT_STRAND_FIRST_SYNC_POLL_MS = 500;
74
+
75
+ /**
76
+ * How long `CadreNode.addStrand` waits for a machine's first sync before rejecting with
77
+ * {@link StrandAwaitingFirstSyncError}. It has to cover the slowest attach sereus carries,
78
+ * which is not a first join: a machine RE-attaching through a relay on a slow link can take
79
+ * longer than one joining for the first time.
80
+ *
81
+ * Measured over a DIRECT connection (2026-09-16): the joiner reads the founder's rows about
82
+ * 1.3 s after `addStrand` would previously have resolved. That is the fast case, and it is not
83
+ * what this budget has to cover.
84
+ *
85
+ * Every harness figure below was taken on one shape: one Windows developer machine, two
86
+ * relay-only `CadreNode`s (`listenAddrs: []`) on a shared loopback dedicated relay, with a
87
+ * 900 ms one-way per-frame outbound delay applied to every websocket by
88
+ * `integration-tests/src/harness/ws-latency.ts` in `pipelined` mode — frames stay overlapped
89
+ * in flight, so the figure is latency and bandwidth stays unlimited; the harness's other mode
90
+ * (`serial`) is a per-socket frame-rate cap and its delays are NOT comparable
91
+ * (`docs/testing.md` → "Link latency"). A round trip therefore costs about 1.8 s. Times run
92
+ * from the `addStrand` call to writable.
93
+ *
94
+ * FRESH JOIN (2026-09-26), a machine holding nothing of the strand yet: 23, 27, 31 and 41 s
95
+ * over four runs at optimystic's 1000 ms cohort read deadline, and 35, 42 and 46 s over three
96
+ * runs at a 5000 ms one; the re-attach scenario below later took 38.7, 38.7 and 56.9 s for
97
+ * the same fresh join. **The 5000 ms band, 35-57 s, is the one in force**: sereus declares
98
+ * 5000 ms (`COHORT_READ_DEADLINE_MS`, `quereus-plugin-sereus/src/cluster-size.ts`), because
99
+ * 1000 ms is shorter than one round trip on that link and leaves every cohort read with no
100
+ * answer to corroborate against. Widening that deadline makes a consult against a peer that
101
+ * cannot answer cost longer, and this phase runs several of those — which is why both bands
102
+ * are recorded here, and why a change to either number has to be weighed against the other.
103
+ *
104
+ * RE-ATTACH (2026-09-26, the opt-in
105
+ * `integration-tests/src/scenarios/strand-reattach-first-sync-measure.integration.ts` — re-run
106
+ * it before changing this number): B attached once on an undelayed link, left with
107
+ * `stopStrand`, A wrote while it was away, the delay was raised, and B called `addStrand`
108
+ * again.
109
+ * - Over an EMPTY store (one minted per launch, cadre's default with no storage provider):
110
+ * gated in all four runs and writable at 38.7-56.8 s, the same as a fresh join.
111
+ * - Over the store it KEPT (a phone with durable storage), bimodal. In 3 of 8 runs the store
112
+ * held everything and the launch came up writable (5.6 s, the launch itself). In the other
113
+ * 5 it lacked the `Strand.Header` collection B had read on its first attach, so B was gated
114
+ * like a joiner and became writable at 64.1-78.7 s — the slowest attach measured, about
115
+ * twice a fresh join. Why syncing over a partial store is slower sits inside optimystic
116
+ * (`tickets/blocked/report-reattach-over-partial-replica-to-optimystic`).
117
+ * - Neither a redial nor catching up on missed writes explains the time: B's strand node
118
+ * connected to A's at 11 s in every arm, fresh join included, and a run with 20 missed
119
+ * writes fell inside the run-to-run spread of those with 1.
120
+ * This budget covers WRITABLE, not caught up: in the gated kept-store runs the row A wrote
121
+ * while B was away became readable only at 140-162 s.
122
+ *
123
+ * Two samples from outside this harness, neither reproduced here and neither discarded: a
124
+ * reporter's re-attach of the same shape (optimystic #22; their harness, storage and delay
125
+ * injector are unknown to us) opened the gate at about 150 s, after the previous 120 s budget
126
+ * had already rejected; and a real Galaxy S7 joining fresh through relay.sereus.org took 178 s.
127
+ *
128
+ * 300 s clears the worst harness sample (79 s) by about 3.8x, the reporter's re-attach by 2x
129
+ * and the device join by about 1.7x. 240 s was rejected: it clears the device join by only
130
+ * 1.35x, and that sample was not a re-attach. History: the original 30 s sat INSIDE the 1000 ms
131
+ * fresh band and refused about half of those joins while their sync was progressing normally;
132
+ * 120 s was sized from fresh joins only and refused the reporter's re-attach.
133
+ *
134
+ * NOTE: accepted tradeoff — what 300 s costs. This wait is what an app's `addStrand` sits in
135
+ * before it is told "not yet", so a strand none of whose members is reachable at all takes five
136
+ * minutes to report instead of two. The slow report was weighed against refusing attaches that
137
+ * were working, and the refusal is the worse failure. The cost is bounded — the rejection is
138
+ * retryable, the strand stays launched and keeps probing, and `strand:writable` fires the
139
+ * moment the sync lands, so an app that listens for the event rather than awaiting the call is
140
+ * unaffected either way. Revisit if the gate ever learns whether ANY other member is connected:
141
+ * "no peer at all" could then be reported at once and this budget would only ever be spent on a
142
+ * sync that is actually in progress — worth more the longer this budget gets.
143
+ */
144
+ export const DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS = 300_000;
145
+
146
+ /** Embedder-facing tuning for the gate, threaded from `CadreNodeConfig.strandFirstSync`. */
147
+ export interface StrandFirstSyncConfig {
148
+ /**
149
+ * Default wait for `CadreNode.addStrand` (and `whenStrandWritable`) before a machine whose
150
+ * first sync has not completed is reported as not yet writable. Default
151
+ * {@link DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS}.
152
+ */
153
+ timeoutMs?: number;
154
+ /** Probe cadence while gated. Default {@link DEFAULT_STRAND_FIRST_SYNC_POLL_MS}. */
155
+ pollIntervalMs?: number;
156
+ }
157
+
158
+ /**
159
+ * Thrown by `CadreNode.addStrand` / `whenStrandWritable` when a machine has not received the
160
+ * strand's data from another member within the wait budget — because no other member is
161
+ * reachable, or because one is and the sync is still arriving over a slow link; the gate
162
+ * cannot tell the two apart. RETRYABLE: the strand stays launched and keeps probing, so a
163
+ * later `addStrand` (or `whenStrandWritable`) for the same strand completes the attach once
164
+ * the data has arrived; the `strand:writable` event fires at that moment too.
165
+ */
166
+ export class StrandAwaitingFirstSyncError extends Error {
167
+ readonly strandId: string;
168
+ readonly waitedMs: number;
169
+
170
+ constructor(strandId: string, waitedMs: number) {
171
+ super(
172
+ `Strand ${strandId} is not writable yet: this machine has not yet received the strand's data ` +
173
+ `from another member (waited ${waitedMs} ms). Either no other member is reachable, or the ` +
174
+ 'link to one is slow and the sync is still in progress. The strand stays launched and keeps ' +
175
+ 'trying — call addStrand or whenStrandWritable(strandId) again to keep waiting, or listen ' +
176
+ "for the 'strand:writable' event."
177
+ );
178
+ this.name = 'StrandAwaitingFirstSyncError';
179
+ this.strandId = strandId;
180
+ this.waitedMs = waitedMs;
181
+ }
182
+ }
183
+
184
+ /**
185
+ * Whether this machine holds the strand's `Strand.Header` row — the "has synced at least
186
+ * once (or founded)" signal. A read, never a write: it cannot invent the collection. A
187
+ * throwing read (an unreachable cohort mid-sync, a store still hydrating) reports `false`
188
+ * with a log, since the caller's only response to either answer is "probe again".
189
+ */
190
+ export async function strandHeaderHeld(db: Database, label: string): Promise<boolean> {
191
+ try {
192
+ for await (const row of db.eval('select count(1) as Count from Strand.Header')) {
193
+ return ((row.Count as number) ?? 0) > 0;
194
+ }
195
+ return false;
196
+ } catch (error) {
197
+ log('[%s] Header probe failed (treated as not yet held): %s', label,
198
+ error instanceof Error ? error.message : String(error));
199
+ return false;
200
+ }
201
+ }
202
+
203
+ /** The tables the sApp declared in the `App` schema, by name; views excluded. */
204
+ function appTableNames(db: Database): string[] {
205
+ const app = db.schemaManager.getSchema('App');
206
+ return app ? Array.from(app.getAllTables()).filter((table) => !table.isView).map((table) => table.name) : [];
207
+ }
208
+
209
+ /**
210
+ * Whether a read of every `App` table settles on this machine — the "each app collection
211
+ * has been fetched from the cohort, or is authoritatively absent" signal. Only a throwing
212
+ * read (an unreachable cohort) reports `false`; an empty table is a settled read.
213
+ */
214
+ export async function appTablesReadable(db: Database, label: string): Promise<boolean> {
215
+ for (const table of appTableNames(db)) {
216
+ try {
217
+ for await (const _row of db.eval(`select count(1) as Count from App."${table}"`)) {
218
+ break;
219
+ }
220
+ } catch (error) {
221
+ log('[%s] App.%s probe failed (treated as not yet synced): %s', label, table,
222
+ error instanceof Error ? error.message : String(error));
223
+ return false;
224
+ }
225
+ }
226
+ return true;
227
+ }
228
+
229
+ /**
230
+ * The gate's whole probe: the Header is held AND every app table has been read once. The
231
+ * app tables are read only once the Header is — a joiner with no peer would otherwise pay
232
+ * one failing network read per table per probe for nothing.
233
+ */
234
+ export async function strandFirstSyncComplete(db: Database, label: string): Promise<boolean> {
235
+ return await strandHeaderHeld(db, label) && await appTablesReadable(db, label);
236
+ }
237
+
238
+ export interface StrandFirstSyncGateDeps {
239
+ /** Log tag naming which strand this gate holds (the strand id). */
240
+ label: string;
241
+ /** The initialized database being held back from the app until the Header is held. */
242
+ database: StrandDatabase;
243
+ /**
244
+ * Called exactly once, on the gate's own probe loop, when the first sync completes.
245
+ * NOT called by {@link StrandFirstSyncGate.open} — a caller that force-opens the gate
246
+ * (a founder bootstrap it just ran) publishes the database itself.
247
+ */
248
+ onHeaderHeld: () => void;
249
+ /** Timer seam for the probe loop; omit for real (unref'd) timeouts. */
250
+ scheduler?: TimeoutScheduler;
251
+ }
252
+
253
+ /**
254
+ * Holds one launch's `StrandDatabase` until {@link strandFirstSyncComplete}. Probes are
255
+ * sequential (the next is scheduled only after the previous read settles), so a slow
256
+ * network read never stacks probes.
257
+ */
258
+ export class StrandFirstSyncGate {
259
+ private readonly scheduler: TimeoutScheduler;
260
+ private readonly pollIntervalMs: number;
261
+ private timer: unknown;
262
+ private stopped = false;
263
+ private opened = false;
264
+ private probing = false;
265
+
266
+ constructor(private readonly deps: StrandFirstSyncGateDeps, config?: StrandFirstSyncConfig) {
267
+ this.scheduler = deps.scheduler ?? defaultTimeoutScheduler;
268
+ this.pollIntervalMs = config?.pollIntervalMs ?? DEFAULT_STRAND_FIRST_SYNC_POLL_MS;
269
+ }
270
+
271
+ /** The database this gate is holding back. */
272
+ get database(): StrandDatabase {
273
+ return this.deps.database;
274
+ }
275
+
276
+ /** True once the Header was seen (or the gate was force-opened); the loop is then done. */
277
+ get isOpen(): boolean {
278
+ return this.opened;
279
+ }
280
+
281
+ /** Arm the probe loop. The first probe runs after one interval — the caller already probed once. */
282
+ start(): void {
283
+ if (this.stopped || this.opened || this.timer !== undefined) return;
284
+ this.schedule();
285
+ log('[%s] first-sync gate armed: no Strand.Header held yet; probing every %dms',
286
+ this.deps.label, this.pollIntervalMs);
287
+ }
288
+
289
+ /**
290
+ * Open without the probe loop: the caller is publishing the database itself — a launch
291
+ * whose first probe passed (or a founder's), which never armed the loop, or a founder
292
+ * bootstrap just run against the gated database. Stops the loop; `onHeaderHeld` is NOT
293
+ * invoked.
294
+ */
295
+ open(): void {
296
+ if (this.opened) return;
297
+ this.opened = true;
298
+ this.stop();
299
+ log('[%s] database published by the caller; first-sync gate not needed or force-opened', this.deps.label);
300
+ }
301
+
302
+ /** Disarm the loop; a probe already in flight completes and is then ignored. */
303
+ stop(): void {
304
+ this.stopped = true;
305
+ if (this.timer !== undefined) {
306
+ this.scheduler.clearTimeout(this.timer);
307
+ this.timer = undefined;
308
+ }
309
+ }
310
+
311
+ private schedule(): void {
312
+ this.timer = this.scheduler.setTimeout(() => {
313
+ this.timer = undefined;
314
+ void this.probe();
315
+ }, this.pollIntervalMs);
316
+ }
317
+
318
+ private async probe(): Promise<void> {
319
+ if (this.stopped || this.opened || this.probing) return;
320
+ this.probing = true;
321
+ const held = await this.probeHeld();
322
+ // Re-check after the await: a release or a force-open may have landed mid-read.
323
+ if (this.stopped || this.opened) return;
324
+ if (!held) {
325
+ this.schedule();
326
+ return;
327
+ }
328
+ this.opened = true;
329
+ this.stop();
330
+ log('[%s] first-sync gate opened: Strand.Header and every App table read from a peer', this.deps.label);
331
+ this.deps.onHeaderHeld();
332
+ }
333
+
334
+ /**
335
+ * One probe's answer, with a throw from OUTSIDE {@link strandFirstSyncComplete}'s own read
336
+ * guards — the `App` schema lookup, or `getDatabase()` itself — reported as "not yet" like
337
+ * any failing read. The loop is scheduled from `void this.probe()`, so an escaping rejection
338
+ * would be unhandled AND leave nothing scheduled: the strand would stay gated for the whole
339
+ * `timeoutMs` and never recover, which is the one failure this gate must not have.
340
+ */
341
+ private async probeHeld(): Promise<boolean> {
342
+ try {
343
+ return await strandFirstSyncComplete(this.deps.database.getDatabase(), this.deps.label);
344
+ } catch (error) {
345
+ log('[%s] first-sync probe threw (treated as not yet synced): %s', this.deps.label,
346
+ error instanceof Error ? error.message : String(error));
347
+ return false;
348
+ } finally {
349
+ this.probing = false;
350
+ }
351
+ }
352
+ }