@helipod/fleet 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1777 @@
1
+ import { PgClient, PgQuerier, PostgresDocStore, NodePgClient } from '@helipod/docstore-postgres';
2
+ import { ShardId } from '@helipod/id-codec';
3
+ export { docKeyToPointRange, keyToPointRange } from '@helipod/id-codec';
4
+ import { JSONValue } from '@helipod/values';
5
+ import { EmbeddedWriteFanoutAdapter, EmbeddedWriteFanoutPayload, FanoutListener, WriteRouter, ClientReplay, EmbeddedRuntime } from '@helipod/runtime-embedded';
6
+ import { InternalDocumentId, DocStore, SchemaSetupOptions, DocumentLogEntry, IndexWrite, ConflictStrategy, ShardId as ShardId$1, CommitUnit, CommitGuardUnit, LatestDocument, Interval, Order, TimestampRange, PrevRevQuery, ClientVerdictRecord, ClientVerdictWrite } from '@helipod/docstore';
7
+ import { SqliteDocStore } from '@helipod/docstore-sqlite';
8
+
9
+ /**
10
+ * Non-blocking per-shard commit-mutex seam (see `Transactor.tryRunExclusiveOnShard` /
11
+ * `EmbeddedRuntime.tryRunExclusiveOnShard`): run `fn` under shard `shardId`'s commit mutex if that
12
+ * shard is IDLE right now (returns `true`), else skip without running it (returns `false`). "Idle"
13
+ * means BOTH the commit mutex is free AND no group-commit batch is staged/flushing — Fleet B4's group
14
+ * commit runs the flush I/O OFF the mutex, so mutex-freedom alone no longer implies "nothing in
15
+ * flight". `closeIdleFrontiers` takes this so each idle shard's frontier bump is mutually exclusive
16
+ * with that shard's own commits AND with any mid-flush batch — the fix for the frontier-inversion race
17
+ * (an idle-closer publishing a frontier ahead of an in-flight commit's or batch's not-yet-landed
18
+ * rows). The writer owns both the closer and every shard's commits, so this single-process check is
19
+ * airtight; cross-node closing of a held shard is impossible by design (only the lease holder closes
20
+ * its own held shards).
21
+ */
22
+ type TryRunExclusiveOnShard = (shardId: ShardId, fn: () => Promise<void>) => Promise<boolean>;
23
+ /** The current fleet writer lease: which epoch is live and which node holds it. */
24
+ interface LeaseState {
25
+ epoch: bigint;
26
+ writerUrl: string;
27
+ /** The row's fenced-frontier high-water mark at acquisition time (Fleet B3). On an `ON CONFLICT`
28
+ * re-acquire the upsert leaves `frontier_ts` untouched, so this returns the value an INTERIM owner
29
+ * advanced it to while this node didn't hold the shard — the exact floor the caller feeds to
30
+ * `runtime.observeWriteTimestamp` so a re-acquired shard's write snapshot never sits stale. */
31
+ frontierTs: bigint;
32
+ }
33
+ /** `LeaseManager.read()`'s full row — every `shard_leases` column (Fenced Frontier B1, D2). The
34
+ * extra columns beyond `LeaseState` are consumed by the D4 eviction/D5 tailer-frontier work; this
35
+ * class only writes them (frontier_ts/prev_ts advance via the commit-guard SQL installed in
36
+ * `node.ts`, never through this class directly), so they're returned loosely typed for now. */
37
+ interface LeaseRow extends LeaseState {
38
+ writerAppName: string | null;
39
+ /** Raw Postgres value for the TIMESTAMPTZ column (a `Date` under `NodePgClient`, ISO-ish string
40
+ * under PGlite) — not a `LeaseManager` client's job to interpret; a heartbeat/fence caller only
41
+ * cares about row-count effects, not the wall-clock value. */
42
+ expiresAt: unknown;
43
+ frontierTs: bigint;
44
+ prevTs: bigint;
45
+ }
46
+ interface LeaseManagerOptions {
47
+ /** URL this node advertises as the writer, recorded into shard_leases on acquire. */
48
+ advertiseUrl: string;
49
+ /** This node's Postgres `application_name` (see `fleetApplicationName`), recorded on acquire so
50
+ * a D4 eviction fencer can `pg_terminate_backend` the exact wedged holder's connection. */
51
+ applicationName?: string;
52
+ /** Interval between tryAcquire() attempts inside acquireLoop(). Default 2000ms. */
53
+ retryMs?: number;
54
+ /** How long a fresh acquisition/heartbeat extends `expires_at` by, in ms. Default 15000ms. The
55
+ * whole failover clock scales with this: a wedged writer's lease expires this long after its last
56
+ * heartbeat, so a follower's eviction can't fire before then. Shortened by
57
+ * `HELIPOD_FLEET_LEASE_TTL_MS` for the wedged-writer E2E (and available to operators as tuning);
58
+ * the LeaseMonitor's probe cadence is derived from it so a live writer always renews in time. */
59
+ ttlMs?: number;
60
+ }
61
+ /** Effectively-once forwarding (Fleet B3, D3): the `fleet_idempotency.value_json` cap. A recorded
62
+ * mutation result larger than this is NOT stored — the row keeps `value_json = NULL` and
63
+ * `oversized = true` instead, so a replay reports `valueMissing: true` rather than growing this
64
+ * control table unboundedly on a large mutation return value. The WRITE itself is unaffected
65
+ * either way (this cap only governs the best-effort RESULT-VALUE cache, not the commit). */
66
+ declare const IDEMPOTENCY_VALUE_CAP_BYTES: number;
67
+ /** A `fleet_idempotency` row read back for replay (Fleet B3, D3). `hasValue` distinguishes a
68
+ * genuinely-recorded value (including a mutation that legitimately returned JSON `null`, which is
69
+ * still stored as the TEXT `"null"`) from `value_json` being SQL NULL — the crash-window
70
+ * (commit landed, the post-run value UPDATE never ran) and the oversized-cap cases both leave
71
+ * `value_json` SQL NULL, and both replay as `valueMissing: true` uniformly. */
72
+ interface IdempotencyReplay {
73
+ commitTs: bigint;
74
+ hasValue: boolean;
75
+ value: JSONValue | null;
76
+ oversized: boolean;
77
+ }
78
+ /**
79
+ * Coordinates the single-writer lease across a fleet of nodes sharing one Postgres database.
80
+ * The advisory lock (PgClient.tryAcquireWriterLock) is the FAST-PATH mutual-exclusion primitive;
81
+ * `shard_leases` is the fencing token + discovery row + frontier chain — one row per shard (B1:
82
+ * only `'default'`) so any node (including read replicas forwarding writes) can find the current
83
+ * writer's URL/epoch, and so `PostgresDocStore`'s installed commit guard can verify — inside every
84
+ * commit transaction — that the writer holding the advisory lock is STILL the epoch on file (see
85
+ * `node.ts`'s `installCommitGuard`). `epoch` bumps on every acquisition (D2); `frontier_ts`/
86
+ * `prev_ts` are the durable-commit chain the guard advances (D3) — this class never writes them
87
+ * itself beyond seeding them to 0 on first creation.
88
+ *
89
+ * Liveness: the LeaseMonitor's periodic probe IS `heartbeat()` (see `node.ts`) — one round-trip
90
+ * serves liveness-probe + TTL maintenance + fence verification, per D2. `heartbeat()` returning 0
91
+ * rows means this node's epoch has been superseded (fenced) even though its connection is still
92
+ * alive — a DEFINITIVE loss, distinct from the probe-miss tolerance used for transient blips.
93
+ */
94
+ declare class LeaseManager {
95
+ private readonly client;
96
+ /** This node's advertised URL, recorded onto `shard_leases`/`fleet_nodes`. Public so the balancer
97
+ * can use it as this node's rendezvous identity (`ShardLeaseBalancer.myUrl`) without threading it
98
+ * separately through `startFleetNode`. */
99
+ readonly advertiseUrl: string;
100
+ private readonly applicationName;
101
+ private readonly retryMs;
102
+ /** The lease TTL in ms this manager stamps onto `expires_at`. Public so the LeaseMonitor can
103
+ * derive its probe cadence from the SAME knob (see `startFleetNode`) — a live writer must renew
104
+ * well within the TTL. */
105
+ readonly ttlMs: number;
106
+ /** `ttlMs` as a whole-ms integer, ready to interpolate into the `interval '<n> milliseconds'` SQL
107
+ * below. Integer-coerced (never a fraction/NaN) so the interpolation can't produce invalid SQL. */
108
+ private readonly ttlMsSql;
109
+ private timer;
110
+ private stopped;
111
+ /** Per-shard epoch map: `shardId → the epoch this node most recently acquired for it`. B1 tracked
112
+ * ONE `lastEpoch` (default shard only); B2a's writer holds N shards, each fenced against its own
113
+ * epoch, so the commit guard/heartbeat/idle-closer look up `currentEpoch(shardId)`. Updated on
114
+ * every successful per-shard `tryAcquire(shardId)` (including re-promotion's epoch bumps) so the
115
+ * guard always fences against the CURRENT epoch, not a boot snapshot. A shard absent from the
116
+ * map is one this node has never acquired. */
117
+ private readonly lastEpochByShard;
118
+ constructor(client: PgClient, opts: LeaseManagerOptions);
119
+ /**
120
+ * Idempotent DDL: creates shard_leases if it doesn't already exist, and — when `shardIds` is
121
+ * non-empty — pre-seeds a row for every shard in the list (Fleet B3, D4: the concurrent-boot
122
+ * count-gate fix, reversing B2a's "no pre-seeding" call).
123
+ *
124
+ * B2a originally left this deliberately EMPTY (no pre-seeded rows): a bare `frontier_ts = 0` row
125
+ * created at DDL time, on a database that already held documents (a pre-`--fleet` upgrade), would
126
+ * be momentarily visible at `frontier_ts = 0` to a concurrently-booting sync node — `count(*) = N`
127
+ * AND `min(frontier_ts) = 0` would fake-report ready with an EMPTY replica (the F1×N hole,
128
+ * recurring ×N). That is why row existence used to be tied to a REAL, frontier-seeded acquisition
129
+ * (`tryAcquire`) rather than to `setup()`.
130
+ *
131
+ * The reversal here is safe ONLY because the seed now travels WITH the row: `documentsExist` (the
132
+ * caller's `documentsTableExists` probe, run BEFORE calling this) selects the exact same
133
+ * `frontierSeedExpr` fragment `tryAcquire` uses — `0` on a fresh database (no `documents` table
134
+ * yet: correct, there is no history to protect against), or `(SELECT COALESCE(MAX(ts), 0) FROM
135
+ * documents)` on an upgrade (`documents` already exists: correct, the row is born at the true
136
+ * high-water mark, never momentarily at 0). A pre-seeded row is created UNACQUIRED — `epoch = 0`,
137
+ * `writer_url = NULL`, already-expired `expires_at` — so the FIRST real `tryAcquire` still lands on
138
+ * its `ON CONFLICT` branch and bumps `epoch` from 0 to 1, byte-identical to a fresh INSERT's
139
+ * `epoch = 1`; every existing epoch/ownership assertion in the test suite is unaffected. Idempotent
140
+ * (`ON CONFLICT (shard_id) DO NOTHING`): a row already created by a peer's concurrent `setup()` or
141
+ * by a real acquisition is left untouched — this can only ever RAISE a row into existence, never
142
+ * clobber one. With every shard row present immediately, the tailer's `count(*) < NUM_SHARDS →
143
+ * not-ready` gate is satisfied the instant `setup()` returns — a concurrent multi-writer boot no
144
+ * longer stalls waiting for a writer to finish its acquire-all pass. See `node.ts`'s
145
+ * `prepareFleetNode` for the `documentsTableExists` probe + call site, and `replica-tailer.ts`'s
146
+ * `readFrontier` for the count gate.
147
+ */
148
+ /** Run one DDL statement, swallowing the concurrent-boot duplicate-object race family
149
+ * (23505 unique_violation / 42P07 duplicate_table / 42710 duplicate_object — the third face
150
+ * is "type <table> already exists", the table's composite rowtype, which is what two nodes
151
+ * racing CREATE TABLE IF NOT EXISTS actually surface). The statement's objective — the
152
+ * object exists — is achieved when any of these fires; anything else propagates. Mirrors
153
+ * docstore-postgres setupSchema's discipline; added when the SIMULTANEOUS multi-writer boot
154
+ * E2E caught the unguarded race here (Plan A's extra boot DDL widened the window). */
155
+ private ddl;
156
+ setup(shardIds?: readonly ShardId[], documentsExist?: boolean): Promise<void>;
157
+ /**
158
+ * The `frontier_ts` seed SQL fragment shared by `setup()`'s row pre-seed and `tryAcquire()`'s
159
+ * first-creation INSERT (Fleet B3, D4 — the one place this reasoning is written down): `0` when the
160
+ * caller has verified `documents` does not exist yet (a fresh database — no history to protect
161
+ * against), else `(SELECT COALESCE(MAX(ts), 0) FROM documents)` (an upgrade — the row is born at the
162
+ * store's true high-water mark, never momentarily visible below it). Both call sites gate this on
163
+ * the SAME `documentsTableExists` probe (`node.ts`), so a row is never seeded incorrectly regardless
164
+ * of which of the two code paths creates it first.
165
+ */
166
+ private frontierSeedExpr;
167
+ /**
168
+ * Upsert this node's `fleet_nodes` presence row (B2b, D3), extending `expires_at` by the lease TTL.
169
+ * Called at boot (BEFORE the writer-election `tryAcquire`, so a losing node is already visible when
170
+ * it boots sync) and on every balancer beat / writer probe — the node's liveness signal now. A node
171
+ * whose process dies stops heartbeating; its row expires after the TTL and it drops out of every
172
+ * survivor's `liveNodes()`, re-converging the rendezvous assignment. `epoch` increments per beat —
173
+ * carried only for observability/debugging (which incarnation of a node the row belongs to); the
174
+ * live-set membership check keys off `expires_at`, not `epoch`.
175
+ */
176
+ heartbeatPresence(): Promise<void>;
177
+ /**
178
+ * The live fleet node set (B2b, D3): distinct unexpired `fleet_nodes` advertise URLs, UNION the live
179
+ * `shard_leases` writer URLs (belt-and-braces — a writer whose presence heartbeat momentarily lapsed
180
+ * but whose shard lease is still live is unambiguously alive). Every node computes rendezvous over
181
+ * this set, so they all derive the same shard→owner assignment. Expiry is `expires_at >= now()` per
182
+ * the DB's OWN clock, authoritative against host clock skew.
183
+ */
184
+ liveNodes(): Promise<string[]>;
185
+ /**
186
+ * Per-shard ownership snapshot for the balancer (B2b, D3): for each EXISTING `shard_leases` row, its
187
+ * `writer_url` (null = orphaned) and whether it has expired per the DB clock. A shard with no row is
188
+ * simply absent from the map — the balancer treats an absent shard as acquirable (never created, or
189
+ * fully reaped), the same as an orphaned one. Read-only; the balancer decides acquire/release from it.
190
+ */
191
+ readShardOwnership(): Promise<Map<ShardId, {
192
+ writerUrl: string | null;
193
+ expired: boolean;
194
+ }>>;
195
+ /**
196
+ * Self-fence a shard this node currently holds, for a GRACEFUL balancer release (B2b, D3): bump the
197
+ * epoch (so this node's own subsequent commit/heartbeat on the now-stale epoch fences cleanly),
198
+ * clear `writer_url` (so the shard reads as orphaned and its rightful rendezvous owner acquires it),
199
+ * and GREATEST-bump `frontier_ts` from the shared sequence (so F never regresses across the handoff).
200
+ * Predicated on this node's currently-held epoch — a silent no-op if the shard is already fenced/
201
+ * relinquished (`currentEpoch === null`). The caller runs this only when the shard is fully IDLE —
202
+ * `runtime.tryRunExclusiveOnShard` grants the mutex only if no commit holds it AND no group-commit
203
+ * batch is staged/flushing (Fleet B4), so no in-flight commit's OR batch's frontier write races the
204
+ * GREATEST-bump here; the release is deferred a beat if the shard is busy rather than fencing
205
+ * mid-flush (see `releaseShard` in `node.ts`). A mutation that begins mid-execute after the fence
206
+ * simply hits `FencedError` at its own commit (OCC-retryable, the forwarder re-routes the retry to
207
+ * the new owner). Mirrors `evictExpired`'s frontier-bump SQL, but unconditional on expiry (this is a
208
+ * voluntary handoff, not an eviction of a wedged peer).
209
+ */
210
+ selfFence(shardId: ShardId): Promise<void>;
211
+ /**
212
+ * The epoch this node most recently acquired for `shardId` (via `tryAcquire`), or `null` if it
213
+ * never has. Read live — not a boot-time snapshot — by the commit guard and the heartbeat probe,
214
+ * so a re-promotion's epoch bump (another `tryAcquire` call) is picked up automatically with no
215
+ * extra threading between `prepareFleetNode`/`startFleetNode`/`promoteFleetNode`. Defaults to the
216
+ * default shard so B1 call sites (`currentEpoch()`) are unchanged.
217
+ */
218
+ currentEpoch(shardId?: ShardId): bigint | null;
219
+ /** The (shard, epoch) pairs this node currently holds — the input to the batched heartbeat,
220
+ * all-rows seed, and idle-shard closer (all fence per-row against the epoch on file). A snapshot
221
+ * of the live per-shard epoch map, so it reflects any re-promotion's epoch bumps. */
222
+ heldPairs(): Array<{
223
+ shardId: ShardId;
224
+ epoch: bigint;
225
+ }>;
226
+ /**
227
+ * Drop `shardId` from this node's held-epoch map (Fenced Frontier B2b, D2 — per-shard relinquish).
228
+ * After this call `currentEpoch(shardId)` returns `null`, so:
229
+ * - the commit guard's "no acquired epoch" branch fences any straggler commit on `shardId`
230
+ * cleanly (rather than a stale epoch happening to still match a row some other node has since
231
+ * re-fenced and moved on from);
232
+ * - `heldPairs()` (and therefore `heartbeatAll()`/`closeIdleFrontiers()`/`seedFrontierAll()`) stop
233
+ * including this shard.
234
+ * Idempotent: forgetting a shard this node doesn't currently hold is a silent no-op — the
235
+ * relinquish dispatcher (`node.ts`) relies on exactly this for ITS OWN idempotency, via
236
+ * `currentEpoch(shardId) === null` as its "already relinquished" check. Deliberately does NOT touch
237
+ * the `shard_leases` ROW — only the caller's advisory-lock release (`PgClient.releaseShardLock`) and
238
+ * this in-memory forget are relinquish's job; the row itself was already epoch-bumped by whoever
239
+ * fenced this node (or will lapse via TTL), and a future re-acquire (`tryAcquire`) is a fresh INSERT/
240
+ * ON CONFLICT UPDATE regardless of what this map remembers.
241
+ */
242
+ forgetShard(shardId: ShardId): void;
243
+ /**
244
+ * One non-blocking attempt: takes the advisory lock (fast path); on success, runs the fencing
245
+ * upsert against `shard_leases` (bumping `epoch`, recording this node's URL/app-name, extending
246
+ * `expires_at`) and returns the new state. On failure to take the lock, returns null.
247
+ * `prev_ts` is seeded to 0 on first creation only, and `frontier_ts` to either 0 or the store's
248
+ * current `MAX(ts)` (see `seedFrontierFromDocuments`) — an `ON CONFLICT` re-acquisition (including
249
+ * promotion) leaves BOTH untouched, so the durable-commit chain survives across epochs (D3 depends
250
+ * on this: frontier must never reset just because the writer changed).
251
+ *
252
+ * `seedFrontierFromDocuments` (the F1×N residual-window fix): when a row is FIRST created, seed its
253
+ * `frontier_ts` to `MAX(ts)` from the `documents` log inside the same INSERT — atomically, so the
254
+ * row is NEVER momentarily visible at `frontier_ts = 0` on a pre-loaded store (which would let a
255
+ * concurrently-booting sync node pass its `count == N ∧ min-F` ready gate with an empty replica).
256
+ * Pass `true` only when the `documents` table is known to exist (post-`setupSchema`, or a pre-loaded
257
+ * store) — the caller guards this; `false` (the default, and the only safe value pre-DDL on a fresh
258
+ * database) creates the row at `frontier_ts = 0`, which is correct precisely because a fresh store
259
+ * holds no data. `seedFrontierAll` remains the idempotent belt-and-braces second pass.
260
+ */
261
+ tryAcquire(shardId?: ShardId, slot?: number, seedFrontierFromDocuments?: boolean): Promise<LeaseState | null>;
262
+ /**
263
+ * Extend `expires_at` for this node's `epoch` — the LeaseMonitor's periodic probe (D2: one
264
+ * round-trip serves liveness-probe + TTL maintenance + fence verification). Returns the number
265
+ * of rows updated: 1 = still this node's epoch (TTL extended), 0 = fenced — some other node has
266
+ * bumped the epoch (a D4 eviction) and this node no longer holds the lease, even though its
267
+ * connection never dropped. Callers (see `node.ts`) treat 0 as definitive lease loss.
268
+ */
269
+ heartbeat(epoch: bigint, shardId?: ShardId): Promise<number>;
270
+ /**
271
+ * Batched heartbeat over EVERY (shard, epoch) pair this node holds — one UPDATE per beat (B2a):
272
+ * the writer holds N leases and must renew all of them in a single round-trip, not N. Returns
273
+ * `{ updated, expected, fencedShardIds }` where `expected` is how many pairs this node believes it
274
+ * holds, `updated` is how many rows actually matched, and `fencedShardIds` (B2b, D2) is PRECISELY
275
+ * which held shards did NOT match — diffing the `RETURNING shard_id` rows against the held set —
276
+ * so the caller can relinquish exactly those shards rather than treat `updated < expected` as an
277
+ * undifferentiated whole-node signal. `updated < expected` (equivalently `fencedShardIds.length >
278
+ * 0`) means at least one shard's epoch was superseded; `fencedShardIds` is always `[]` when this
279
+ * node holds nothing (never a writer) — see the early return. The `(shard_id, epoch) IN ((..),(..))`
280
+ * tuple form fences per row.
281
+ */
282
+ heartbeatAll(): Promise<{
283
+ updated: number;
284
+ expected: number;
285
+ fencedShardIds: ShardId[];
286
+ }>;
287
+ /**
288
+ * Read-only expiry check: does the lease row exist AND is `expires_at` in the past, per the DB's
289
+ * OWN clock? Used as the acquire loop's per-tick pre-gate before the heavier `evictExpired`
290
+ * transaction — a healthy (live) lease must NOT make every follower take the row lock every tick and
291
+ * contend with the writer's commit guard, so the common case stays a single cheap SELECT. The
292
+ * comparison is `now()` in SQL (not `read().expiresAt` in JS) so it's authoritative against clock
293
+ * skew between a follower's host and Postgres.
294
+ */
295
+ isExpired(shardId?: ShardId): Promise<boolean>;
296
+ /**
297
+ * Fencing-first eviction of an EXPIRED lease (Fenced Frontier B1, D4). Bumps `epoch` (fencing the
298
+ * wedged holder — its next commit guard / heartbeat now matches 0 rows), clears `writer_url`/
299
+ * `writer_app_name`, and advances `frontier_ts`, all predicated on the lease actually being
300
+ * expired. Returns `{ fenced: true, oldAppName }` capturing the evicted holder's `writer_app_name`
301
+ * (so the acquire loop can `pg_terminate_backend` its lingering connection); `{ fenced: false }`
302
+ * when the row is live (no-op) OR the row lock couldn't be taken within `lock_timeout` (retry next
303
+ * tick — never throws that to the loop).
304
+ *
305
+ * Why a `SELECT ... FOR UPDATE` then a separate `UPDATE`, NOT a single `UPDATE ... RETURNING (SELECT
306
+ * writer_app_name FROM cte)`: on Postgres, RETURNING — and the inlined CTE subquery it evaluates —
307
+ * sees the NEW (already-nulled) row, so a single statement reads back NULL for the old app name
308
+ * (verified on PGlite; a `MATERIALIZED` CTE with its own `FOR UPDATE` instead collides with the
309
+ * same-statement UPDATE's row lock and yields zero rows). The two-statement form captures the old
310
+ * value cleanly, and the `FOR UPDATE` takes the very row lock a concurrent commit-guard UPDATE
311
+ * contends on — serializing eviction against an in-flight commit. That contention is single-
312
+ * connection-untestable and covered E2E only (see the test header).
313
+ */
314
+ evictExpired(shardId?: ShardId): Promise<{
315
+ fenced: boolean;
316
+ oldAppName: string | null;
317
+ }>;
318
+ /**
319
+ * Kill the evicted holder's lingering Postgres backend(s) by `application_name` so its session-level
320
+ * advisory writer lock is released and the NEXT acquire tick can win. Matches the fleet E2E's query
321
+ * shape exactly, including the `pid <> pg_backend_pid()` self-exclusion (the fencer never terminates
322
+ * its own connection — its app name differs anyway, so this is belt-and-braces). Best-effort: if the
323
+ * backend is already gone, the query simply affects zero rows.
324
+ */
325
+ terminateBackend(appName: string): Promise<void>;
326
+ /**
327
+ * A tick observed the advisory-lock try FAIL — another backend still holds the writer lock. If the
328
+ * lease has ALSO expired, that holder is wedged (its heartbeat stopped) while its connection lingers
329
+ * (a hung/paused process still owns the advisory lock, so no other node can acquire). Fence it
330
+ * (`evictExpired` bumps the epoch) then terminate its backend to release the lock; the next tick's
331
+ * advisory try then succeeds → normal acquisition → promotion. No-op when the lease is still live.
332
+ */
333
+ private maybeEvictWedged;
334
+ /** Loop tryAcquire() every retryMs until it succeeds, then invoke onAcquired once. stop() cancels.
335
+ * On a tick where the advisory try fails AND the lease has expired, the wedged holder is fenced +
336
+ * its backend terminated (`maybeEvictWedged`) so a later tick can take over. */
337
+ acquireLoop(onAcquired: (s: LeaseState) => void): void;
338
+ /** Cancels any pending acquireLoop() retry. */
339
+ stop(): void;
340
+ /**
341
+ * F1 fix (Fenced Frontier B1 whole-branch review, BLOCKER): seed `frontier_ts` up to at least
342
+ * `maxTs` — the writer-BOOT step that closes the "pre-loaded database" hole. `tryAcquire()` only
343
+ * seeds `frontier_ts` to 0 on the row's FIRST creation (see above); a single-node Postgres `serve`
344
+ * that accumulates real data BEFORE `--fleet` is ever turned on has no `shard_leases` row at all,
345
+ * so the first `tryAcquire()` after enabling `--fleet` creates one at `frontier_ts=0` even though
346
+ * the store already holds history. Because the tailer targets `F=frontier_ts` (not
347
+ * `primary.maxTimestamp()`, see `replica-tailer.ts`'s D5), a fresh sync node's ready gate
348
+ * (`wm=0 < target=0`) is then a silent no-op — it reports ready with an EMPTY replica while the
349
+ * primary holds everything. Callers must invoke this at WRITER BOOT, after this node's own
350
+ * `setupSchema()` has definitely run (the `documents` table this reads may not exist before
351
+ * that — see `prepareFleetNode`/`startFleetNode`'s boot ordering) and BEFORE the node is reported
352
+ * ready; a later promotion does NOT need to call this — by then `frontier_ts` is already live,
353
+ * tracking every commit via the guard installed by `installCommitGuard`.
354
+ *
355
+ * `GREATEST` makes this a no-op on an already-live fleet (frontier already tracks every commit)
356
+ * and on re-acquisition — never regresses `frontier_ts`. Epoch-fenced like every other
357
+ * `shard_leases` write: a stale `epoch` here (this node lost the writer race to someone else
358
+ * between its `tryAcquire()` and this call) affects 0 rows rather than clobbering the new
359
+ * writer's state — that writer's own commit guard is the source of truth for `frontier_ts` going
360
+ * forward regardless. Deliberately does NOT touch `prev_ts`: this is a seed of the high-water
361
+ * mark, not a commit, so there is no new "previous commit" to record — the next REAL commit's
362
+ * guard sets `prev_ts := frontier_ts` (now the seeded value) exactly as it always does.
363
+ */
364
+ seedFrontier(epoch: bigint, maxTs: bigint, shardId?: ShardId): Promise<void>;
365
+ /**
366
+ * All-rows frontier seed (B2a — the F1×N fix): seed EVERY shard this node holds up to `maxTs` in
367
+ * ONE batched, per-row-epoch-fenced UPDATE, run at writer boot BEFORE the node reports ready. The
368
+ * same reasoning as single-shard `seedFrontier` applies to every row at once: any future commit on
369
+ * any shard takes a later `nextval`, so seeding all held frontiers to the store's current max can
370
+ * never over-shoot a real commit, and `GREATEST` keeps it a no-op on an already-live fleet.
371
+ * Epoch-fenced per pair (`(shard_id, epoch) IN (...)`), so a shard this node lost between acquiring
372
+ * it and this call is simply skipped rather than clobbered. See `node.ts`'s writer boot.
373
+ */
374
+ seedFrontierAll(maxTs: bigint): Promise<void>;
375
+ /**
376
+ * Per-shard idle-shard frontier closing (B2a, D5 — needed the moment N>1: an idle shard pins the
377
+ * fleet's `F = min(frontier_ts)`). Allocate a single fresh `nextval` `N` from the shared
378
+ * `helipod_ts` sequence per beat, then for EACH held shard advance its frontier up to `N` —
379
+ * `UPDATE ... SET frontier_ts = GREATEST(frontier_ts, $N) WHERE shard_id=$s AND epoch=$e AND
380
+ * frontier_ts < $N` — but run each shard's UPDATE UNDER THAT SHARD'S COMMIT MUTEX
381
+ * (`runExclusiveOnShard`), skipping any shard currently mid-commit (mutex busy → left for the next
382
+ * beat). Returns the `nextval` used (the ceiling this beat closed idle shards up to).
383
+ *
384
+ * Why skip-if-busy, not the old bare batched UPDATE (the frontier-inversion fix): the commit
385
+ * guard writes `frontier_ts := commitTs` for a commit that drew its ts `T` from the SAME sequence,
386
+ * *inside* the commit transaction — which runs under the shard's commit mutex (single-commit) or at
387
+ * FLUSH time for a group-commit batch (Fleet B4). If the closer drew `N > T` and bare-wrote
388
+ * `frontier_ts = N` while that commit's/batch's rows had not yet landed, a tailer could read `F ≥
389
+ * N`, pull `(watermark, N]`, and MISS `T`'s rows when they land afterward (silent replica miss), or
390
+ * the guard's later write of `T` would trip a frontier-regression assert. `runExclusiveOnShard`
391
+ * treats a shard as available only when it is fully IDLE — mutex free AND no staged/flushing
392
+ * group-commit batch (`ShardWriter.hasInFlightWork()`): group commit runs the flush OFF the mutex,
393
+ * so a mid-flush batch (ts's already drawn, rows not yet landed) reads BUSY even though the mutex is
394
+ * free, and is skipped for the next beat. Given that, the closer and that shard's commits are
395
+ * mutually exclusive on this (single, writer-owned) node: any commit/batch STARTED before the draw
396
+ * either holds the mutex or is staged/flushing → skipped this beat; any commit/batch that STARTS
397
+ * after the closer's `nextval` draws its `T` at commit/flush time, AFTER `N`, so `T > N` and
398
+ * `GREATEST` keeps `frontier_ts ≥ N` with no regression and nothing in-flight ever sits below `N`.
399
+ * Cross-node closing of a held shard is impossible by design (only
400
+ * the lease holder closes its own held shards; orphan bumping via `evictExpired` targets
401
+ * writer-less rows, where no commit can be in flight). Epoch-fenced per pair. No-op (returns the
402
+ * allocated ts anyway) when this node holds nothing.
403
+ */
404
+ closeIdleFrontiers(runExclusiveOnShard: TryRunExclusiveOnShard): Promise<bigint>;
405
+ /**
406
+ * Orphan frontier bumping (B2b, D4): advance the frontier of every WRITER-LESS shard row
407
+ * (`writer_url IS NULL`) that lags `ceiling`, so an unassigned/relinquished/expired shard never
408
+ * pins the fleet's `F = min(frontier_ts)` below the live commit position. Runs on the WRITER beat
409
+ * alongside `closeIdleFrontiers` (which only ever touches THIS node's OWN held rows via
410
+ * `heldPairs()` and so structurally cannot un-pin a shard nobody holds).
411
+ *
412
+ * Safe with NO commit mutex, unlike `closeIdleFrontiers`: a writer-less shard has no in-flight
413
+ * commit by construction — its last writer was fenced (epoch-bumped, `writer_url` nulled) BEFORE
414
+ * the row became orphaned, so nothing can be mid-commit writing `frontier_ts` on it. The `UPDATE`'s
415
+ * own row lock serializes any two writers racing this bump, and `GREATEST` + `frontier_ts < ceiling`
416
+ * keep it monotone regardless. Reuses the SAME `ceiling` (one `nextval` per beat) the idle closer
417
+ * drew, so a beat costs one sequence draw, not two.
418
+ */
419
+ bumpOrphanFrontiers(ceiling: bigint): Promise<void>;
420
+ /** Reads the current lease row for `shardId` (discovery for forwarding, plus the full fencing/
421
+ * frontier state); null if none exists yet. Defaults to the default shard (B1 call sites). */
422
+ read(shardId?: ShardId): Promise<LeaseRow | null>;
423
+ /** All shard rows' `(shard_id, frontier_ts)` — the fleet-wide frontier picture the writer's
424
+ * frontier-lag monitor reads to compute `min(frontier_ts)` + which shard is pinning it (D5's
425
+ * health observability). Ordered by frontier ascending so `rows[0]` is the pinning shard. */
426
+ readAllFrontiers(): Promise<Array<{
427
+ shardId: ShardId;
428
+ frontierTs: bigint;
429
+ }>>;
430
+ /**
431
+ * Read back `key`'s `fleet_idempotency` row for a replay decision (Fleet B3, D3), or `null` if no
432
+ * such key has ever committed (a genuine miss — proceed to run). Called by `packages/cli`'s
433
+ * `/_fleet/run` handler BOTH before running (the SELECT-first check) and after catching a
434
+ * unique_violation on this table (the concurrent-duplicate race's loser re-selecting the
435
+ * winner's row). See `IdempotencyReplay` for what `hasValue`/`oversized` distinguish.
436
+ */
437
+ lookupIdempotency(key: string): Promise<IdempotencyReplay | null>;
438
+ /**
439
+ * Best-effort post-run recording of a forwarded mutation's return VALUE onto its already-committed
440
+ * `fleet_idempotency` row (Fleet B3, D3) — the value isn't known inside the commit transaction (the
441
+ * guard only sees `commitTs`), so this runs AFTER `runtime.run`/`runAction`/`runSystem` returns,
442
+ * from `packages/cli`'s `/_fleet/run` handler. Over `IDEMPOTENCY_VALUE_CAP_BYTES` → `value_json`
443
+ * stays/goes NULL and `oversized = true` instead of storing it (a replay then reports
444
+ * `valueMissing: true`, same as the crash-window case where this UPDATE never ran at all). A
445
+ * missing row (no matching `key` — e.g. a no-op mutation that staged nothing, so the guard never
446
+ * ran) silently affects 0 rows; the caller doesn't need to check for that itself.
447
+ */
448
+ recordIdempotencyValue(key: string, value: JSONValue): Promise<void>;
449
+ /**
450
+ * Reclaim `fleet_idempotency` rows older than the TTL (Fleet B3, D3) — a cheap indexed (PK-only
451
+ * table, no index needed at this scale) delete run on the balancer beat of every WRITER-ish node
452
+ * (see `ShardLeaseBalancerDeps.sweepIdempotency` / `node.ts`'s wiring). A retry arriving after a
453
+ * row has been swept simply re-executes (documented boundary — retries are seconds-scale, the
454
+ * sweep window is 1h).
455
+ */
456
+ sweepIdempotency(): Promise<void>;
457
+ /**
458
+ * Build a `(shard_id, epoch) IN ((...),(...))` VALUES-list clause + its flat params array for a
459
+ * batched per-row-fenced statement, with placeholder numbers starting AFTER `offset` positional
460
+ * params the caller already consumed (e.g. a leading `$1 = maxTs`). Returns `{ clause, params }`
461
+ * where `params` is `[shardA, epochA, shardB, epochB, …]` in placeholder order.
462
+ */
463
+ private tupleInClause;
464
+ }
465
+
466
+ /**
467
+ * Fleet writer liveness monitor (Task 4, C4). Runs ONLY while a node is the writer. Its job: notice
468
+ * when this node has lost its exclusive writer lease and exit the process so it can rejoin the fleet
469
+ * as a fresh sync node (exit-and-rejoin) — rather than lingering as a zombie writer that believes it
470
+ * still owns the lock while some other node has taken over.
471
+ *
472
+ * Three independent loss signals feed it:
473
+ *
474
+ * - `connectionLost()` — DEFINITIVE. The advisory lock lives on `NodePgClient`'s single pinned
475
+ * connection, which never reconnects (see `NodePgClient.ensure()`); if that connection closes or
476
+ * errors, the session-level `pg_advisory_lock` is released by Postgres the instant the backend
477
+ * goes away. So a closed pinned connection == lease definitely lost → exit immediately.
478
+ *
479
+ * - `fenced()` — DEFINITIVE (Fenced Frontier B1, D2/D3). The connection is still alive, but this
480
+ * node's epoch has been superseded — either the heartbeat probe found 0 rows for `(shard_id,
481
+ * epoch)`, or an epoch-fenced commit guard's UPDATE matched 0 rows. Both mean some OTHER node
482
+ * now holds the lease, so — exactly like `connectionLost()` — this bypasses the probe-miss
483
+ * tolerance below (which exists only for transient/ambiguous blips; a fenced epoch never is)
484
+ * and exits immediately.
485
+ *
486
+ * - `probe()` misses — a BACKSTOP for a silently-wedged connection that isn't emitting error/end
487
+ * events (e.g. a half-open TCP connection). The probe is a caller-supplied liveness round-trip
488
+ * (`() => client.query("SELECT 1")`); it must NEVER be `pg_try_advisory_lock` — that is re-entrant
489
+ * on the session already holding the lock and would leak a lock count on every probe. We tolerate
490
+ * `maxMisses` consecutive failures (transient blips) and exit on the NEXT one; any success resets
491
+ * the counter. Each probe is itself bounded by `probeTimeoutMs` (default `probeMs`) via
492
+ * `Promise.race` — a silently-wedged connection (e.g. a half-open TCP socket) can leave
493
+ * `SELECT 1` hanging forever with no error/end event; without a per-probe timeout `inFlight`
494
+ * would stay true and every subsequent tick would early-return, so misses would never accrue and
495
+ * this exact backstop would never fire. A timed-out probe counts as a miss; if the underlying
496
+ * query later settles anyway, that late settlement is discarded (`Promise.race` only ever acts on
497
+ * whichever of the probe/timeout settles first) — no reset, no double-count. `inFlight` still
498
+ * clears once the race settles, so the next tick starts its own fresh attempt regardless of what
499
+ * the abandoned probe call eventually does.
500
+ *
501
+ * `onExit` is injected: production passes `(reason) => { console.error(...); process.exit(1); }`;
502
+ * tests pass a spy. It fires AT MOST ONCE — the first of (connectionLost, probe-exhaustion) wins,
503
+ * and `stop()` (graceful shutdown / promotion hand-off) halts everything so nothing fires after.
504
+ */
505
+ interface LeaseMonitorDeps {
506
+ /** Liveness round-trip against the pinned writer connection. MUST be a plain query (e.g.
507
+ * `SELECT 1`), NEVER `pg_try_advisory_lock` (re-entrant → leaks a lock count per probe). */
508
+ probe: () => Promise<void>;
509
+ /** Fired at most once when the lease is deemed lost. Production: `console.error` + `process.exit(1)`. */
510
+ onExit: (reason: string) => void;
511
+ /** Interval between probes. Default 5000ms. */
512
+ probeMs?: number;
513
+ /** Consecutive probe failures tolerated before exit — the NEXT (maxMisses+1-th) miss exits. Default 3. */
514
+ maxMisses?: number;
515
+ /** Per-probe timeout — a probe that hasn't settled by this point counts as a miss (the wedged
516
+ * connection is abandoned, not awaited further). Default `probeMs`. */
517
+ probeTimeoutMs?: number;
518
+ }
519
+ declare class LeaseMonitor {
520
+ private readonly probe;
521
+ private readonly onExit;
522
+ private readonly probeMs;
523
+ private readonly maxMisses;
524
+ private readonly probeTimeoutMs;
525
+ private timer;
526
+ private misses;
527
+ private inFlight;
528
+ private exited;
529
+ private stopped;
530
+ constructor(deps: LeaseMonitorDeps);
531
+ /** Begin probing every `probeMs`. Idempotent; a no-op once stopped or after exit. */
532
+ start(): void;
533
+ private tick;
534
+ private recordMiss;
535
+ /** DEFINITIVE lease loss (pinned connection closed/errored) → exit immediately. */
536
+ connectionLost(): void;
537
+ /** DEFINITIVE lease loss (epoch superseded — heartbeat or commit guard found 0 rows for this
538
+ * node's epoch) → exit immediately, mirroring `connectionLost()`. Idempotent/at-most-once via
539
+ * the same `fireExit` guard — safe to call from both the heartbeat probe and a commit guard. */
540
+ fenced(reason?: string): void;
541
+ private fireExit;
542
+ /** Halt all probing and disarm exit — graceful shutdown or promotion hand-off. */
543
+ stop(): void;
544
+ private clearTimer;
545
+ }
546
+
547
+ /**
548
+ * Thrown when this node's epoch has been superseded by another writer (Fenced Frontier B1, D2/D3).
549
+ * Two sources:
550
+ *
551
+ * - `LeaseManager.heartbeat(epoch)` finds zero rows matching `(shard_id, epoch)` — some other
552
+ * node has fenced this one (bumped the epoch, e.g. the D4 wedged-writer eviction path) and
553
+ * this node's lease is gone even though it never lost its Postgres connection.
554
+ * - `PostgresDocStore`'s installed commit guard's epoch-predicated `UPDATE` matches zero rows —
555
+ * a straggler commit that raced a fencer and lost; the whole transaction aborts.
556
+ *
557
+ * Both are DEFINITIVE lease loss — exactly like a dropped connection — and are routed to
558
+ * `LeaseMonitor.fenced()` for immediate exit (bypassing the probe-miss tolerance, which exists
559
+ * only for transient/ambiguous blips; a fenced epoch is never ambiguous).
560
+ */
561
+ declare class FencedError extends Error {
562
+ constructor(message: string);
563
+ }
564
+
565
+ /**
566
+ * Cross-process commit propagation for a Postgres-backed fleet (scale-seam #4, multi-process
567
+ * variant): the in-memory `EmbeddedWriteFanoutAdapter` only fans a commit out within ONE process.
568
+ * `NotifyingFanoutAdapter` (writer side) wraps it so every publish ALSO does
569
+ * `pg_notify('helipod_commits', …)`; each follower's `ReplicaTailer` (`replica-tailer.ts`)
570
+ * LISTENs for that channel (plus a wall-clock poll fallback, since NOTIFY delivery is not
571
+ * guaranteed while a listener connection is reconnecting) and, on each wake, verbatim-applies the
572
+ * primary's MVCC log onto its local replica and derives what to invalidate from that same batch —
573
+ * no in-process `OplogDelta` ever crosses a process boundary.
574
+ *
575
+ * The `CommitChannelClient` seam below (the LISTEN/NOTIFY + parameterized-query slice of
576
+ * `NodePgClient`) is shared by both the writer-side adapter here and the follower-side
577
+ * `ReplicaTailer`. The slice-1 `CommitTailer` (which derived-only, never applied to a replica)
578
+ * was removed in slice 2 once `ReplicaTailer` subsumed it.
579
+ */
580
+
581
+ /**
582
+ * The narrow slice of `NodePgClient` this module (and `ReplicaTailer`) depends on (LISTEN/NOTIFY
583
+ * plus a plain parameterized query) — kept as a structural interface, matching the `PgClient` seam's
584
+ * own philosophy of never tying engine/fleet logic to a concrete driver class. A `NodePgClient`
585
+ * instance satisfies this; so does any test double that implements the same shape.
586
+ */
587
+ interface CommitChannelClient extends PgQuerier {
588
+ /** LISTEN on `channel`; returns a closer. Rejecting (e.g. no LISTEN support) is tolerated by
589
+ * `ReplicaTailer.start()`, which falls back to poll-only. */
590
+ listen(channel: string, onNotify: (payload: string) => void): Promise<() => Promise<void>>;
591
+ notify(channel: string, payload: string): Promise<void>;
592
+ }
593
+ /** Writer side: wraps the in-memory adapter; every publish ALSO does
594
+ * `pg_notify('helipod_commits', String(commitTs))` so followers wake promptly instead of
595
+ * waiting out the poll interval. NOTIFY is a latency optimization only — the poll fallback in
596
+ * `CommitTailer` is the correctness path if it's ever dropped or a listener misses it. */
597
+ declare class NotifyingFanoutAdapter implements EmbeddedWriteFanoutAdapter {
598
+ private readonly inner;
599
+ private readonly client;
600
+ constructor(inner: EmbeddedWriteFanoutAdapter, client: CommitChannelClient);
601
+ publish(payload: EmbeddedWriteFanoutPayload): void;
602
+ subscribe(listener: FanoutListener): () => void;
603
+ }
604
+
605
+ /**
606
+ * `ReplicaTailer` (Fleet slice 2, Task 2; Fenced Frontier B1, D5/D6) — tails the Postgres
607
+ * primary's MVCC log VERBATIM onto a local embedded replica `DocStore` (an in-process SQLite
608
+ * instance, in the intended deployment), deriving invalidation ranges straight from the SAME
609
+ * batch it just applied — not a separate query — so a follower's reactive fan-out is provably
610
+ * consistent with what actually landed on the replica.
611
+ *
612
+ * Evolves `CommitTailer` (`commit-notifier.ts`, slice 1)'s LISTEN + poll wake posture, the
613
+ * `draining` re-entrancy guard, and watermark-advances-only-after-`onInvalidation`-resolves
614
+ * semantics — but diverges in structural ways `CommitTailer` never needed:
615
+ *
616
+ * 1. VERBATIM APPLY — `CommitTailer` only ever derived ranges to wake a LIVE in-process runtime;
617
+ * it never touched a second store. This class additionally re-materializes the primary's
618
+ * actual `DocumentLogEntry`/`IndexWrite` rows onto a real replica `DocStore` via `write(...,
619
+ * "Overwrite")`, so the replica is a byte-for-byte MVCC mirror (historical reads included),
620
+ * not just a wake signal.
621
+ * 2. BOOTSTRAP CATCH-UP — a fresh replica starts at watermark 0 (or wherever it last left off),
622
+ * which can be arbitrarily far behind the primary. `start()` doesn't resolve until the
623
+ * replica has caught up to the FENCED FRONTIER `F` AT CALL TIME (the ready gate), batching
624
+ * the catch-up in `batchSize`-sized ticks instead of one unbounded pull.
625
+ * 3. FENCED-FRONTIER TARGETING (D5) — the pull target is no longer `primary.maxTimestamp()` (the
626
+ * log's live high-water mark, which can include a commit that raced past the last fence
627
+ * check under contention) but `F = shard_leases.frontier_ts` — the durably-fenced, dense
628
+ * prefix the epoch-fenced commit guard (`node.ts`'s `installCommitGuard`) advances inside
629
+ * every commit transaction. At one shard F advances exactly with commits, so this is
630
+ * behavior-identical to the old target in the steady state; the difference only matters
631
+ * under a wedged-writer fencing race (B1 D3/D4), where `maxTimestamp()` could momentarily
632
+ * run ahead of what's actually safe to consider durable. See `stable-prefix.ts` for the
633
+ * `StablePrefixTs` brand this introduces.
634
+ * 4. DENSITY ASSERTIONS (D5) — defense-in-depth: the construction (D1 store-allocated ts + D3
635
+ * epoch-fenced commits) is what actually PREVENTS a skipped commit from ever landing on a
636
+ * replica, but the apply loop additionally verifies it, per document entry, before applying:
637
+ * the replica's current head for that doc must chain from `prev_ts` exactly (or be absent,
638
+ * for an insert). A violation throws `DensityViolationError` — crash loudly rather than serve
639
+ * silently corrupted state; the operator remedy is the already-shipped delete-and-re-bootstrap
640
+ * (a replica file is a rebuildable mirror).
641
+ *
642
+ * `CommitTailer` was the slice-1 derive-only precursor; slice 2 (Task 4) deleted it once this class
643
+ * subsumed its wake/derive posture with verbatim replica apply. `AppliedInvalidation` below carries
644
+ * the invalidation shape (identical members to what `CommitTailer.DerivedInvalidation` had).
645
+ *
646
+ * Per-tick pipeline (see the class body for the full step-by-step):
647
+ * 1. `F = await readFrontier()` (D5: `shard_leases.frontier_ts`, branded `StablePrefixTs`); no-op
648
+ * if `<= watermark`. F is asserted non-decreasing across reads (D5's other defense-in-depth
649
+ * invariant) — a regression means `shard_leases` itself was corrupted/hand-tampered.
650
+ * 2. Pull `DocumentLogEntry` rows for `(watermark, F]` via `primary.load_documents`, capped at
651
+ * `batchSize` — but never splitting a single commit's ts group across ticks (a transaction
652
+ * shares exactly one commit `ts` across all its writes, see `postgres-docstore.ts`'s
653
+ * `write()` doc comment; if the batch fills mid-way through a ts group, the remaining
654
+ * same-ts rows are drained too before capping, so a partial transaction's writes are never
655
+ * applied on the replica).
656
+ * 3. Pull the matching `indexes` rows for the SAME `(watermark, cappedMax]` via raw SQL and
657
+ * invert `postgres-docstore.ts`'s `write()` serialization exactly: `deleted=true` → the
658
+ * `Deleted` variant, else `NonClustered` carrying the decoded `docId`.
659
+ * 4. Density-assert the batch (D5) against the replica's PRE-apply state, THEN
660
+ * `replica.write(docs, indexWrites, "Overwrite")` — verbatim, idempotent re-apply.
661
+ * 5. Build `AppliedInvalidation` from the SAME in-memory batch (index-derived writtenTables/
662
+ * writtenKeys, DISTINCT-by-(tableId,internalId) writtenDocs from the doc entries).
663
+ * 6. `await onInvalidation(inv)`, THEN advance the watermark (branded `StablePrefixTs`), THEN
664
+ * resolve any satisfied `waitFor()`s — a throwing/slow handler must not cause a range to be
665
+ * silently skipped.
666
+ */
667
+
668
+ /**
669
+ * Thrown by the apply loop's density assertion (D5, defense-in-depth — the construction, D1 store-
670
+ * allocated ts + D3 epoch-fenced commits, is the actual guarantee that prevents this) when a
671
+ * document entry's `prev_ts` doesn't chain from the replica's actual current head: either a
672
+ * non-null `prev_ts` that doesn't match (or is missing entirely — no live head), or a null
673
+ * `prev_ts` (an insert) where the replica already has a live head. Either shape means a commit
674
+ * touching this document was skipped somewhere between the primary and this replica — serving
675
+ * reads off it from here on would be silent corruption, so the tailer crashes loudly instead. The
676
+ * shipped operator remedy is deleting the local replica file and letting it re-bootstrap from the
677
+ * primary (a replica is always a rebuildable mirror).
678
+ */
679
+ declare class DensityViolationError extends Error {
680
+ readonly docId: InternalDocumentId;
681
+ readonly expectedPrevTs: bigint | null;
682
+ readonly actualHeadTs: bigint | null;
683
+ constructor(docId: InternalDocumentId, expectedPrevTs: bigint | null, actualHeadTs: bigint | null);
684
+ }
685
+ /** Mirrors slice 1's `DerivedInvalidation` shape byte-for-byte (see the module doc comment for why
686
+ * this is a deliberate parallel type, not an import). `newMaxTs` is the tick's APPLIED ceiling,
687
+ * which may be less than the primary's live `maxTimestamp()` when a batch was capped. */
688
+ interface AppliedInvalidation {
689
+ newMaxTs: bigint;
690
+ /** DISTINCT `table_id` values touched, as strings (the storage-encoded table id). */
691
+ writtenTables: string[];
692
+ /** Raw written index keys — point invalidation input, NOT yet point ranges. */
693
+ writtenKeys: Array<{
694
+ indexId: string;
695
+ key: Uint8Array;
696
+ }>;
697
+ /** DISTINCT `(table_id, internal_id)` pairs written in this batch, deduped from the applied
698
+ * `DocumentLogEntry` rows themselves (one entry per doc regardless of how many revisions or
699
+ * index rows accompanied it in this batch). Point invalidation input for the DOCUMENT
700
+ * keyspace, NOT yet point ranges — same split as `writtenKeys` above. */
701
+ writtenDocs: Array<{
702
+ tableId: string;
703
+ internalId: Uint8Array;
704
+ }>;
705
+ }
706
+ interface ReplicaTailerOptions {
707
+ /**
708
+ * Tailer mode (B2b, D5/T5-c):
709
+ * - `"replica"` (default, sync node): verbatim-apply the primary's MVCC log onto the local replica
710
+ * `DocStore` AND derive invalidation from the applied batch — the shipped follower behavior.
711
+ * - `"invalidateOnly"` (writer-ish node in MULTI-WRITER mode): DERIVE-ONLY. The node already has
712
+ * every shard's data (it reads the primary directly), so it never applies to a replica and never
713
+ * runs the density assertions — it only pulls `(watermark, F]` to derive what a FOREIGN writer's
714
+ * commit touched, then fires `onInvalidation` so its own live subscriptions re-run and its oracle
715
+ * observes the new ts. Watermark seeds at boot F (only invalidate commits from here on, not
716
+ * re-derive all history). In this mode the `replica` constructor arg is NEVER dereferenced.
717
+ */
718
+ mode?: "replica" | "invalidateOnly";
719
+ /** Wall-clock poll fallback interval, in ms. Default 1000. */
720
+ pollMs?: number;
721
+ /** Max `DocumentLogEntry` rows pulled per tick (bootstrap catch-up + steady state). Default
722
+ * 1000. A tick may apply slightly more than this to avoid splitting a commit's ts group. */
723
+ batchSize?: number;
724
+ /** Number of shards the fleet runs (B2a). The tailer's target is `F = min(frontier_ts)` over ALL
725
+ * shard rows, and it refuses to treat a partial `shard_leases` (`count(*) < numShards`) as ready —
726
+ * a half-created lease table must not fake a min (belt-and-braces against the F1×N hole). Default
727
+ * 1 → B1's single-shard behavior. */
728
+ numShards?: number;
729
+ /** Invoked once per non-empty applied batch, in watermark order, AFTER the batch has already
730
+ * been written to the replica. The watermark only advances after this resolves. */
731
+ onInvalidation: (inv: AppliedInvalidation) => Promise<void>;
732
+ }
733
+ type WaitOutcome = "reached" | "timeout" | "released";
734
+ /** Verbatim log-apply tailer: primary Postgres MVCC log -> local replica `DocStore`, plus the
735
+ * batch-derived invalidation feed and the `waitFor` read-your-own-writes primitive Task 3 needs. */
736
+ declare class ReplicaTailer {
737
+ private readonly client;
738
+ private readonly primary;
739
+ private readonly replica;
740
+ private readonly mode;
741
+ private readonly pollMs;
742
+ private readonly batchSize;
743
+ private readonly numShards;
744
+ private readonly onInvalidation;
745
+ /** The tailer's own applied high-water mark — a `StablePrefixTs` (D6): only ever seeded from the
746
+ * replica's own persisted `maxTimestamp()` (a prior run's watermark, on restart) or advanced to
747
+ * a freshly-read/capped `F`. Never assigned a raw, un-branded `bigint` directly. */
748
+ private wm;
749
+ /** The last-observed fenced frontier (D5) — tracked purely to assert F never regresses across
750
+ * reads (`null` = no read yet, so the first `readFrontier()` establishes the baseline without
751
+ * asserting anything). */
752
+ private lastF;
753
+ private timer;
754
+ private unlisten;
755
+ private stopped;
756
+ /** Reentrancy guard: a NOTIFY wake and a poll tick can land back-to-back — only one
757
+ * pull-apply-invalidate walk runs at a time, so `onInvalidation` never sees overlapping ranges,
758
+ * and the bootstrap loop in `start()` never races a concurrent LISTEN-triggered tick. */
759
+ private draining;
760
+ private readonly waiters;
761
+ constructor(client: CommitChannelClient, primary: PostgresDocStore, replica: DocStore, opts: ReplicaTailerOptions);
762
+ /** Plain `bigint` on purpose (not the branded `StablePrefixTs`) — external callers (tests, the
763
+ * `PromotionDeps` seam) only ever compare/print this value; widening away the internal brand at
764
+ * the public boundary costs nothing and avoids forcing every caller to know about the brand. */
765
+ watermark(): bigint;
766
+ /**
767
+ * Reads the fenced frontier `F` (Fenced Frontier B1 D5, generalized to N shards in B2a) —
768
+ * `F = min(frontier_ts)` over ALL `shard_leases` rows — via the same `CommitChannelClient` already
769
+ * threaded in for LISTEN/NOTIFY. The min is the dense prefix the WHOLE fleet has durably fenced: a
770
+ * replica may serve reads up to it and never past a commit that hasn't been fenced on every shard.
771
+ *
772
+ * Belt-and-braces (B2a — the F1×N hole otherwise recurs ×N): treat `count(*) < numShards` as
773
+ * F=0-equivalent (NOT ready). A half-created `shard_leases` — some shards claimed+seeded, others not
774
+ * yet — must not let `min` over the present rows fake a ready frontier; the writer's acquire-all +
775
+ * seed-all completes (all N rows present, each seeded ≥ max) BEFORE it reports ready, so a real min
776
+ * only appears once every shard is genuinely fenced. `count(*) = 0` (fresh fleet, pre-acquisition)
777
+ * is the same F=0.
778
+ *
779
+ * Asserts F is monotonically non-decreasing across reads once the count gate is satisfied (D5's
780
+ * defense-in-depth invariant) — a regression can only mean `shard_leases` was corrupted/hand-
781
+ * tampered. The assertion is skipped while `count(*) < numShards` (F is a placeholder 0 there, so a
782
+ * later real min is legitimately larger, not a regression).
783
+ */
784
+ private readFrontier;
785
+ /**
786
+ * @param seedWm ONLY consulted in `"invalidateOnly"` mode (ignored in `"replica"` mode, which always
787
+ * seeds from the replica's own persisted high-water mark — see below). When provided, seeds the
788
+ * watermark from THIS value instead of a fresh `readFrontier()` read.
789
+ *
790
+ * Fixes the PROMOTION-HANDOFF INVALIDATION GAP (B2b whole-branch review): a promoting node stops
791
+ * its `ReplicaTailer` (sync mode) at some watermark `W`, then starts this derive-only listener.
792
+ * Between the tailer's stop and the listener's start, a co-writer's commit can land and bump the
793
+ * fenced frontier `F` past `W` — if the listener then seeds from a FRESH `readFrontier()` (i.e.
794
+ * that same later `F`), it treats everything up to `F` as already-known and never derives/
795
+ * invalidates the `(W, F]` range, even though nothing ever actually invalidated it: the sync
796
+ * tailer was already stopped, and this listener starts past it. A subscription open across the
797
+ * promotion goes silently stale for exactly those commits.
798
+ *
799
+ * The caller (`node.ts`'s `startWriterInvalidationListener`) closes this gap by passing the
800
+ * OUTGOING tailer's own final `watermark()` (captured right after `tailer.stop()`) as `seedWm` on
801
+ * promotion, so the listener picks up exactly where the tailer left off — contiguous coverage,
802
+ * with the tailer's own last-applied range harmlessly re-derived (idempotent double-invalidation).
803
+ * Writer BOOT (no prior tailer, so no gap by construction) omits `seedWm` and keeps the original
804
+ * fresh-`readFrontier()` behavior.
805
+ */
806
+ start(seedWm?: bigint): Promise<void>;
807
+ stop(): Promise<void>;
808
+ /** Resolves when `watermark() >= ts` (immediately if already true), or when `release()` fires.
809
+ * Task 3's read-your-own-writes primitive: a client that just committed at `ts` can wait for
810
+ * a follower's replica to have caught up before serving that client's next read from it. */
811
+ waitFor(ts: bigint, timeoutMs: number): Promise<WaitOutcome>;
812
+ /** Releases ALL pending `waitFor()`s with `"released"` (e.g. this node was just promoted to
813
+ * writer, or is shutting down, and callers should stop waiting on replica catch-up). */
814
+ release(): void;
815
+ private wakeSatisfiedWaiters;
816
+ private tick;
817
+ /** One pull-apply-invalidate walk. Split out from `tick()` so `tick()` can wrap it with the
818
+ * DensityViolationError halt handling without that catch swallowing the `draining` bookkeeping. */
819
+ private drainOnce;
820
+ /**
821
+ * Density assertion (D5, defense-in-depth): for EVERY document entry in this batch, in
822
+ * `DocumentLogEntry` order, the replica's head for that document IMMEDIATELY BEFORE this entry
823
+ * lands must chain from `entry.prev_ts` exactly — `prev_ts !== null` requires a live head equal
824
+ * to it; `prev_ts === null` (an insert) requires no live head at all. Throws `DensityViolationError`
825
+ * on the first mismatch.
826
+ *
827
+ * IDEMPOTENT RE-APPLY exception: if the replica's head is ALREADY at this exact entry's own
828
+ * `ts`, that's not a violation — it's this exact revision being re-applied (the whole class is
829
+ * built around `"Overwrite"` idempotency, e.g. a restart re-walking an already-applied range).
830
+ * Since `ts` is the log's globally unique per-shard commit position, a head landing on it can
831
+ * only mean this precise commit already landed here, never an unrelated collision.
832
+ *
833
+ * A batch can carry more than one revision of the SAME document (it was written more than once
834
+ * inside the pulled `(wm, F]` range), so "the head immediately before this entry" is a running
835
+ * per-batch simulation — seeded from the replica's REAL state (`replica.get`) the first time a
836
+ * doc is encountered, then advanced in-memory to each entry's own `ts` as it's validated —
837
+ * rather than re-reading the replica for every entry (which would see later, not-yet-applied
838
+ * entries' predecessor as still absent).
839
+ */
840
+ private assertDensity;
841
+ /**
842
+ * Pull `DocumentLogEntry` rows for `(after, upTo]` from `load_documents`, capped at
843
+ * `this.batchSize` — but if the cap lands mid-way through a group of rows sharing the same
844
+ * commit `ts`, keep draining until that ts group is fully collected before stopping. A single
845
+ * transaction shares exactly one commit `ts` across every document it wrote (see
846
+ * `postgres-docstore.ts`'s `write()`), so cutting a batch mid-group would apply a transaction's
847
+ * writes partially, which `load_documents`'s `(after, cap]`-shaped next-tick range would then
848
+ * never revisit (the excluded rows share `ts === cap`, which the next tick's range excludes as
849
+ * its own lower bound).
850
+ *
851
+ * `TimestampRange` is `{minInclusive, maxExclusive}` (see `packages/docstore/src/types.ts`) —
852
+ * the half-open opposite skew from the `(after, upTo]` shape this tailer needs, so the bounds
853
+ * are translated with a `+1n` offset on both sides (safe: `ts` is a monotonic integer log
854
+ * position, never a real number needing density guarantees).
855
+ */
856
+ private pullDocs;
857
+ }
858
+
859
+ /**
860
+ * `WriteForwarder` — the non-owner side of the fleet write path. Implements the engine's
861
+ * per-shard `WriteRouter` seam (B2b, D1): for any shard this node does NOT currently hold,
862
+ * `forward()` POSTs the call to that shard's owner (`/_fleet/run`, discovered per-shard from the
863
+ * `shard_leases` row) and returns its JSON result. `isLocalWriter(shardId)` is a live view of the
864
+ * node's held-shard set (`LeaseManager.currentEpoch(shardId) !== null` — the SAME source of truth
865
+ * `relinquish()` uses for its own idempotency check), so a shard acquired/relinquished mid-flight
866
+ * (balancer rebalancing, failover) takes effect on the very next call — no caching, no stale role.
867
+ *
868
+ * The forwarder learns each shard's writer URL from that shard's `shard_leases` discovery row (via
869
+ * `LeaseManager.read(shardId)`), never from static config — so a failover to a new owner is picked
870
+ * up by re-reading the lease. `writerUrlFor(shardId)` caches per shard (one node normally forwards
871
+ * the same shard repeatedly) and refreshes on a POST failure, retrying once — same shape as the
872
+ * shipped single-shard `writerUrl()` this replaces, generalized per shard.
873
+ *
874
+ * Single-hop guard (B2b, D1 spec-review edit): every forward body carries `forwarded: true`. If it
875
+ * lands on a node that is ALSO not the shard's owner (a point-in-time race during rebalance
876
+ * convergence), the receiver's `/_fleet/run` handler rejects with a typed, retryable
877
+ * `NotShardOwnerError` instead of re-forwarding itself (which would let a forward chase a moving
878
+ * target unboundedly). `forward()` treats that ONE error shape like a transport failure — refresh
879
+ * the shard's cached URL and retry once — then surfaces whatever the second attempt does. Any OTHER
880
+ * typed `HelipodError` (an OCC conflict, a validation failure, …) means a LIVE owner answered
881
+ * DEFINITIVELY and is re-thrown unchanged, exactly as before.
882
+ *
883
+ * Task 3 (read-your-own-writes): `/_fleet/run`'s response carries the write's `commitTs`
884
+ * (stringified — bigints don't survive `JSON.stringify`). If a `ReplicaTailer` has been attached
885
+ * via `attachTailer()` (this node is a fleet SYNC node reading off a local replica), `forward()`
886
+ * waits for that replica's watermark to reach `commitTs` before resolving — otherwise a client that
887
+ * just wrote through this node could immediately read its own write's absence off a replica that
888
+ * hasn't caught up yet. `promote()` also releases any pending wait: once this node becomes the
889
+ * writer, replica catch-up is no longer the right thing to block on.
890
+ */
891
+
892
+ interface WriteForwarderOptions {
893
+ /** Admin bearer token — the `/_fleet/run` endpoint authenticates with the deployment admin key. */
894
+ adminKey: string;
895
+ /** This node's own advertised URL (recorded on the lease when/if it becomes the writer). */
896
+ selfUrl: string;
897
+ }
898
+ /**
899
+ * The narrow seam `WriteForwarder` needs from `ReplicaTailer` — declared locally (rather than
900
+ * typing `attachTailer` as `ReplicaTailer` itself) so tests can pass a lightweight structural stub
901
+ * instead of standing up a real tailer (which needs a live Postgres primary + replica store).
902
+ * A real `ReplicaTailer` satisfies this trivially since it's a strict subset of its public API.
903
+ */
904
+ type ReplicaWaiter = Pick<ReplicaTailer, "waitFor" | "release">;
905
+ declare class WriteForwarder implements WriteRouter {
906
+ private readonly lease;
907
+ private readonly opts;
908
+ private tailer;
909
+ /** Guard each distinct malformed-response warning to once per process (independently — an absent
910
+ * commitTs and an unparseable one are different failure modes and each deserves its own log
911
+ * line) so a bad/old writer response doesn't spam the log on every subsequent forwarded write. */
912
+ private warnedMissingCommitTs;
913
+ private warnedUnparseableCommitTs;
914
+ /** Per-shard writer-URL cache (B2b, T2): populated on first forward for a shard, refreshed on a
915
+ * POST failure (transport error OR a not-the-owner rejection) and retried once. A node normally
916
+ * forwards the same shard repeatedly, so caching avoids a `shard_leases` read on every call. */
917
+ private readonly writerUrlCache;
918
+ constructor(lease: LeaseManager, opts: WriteForwarderOptions);
919
+ /** Attach the local `ReplicaTailer` this node reads off, enabling the read-your-own-writes wait
920
+ * in `forward()`. Fleet WRITER nodes never call this — they have no replica to wait on. */
921
+ attachTailer(t: ReplicaWaiter): void;
922
+ /** Called on promotion (sync → writer-ish). Releases any read-your-own-writes wait in flight —
923
+ * this node no longer serves reads off a replica for its own forwarded writes, so waiting on
924
+ * catch-up would only add latency. `isLocalWriter` itself needs no flip here: it already reads
925
+ * the live `LeaseManager` state directly (see below), which promotion's `tryAcquire` calls
926
+ * update as a side effect.
927
+ *
928
+ * NOTE (Fleet B3): a HYBRID promotion (multi-writer) does NOT call this — a hybrid keeps its
929
+ * replica + tailer running past promotion and STILL serves reads (and forwarded writes to shards
930
+ * it doesn't own) off the replica, so releasing the RYOW waits would be wrong. Only the
931
+ * single-writer failover promotion (`promoteFleetNode`, no replica read path afterward) calls it. */
932
+ promote(): void;
933
+ /**
934
+ * Fleet B3 (D2) — the HYBRID own-commit RYOW gate. Wired as the runtime's `beforeNotify` drain
935
+ * hook: awaited before a LOCALLY-committed mutation's subscription re-runs fire, so those re-runs
936
+ * (which read this node's replica, via the hybrid query path) don't observe the commit's absence on
937
+ * a replica that hasn't applied it yet. Delegates to the SAME attached `ReplicaTailer.waitFor` the
938
+ * forwarded-write RYOW uses (`attachTailer`), so local and forwarded writes share one catch-up
939
+ * primitive. No-op when no tailer is attached (a single-writer node with no replica, or before the
940
+ * tailer exists) or when nothing committed (`0n`). A timeout is swallowed — reactivity is
941
+ * best-effort; the read path stays correct regardless (the re-run just fires against a replica that
942
+ * is still catching up, no worse than the pre-gate behavior).
943
+ */
944
+ waitForReplica(commitTs: bigint): Promise<void>;
945
+ /**
946
+ * True iff this node currently holds `shardId`'s write lease (B2b, D1: per-shard membership, not
947
+ * a whole-node binary role). `LeaseManager.currentEpoch(shardId) !== null` is the SAME live
948
+ * held-set accessor `relinquish()` uses as its own idempotency check (`node.ts`) — so this can
949
+ * never disagree with what the commit guard / relinquish dispatcher consider "held". Consulted
950
+ * fresh on every call (never cached): a shard acquired or relinquished mid-flight (balancer
951
+ * rebalance, failover, promotion) takes effect on the very next mutation.
952
+ */
953
+ isLocalWriter(shardId?: ShardId): boolean;
954
+ forward(kind: "mutation" | "action", path: string, args: JSONValue, identity: string | null, shardId?: ShardId, dedup?: {
955
+ clientId: string;
956
+ seq: number;
957
+ }): Promise<{
958
+ value: JSONValue;
959
+ commitTs?: number;
960
+ shardId?: string;
961
+ replay?: ClientReplay;
962
+ }>;
963
+ /** Discover shard `shardId`'s current writer URL — cached after the first read. */
964
+ private writerUrlFor;
965
+ /** Force a fresh `shard_leases` read for `shardId`, overwriting any cached URL — the refresh half
966
+ * of `writerUrlFor`'s cache, also used directly by `forward()`'s retry-once path. */
967
+ private refreshWriterUrlFor;
968
+ private post;
969
+ /** Waits for the local replica to observe `commitTsStr`, when a tailer is attached. No-op on a
970
+ * fleet WRITER node (no tailer attached) or when the write committed nothing (`0`/absent). */
971
+ private waitForReplicaCatchUp;
972
+ }
973
+
974
+ /**
975
+ * `ShardLeaseBalancer` — the deterministic, coordinator-free shard placement loop (B2b, D3).
976
+ *
977
+ * Every fleet node (writer OR sync) runs one. Each ~2s beat:
978
+ * 1. HEARTBEAT this node's `fleet_nodes` presence row (its liveness signal — the balancer is the
979
+ * presence heartbeat for a shardless SYNC node, which has no `shard_leases` row and no
980
+ * LeaseMonitor; a writer node's LeaseMonitor probe also heartbeats presence, redundantly and
981
+ * harmlessly).
982
+ * 2. READ the live node set (`fleet_nodes` ∪ live `shard_leases` holders) and compute this node's
983
+ * rendezvous TARGET shards — the shards HRW assigns to `myUrl` over that set. Every node derives
984
+ * the same assignment from the same rows (`rendezvous.ts`), so there is nothing to negotiate.
985
+ * 3. If this node is a pure SYNC node (`!isWriterish()`) and any NON-default target is currently
986
+ * acquirable (orphaned/expired/missing) — a peer released it, or its holder died — request the
987
+ * shipped whole-node PROMOTION (sync → writer-ish); a promoted node then acquires on the next
988
+ * steps. (The DEFAULT shard has its own shipped election path — `LeaseManager.acquireLoop` — so
989
+ * it is not a promotion trigger here; that avoids a redundant second promotion driver racing the
990
+ * election.)
991
+ * 4. Writer-ish node: ACQUIRE any target shard it does not hold that is acquirable (orphaned/
992
+ * expired/missing). This is failover — un-damped and prompt; it NEVER touches a live non-target
993
+ * holder's row (advisory locks + fencing already guarantee at most one holder, so acquisition is
994
+ * always safe).
995
+ * 5. Writer-ish node, under DAMPING (live set identical for ≥2 consecutive beats): gracefully
996
+ * RELEASE any held shard NOT in its target set — a point-in-time self-fence (`releaseShard`, wired
997
+ * in `node.ts`) that hands the shard to its rightful owner, which acquires it on its own next
998
+ * beat (~2s), no TTL wait, no failover event. Damping keeps a flapping membership from thrashing
999
+ * placement; acquisition (step 4) is intentionally NOT damped so a real death recovers promptly.
1000
+ *
1001
+ * The balancer owns ZERO SQL beyond the three lease reads — all acquire/release/promote effects are
1002
+ * injected thunks (`node.ts` wires them over `LeaseManager`/the runtime), so this class is a pure
1003
+ * orchestrator and unit-testable with lightweight fakes.
1004
+ */
1005
+
1006
+ /** The narrow slice of `LeaseManager` the balancer reads — a structural interface so tests can pass a
1007
+ * fake, and so the balancer never reaches past these three reads into the lease's write surface. */
1008
+ interface BalancerLease {
1009
+ /** Upsert this node's `fleet_nodes` presence row, extending `expires_at` by the lease TTL. */
1010
+ heartbeatPresence(): Promise<void>;
1011
+ /** Distinct unexpired advertise URLs — `fleet_nodes` ∪ live `shard_leases.writer_url` holders. */
1012
+ liveNodes(): Promise<string[]>;
1013
+ /** Per-shard ownership snapshot: for each existing `shard_leases` row, its `writer_url` (null =
1014
+ * orphaned) and whether it has expired per the DB clock. A shard with NO row is absent from the map
1015
+ * (treated as acquirable). */
1016
+ readShardOwnership(): Promise<Map<ShardId, {
1017
+ writerUrl: string | null;
1018
+ expired: boolean;
1019
+ }>>;
1020
+ }
1021
+ interface ShardLeaseBalancerDeps {
1022
+ lease: BalancerLease;
1023
+ /** This node's advertised URL — its identity in the rendezvous candidate set. */
1024
+ myUrl: string;
1025
+ /** Shard count (the fleet's fixed NUM_SHARDS). Drives the `shardIdList` this balancer ranges over. */
1026
+ numShards: number;
1027
+ /** Live per-shard held-set membership (`LeaseManager.currentEpoch(shardId) !== null`). */
1028
+ isHeld(shardId: ShardId): boolean;
1029
+ /** Is this node writer-ish (has completed promotion / booted as the writer)? A pure sync node only
1030
+ * heartbeats presence and may request promotion; it never acquires/releases. */
1031
+ isWriterish(): boolean;
1032
+ /** Acquire `shardId`'s lease (+ its per-slot advisory lock), evicting a wedged expired holder if
1033
+ * needed. Returns whether the shard is now held. One tick's attempt — a miss retries next beat. */
1034
+ tryAcquireShard(shardId: ShardId): Promise<boolean>;
1035
+ /** Gracefully release a held shard (point-in-time self-fence under the shard's commit mutex, then
1036
+ * the T3 relinquish-unwind) so its rightful owner can acquire it. */
1037
+ releaseShard(shardId: ShardId): Promise<void>;
1038
+ /** Run the shipped whole-node promotion (sync → writer-ish). Idempotent at the node level. */
1039
+ requestPromotion(): Promise<void>;
1040
+ /**
1041
+ * Fleet B3, D3 (effectively-once forwarding): reclaim `fleet_idempotency` rows older than the
1042
+ * TTL — a cheap indexed delete, run on every WRITER-ish beat (see `LeaseManager.sweepIdempotency`).
1043
+ * Optional so an older/stub `ShardLeaseBalancerDeps` (and every existing balancer test) needs no
1044
+ * change; a pure sync node never has this called regardless (see `tick()`).
1045
+ */
1046
+ sweepIdempotency?(): Promise<void>;
1047
+ /**
1048
+ * Multi-writer scale-out mode (B2b, D3 — the "writer nodes vs sync nodes" scaling knob, off by
1049
+ * default). When OFF (the default, and the shipped single-writer behavior): this node's target set is
1050
+ * EVERY shard (the sole writer owns them all), the balancer NEVER gracefully releases a live-held
1051
+ * shard to a peer, and a pure sync node is NEVER auto-promoted by rendezvous — additional nodes stay
1052
+ * READ REPLICAS (sync), reading off their local replica whose tailer sees every shard's commits.
1053
+ * Failover acquisition (picking up an orphaned/expired shard) and presence heartbeating are ON
1054
+ * regardless. When ON: full rendezvous distribution — each live node owns and writes its HRW share,
1055
+ * peers gracefully release a newcomer's share, and a sync node promotes to claim an orphaned share.
1056
+ * (Enabling it makes a joining node a CO-WRITER, whose own subscriptions do not yet receive
1057
+ * cross-writer reactivity for shards it doesn't hold — that is T6's proving ground.)
1058
+ */
1059
+ multiWriter?: boolean;
1060
+ /** Beat interval (ms). Default 2000. */
1061
+ beatMs?: number;
1062
+ /** Structured-log seam for a tick error, defaults to `console.error`. */
1063
+ log?: (msg: string) => void;
1064
+ }
1065
+ declare class ShardLeaseBalancer {
1066
+ private readonly deps;
1067
+ private readonly shards;
1068
+ private readonly multiWriter;
1069
+ private readonly beatMs;
1070
+ private readonly log;
1071
+ private timer;
1072
+ private stopped;
1073
+ private running;
1074
+ /** Canonical (sorted, comma-joined) live set from the PREVIOUS beat — damping compares against it:
1075
+ * a RELEASE only fires when the current live set equals this (identical for ≥2 consecutive beats). */
1076
+ private previousLiveSet;
1077
+ constructor(deps: ShardLeaseBalancerDeps);
1078
+ /** Begin the periodic beat. The FIRST beat fires after `beatMs` (never synchronously) — prompt boot
1079
+ * acquisition is done explicitly via `acquireTargetsNow()`, not the periodic loop. Idempotent. */
1080
+ start(): void;
1081
+ stop(): void;
1082
+ /** Resolve the live candidate set for a beat: the DB's live nodes ∪ this node itself (a node is
1083
+ * always live to itself — belt-and-braces so its own boot never depends on its presence row having
1084
+ * landed/propagated yet). Returns both the URL array (rendezvous input) and its canonical form. */
1085
+ private liveCandidates;
1086
+ /** This node's target shards. In single-writer mode (the default) the sole writer's targets are ALL
1087
+ * shards — byte-identical to B2a's acquire-all — so a joining node never steals a share. In
1088
+ * multi-writer mode it is this node's rendezvous (HRW) share over `liveUrls`. */
1089
+ private targetsFor;
1090
+ /** A shard is acquirable iff it has no row (absent from the map), is orphaned (`writer_url NULL`), or
1091
+ * has expired — i.e. NOT held live by a peer. Acquisition only ever targets these; a live non-target
1092
+ * holder is never fenced by the balancer (that's failover's job). */
1093
+ private acquirable;
1094
+ /**
1095
+ * Un-damped acquire pass over this node's CURRENT rendezvous targets — called at writer boot and on
1096
+ * promotion so the node holds its share PROMPTLY (before the ready line / before the first periodic
1097
+ * beat), rather than waiting out a 2s tick. In a single-node fleet the target set is EVERY shard, so
1098
+ * this acquires all N — byte-identical steady state to B2a's acquire-all. Seeds `previousLiveSet` so
1099
+ * the first periodic beat treats the boot membership as the damping baseline.
1100
+ */
1101
+ acquireTargetsNow(): Promise<void>;
1102
+ /** One balancer beat (see the class doc for the five steps). Never throws — a tick error is logged
1103
+ * and the loop continues; the next beat retries against fresh state. */
1104
+ tick(): Promise<void>;
1105
+ }
1106
+
1107
+ /**
1108
+ * Rendezvous (highest-random-weight) hashing for the shard balancer (B2b, D3).
1109
+ *
1110
+ * Every fleet node runs the SAME pure function over the SAME live-node set (discovered from
1111
+ * `fleet_nodes` ∪ live `shard_leases` holders — see `LeaseManager.liveNodes`), so all nodes derive
1112
+ * an identical shard→owner assignment with NOTHING to negotiate. For each shard we compute a stable
1113
+ * 64-bit weight `hash(shardId, advertiseUrl)` per candidate node and pick the max — the owner. When
1114
+ * the membership set changes, HRW moves ONLY the shards whose max-weight node changed (minimality),
1115
+ * so a join/leave reshuffles a bounded, minimal set of shards rather than the whole ring.
1116
+ *
1117
+ * The weight is FNV-1a-64 over the UTF-8 bytes of `${shardId}${advertiseUrl}` — the same
1118
+ * hashing discipline `@helipod/id-codec`'s jump-hash router uses (FNV-1a-64 → a well-mixed 64-bit
1119
+ * key), reimplemented locally rather than importing that package's private helper so the rendezvous
1120
+ * weight function is self-contained and its exact bytes are pinned here. It MUST stay a pure function
1121
+ * of the two strings and identical across nodes/releases — every node's placement decision depends on
1122
+ * byte-for-byte agreement.
1123
+ */
1124
+
1125
+ /**
1126
+ * The rendezvous weight of `advertiseUrl` for `shardId` — a stable 64-bit value. A space separates
1127
+ * the two fields so `("s1","x")` and `("s","1x")` can never collide into the same input — safe
1128
+ * because neither `shardId` nor `advertiseUrl` ever contains a space (shard ids are short identifiers
1129
+ * like `"default"`/`"s1"`; advertise URLs are `scheme://host:port` with no whitespace).
1130
+ */
1131
+ declare function rendezvousWeight(shardId: ShardId, advertiseUrl: string): bigint;
1132
+ /**
1133
+ * The owner of `shardId` under rendezvous hashing over `liveUrls`: the node with the maximum weight,
1134
+ * ties broken by the lexicographically-smaller URL (so the choice is fully deterministic even in the
1135
+ * astronomically unlikely event of a 64-bit weight collision). Returns `""` for an empty candidate
1136
+ * set (no live node — the caller always includes at least its own URL, so this is defensive only).
1137
+ * Pure and allocation-light; identical output on every node given the same `liveUrls` (order-
1138
+ * independent — the max/tie-break does not depend on the input array's order).
1139
+ */
1140
+ declare function rendezvousOwner(shardId: ShardId, liveUrls: readonly string[]): string;
1141
+
1142
+ /**
1143
+ * A `DocStore` that forwards every call to a swappable inner delegate, so a replica-to-primary
1144
+ * promotion (Task 4) can become a single atomic pointer swap instead of a coordinated teardown of
1145
+ * every in-flight caller.
1146
+ *
1147
+ * Atomicity contract: each method reads `this.delegate` exactly once, at call entry, into a local
1148
+ * binding, and completes its whole operation against that snapshot. A call already in progress when
1149
+ * `swapTo()` runs finishes against the OLD delegate; a call started after `swapTo()` returns sees
1150
+ * the NEW one. There is no instant where a single call is torn between two delegates. This holds for
1151
+ * the async generators too (`index_scan`/`load_documents`): the delegate is captured before the
1152
+ * first `yield`, so a scan already draining rows from the old store keeps draining the old store to
1153
+ * completion even if `swapTo()` runs concurrently.
1154
+ *
1155
+ * `close()` semantics: closes only the CURRENT delegate (whatever `this.delegate` is at the moment
1156
+ * `close()` is called). It does NOT close a delegate that was swapped out by an earlier `swapTo()` —
1157
+ * this class has no memory of former delegates once replaced. Closing a swapped-out (e.g. demoted
1158
+ * primary, or promoted-from replica) delegate is the lifecycle owner's responsibility (Task 4
1159
+ * decides when that's safe — e.g. after draining its in-flight callers), not this wrapper's.
1160
+ */
1161
+ declare class SwitchableDocStore implements DocStore {
1162
+ private delegate;
1163
+ constructor(initial: DocStore);
1164
+ /** Atomically repoint all future calls at `next`. Does not touch the outgoing delegate. */
1165
+ swapTo(next: DocStore): void;
1166
+ /** The delegate current calls are being forwarded to. */
1167
+ current(): DocStore;
1168
+ setupSchema(options?: SchemaSetupOptions): Promise<void>;
1169
+ write(documents: readonly DocumentLogEntry[], indexUpdates: readonly IndexWrite[], conflictStrategy: ConflictStrategy, shardId?: ShardId$1): Promise<void>;
1170
+ commitWrite(documents: readonly DocumentLogEntry[], indexUpdates: readonly IndexWrite[], shardId?: ShardId$1, opts?: {
1171
+ meta?: Record<string, string>;
1172
+ }): Promise<bigint>;
1173
+ commitWriteBatch(units: readonly CommitUnit[], shardId?: ShardId$1): Promise<bigint[]>;
1174
+ /** Forwards to the CURRENT delegate at call entry, same atomicity contract as every other
1175
+ * method here — but note the resulting registration is NOT itself re-forwarded on a later
1176
+ * `swapTo()`: the guard lands on whichever concrete store was `this.delegate` at the moment
1177
+ * `addCommitGuard` was called, and stays registered there even after the switchable repoints
1178
+ * elsewhere. In practice fleet code (`node.ts`) always calls `addCommitGuard` on the concrete
1179
+ * `PostgresDocStore` directly (never through this wrapper), so this pass-through exists purely
1180
+ * for `DocStore` interface conformance. */
1181
+ addCommitGuard(guard: (q: any, units: readonly CommitGuardUnit[], shardId: ShardId$1) => void | Promise<void>): () => void;
1182
+ get(id: InternalDocumentId, readTimestamp?: bigint): Promise<LatestDocument | null>;
1183
+ index_scan(indexId: string, tableId: string, readTimestamp: bigint, interval: Interval, order: Order, limit?: number): AsyncGenerator<readonly [Uint8Array, LatestDocument]>;
1184
+ load_documents(range: TimestampRange, order: Order, limit?: number): AsyncGenerator<DocumentLogEntry>;
1185
+ previous_revisions(queries: readonly PrevRevQuery[]): Promise<Map<string, DocumentLogEntry>>;
1186
+ scan(tableId: string, readTimestamp?: bigint): Promise<LatestDocument[]>;
1187
+ count(tableId: string): Promise<number>;
1188
+ maxTimestamp(): Promise<bigint>;
1189
+ getGlobal(key: string): Promise<JSONValue | null>;
1190
+ writeGlobal(key: string, value: JSONValue): Promise<void>;
1191
+ writeGlobalIfAbsent(key: string, value: JSONValue): Promise<boolean>;
1192
+ getClientVerdict(identity: string, clientId: string, seq: number): Promise<ClientVerdictRecord | null>;
1193
+ getClientFloor(identity: string, clientId: string): Promise<number | null>;
1194
+ recordClientVerdict(identity: string, clientId: string, seq: number, record: ClientVerdictWrite): Promise<void>;
1195
+ updateClientVerdictValue(identity: string, clientId: string, seq: number, value: JSONValue): Promise<void>;
1196
+ pruneClientMutations(identity: string, clientId: string, opts: {
1197
+ ackedThrough?: number;
1198
+ ttlBeforeMs?: number;
1199
+ }): Promise<{
1200
+ prunedThroughSeq: number;
1201
+ }>;
1202
+ sweepExpiredClientMutations(beforeMs: number): Promise<{
1203
+ deletedCount: number;
1204
+ }>;
1205
+ close(): void | Promise<void>;
1206
+ }
1207
+
1208
+ /** The filename of a sync node's local embedded replica, under the serve data dir. */
1209
+ declare const REPLICA_DB_FILENAME = "fleet-replica.db";
1210
+ /** Default number of shards a fleet node runs (B2a). T5 owns the persist-once/env/`HELIPOD_FLEET_
1211
+ * SHARDS` story; this task threads it as a plain parameter (`prepareFleetNode`'s `numShards` dep),
1212
+ * defaulting here. The one node holds ALL N shard leases, commits in parallel across them (per-shard
1213
+ * commit-connection pool + per-shard `ShardedTransactor` mutexes), and F = min over the N frontiers. */
1214
+ declare const DEFAULT_NUM_SHARDS = 8;
1215
+ /** A point-in-time frontier-lag reading for the health endpoint (D5): the fleet-wide fenced frontier
1216
+ * (`min(frontier_ts)` across all shard rows), how long (wall-clock ms) it has been stuck at that
1217
+ * value, and which shard is holding it there. */
1218
+ interface FrontierStats {
1219
+ frontier: bigint;
1220
+ lagMs: number;
1221
+ pinningShard: ShardId;
1222
+ }
1223
+ /**
1224
+ * Fleet frontier-lag monitor + (writer-only) idle-shard closer (B2a, D5). One runs per fleet node:
1225
+ *
1226
+ * - On the WRITER (`closeIdle: true`) each beat first CLOSES idle shards — `lease.closeIdleFrontiers`
1227
+ * allocates one `nextval` and advances every held shard whose frontier lags it — so an idle shard
1228
+ * can't pin `F = min(frontier_ts)` below the live commit position for more than a beat. A local
1229
+ * commit also schedules a coalesced beat (~10ms) so F reacts promptly to writes on sibling shards.
1230
+ * - On EITHER role each beat then READS all shard frontiers to compute `min` + the pinning shard and
1231
+ * track how long `min` has been stuck (wall-clock since it last advanced) → the `FrontierStats`
1232
+ * the health endpoint reports. A lag past `lagWarnMs` logs a warning naming the pinning shard
1233
+ * (once per stall, reset when it recovers), the console signal D5 asks for.
1234
+ *
1235
+ * A sync node runs it read-only (`closeIdle: false`) — it holds no leases, so it must NOT allocate
1236
+ * `nextval` (only the writer drives the frontier); it just observes whether the fleet frontier is
1237
+ * advancing.
1238
+ */
1239
+ declare class FrontierMonitor {
1240
+ private readonly lease;
1241
+ private readonly opts;
1242
+ private timer;
1243
+ private coalesceTimer;
1244
+ private stopped;
1245
+ private running;
1246
+ private cached;
1247
+ /** The last-observed `min(frontier_ts)` and the wall-clock time it was first seen — lag is the age
1248
+ * of the CURRENT min (how long it's been stuck), so both reset whenever min advances. */
1249
+ private lastMin;
1250
+ private minSinceMs;
1251
+ private warned;
1252
+ /** D4 — replica-lag tracking: wall-clock time this node's OWN tailer watermark was first observed
1253
+ * BEHIND the fleet frontier F, or `null` while it's caught up. Mirrors `minSinceMs`/`lastMin`'s
1254
+ * "age of the current stuck condition" shape, but for the replica catch-up gap rather than F's own
1255
+ * advance. Reset the instant `wm` reaches `F` again — a transient blip re-arms silently. */
1256
+ private replicaLagSinceMs;
1257
+ private replicaLagWarned;
1258
+ constructor(lease: LeaseManager, opts: {
1259
+ closeIdle: boolean;
1260
+ /** Per-shard commit-mutex seam the idle-frontier closer runs each bump under (see
1261
+ * `closeIdleFrontiers`). Required when `closeIdle` is true; ignored otherwise. */
1262
+ runExclusiveOnShard?: TryRunExclusiveOnShard;
1263
+ beatMs?: number;
1264
+ coalesceMs?: number;
1265
+ lagWarnMs?: number;
1266
+ now?: () => number;
1267
+ warn?: (msg: string) => void;
1268
+ /**
1269
+ * D4 — this node's own local replica-tailer watermark, when it has one: a sync node's
1270
+ * `ReplicaTailer`, or (in the hybrid/multi-writer regime) a writer's own read-replica tailer.
1271
+ * Omitted for a non-hybrid writer (it reads the primary directly — no replica, no lag to track).
1272
+ * Compared against the fleet frontier `F = min(frontier_ts)` each beat; `wm` falling more than
1273
+ * `lagWarnMs` behind `F` logs ONE operator warning naming this node's replica (`lease.advertiseUrl`
1274
+ * — there is only ever one replica per node, so naming the NODE identifies it), silent below the
1275
+ * threshold and while caught up. Same shape/once-ness as the pinning-shard warning above; applies
1276
+ * identically whether this monitor is a sync node's or a hybrid writer's (same `beat()` code path).
1277
+ */
1278
+ tailerWatermark?: () => bigint;
1279
+ /**
1280
+ * T3.5 (the fleet-connections bench's notify-diagnosis fix): fired at most once per beat, only
1281
+ * when this beat's `closeIdleFrontiers`/`bumpOrphanFrontiers` (or a concurrent commit) actually
1282
+ * pushed the fleet ceiling `F = min(frontier_ts)` past what the PRIOR beat observed. Neither
1283
+ * closer NOTIFYs on its own bare `UPDATE` — without this, an idle-shard close that finally
1284
+ * un-pins F is invisible to every `ReplicaTailer`'s LISTEN, and delivery silently falls through
1285
+ * to the tailer's `DEFAULT_POLL_MS` (1000ms) fallback even though NOTIFY itself is wired and
1286
+ * fires within ~1ms of the real commit (see `.superpowers/sdd/notify-diagnosis.md`). Omitted
1287
+ * entirely on a sync node (`closeIdle: false`) — only the writer's own beat performs the
1288
+ * advancing work, so only it needs to announce it. The callback owns its own failure handling
1289
+ * (fire-and-forget, matching `NotifyingFanoutAdapter.publish`'s own NOTIFY) — a rejected notify
1290
+ * must not, and structurally cannot, break the beat.
1291
+ */
1292
+ notifyOnAdvance?: (ceiling: bigint) => void;
1293
+ });
1294
+ private get now();
1295
+ start(): void;
1296
+ /** Schedule a single coalesced beat `coalesceMs` out — called on each local commit so an idle
1297
+ * sibling shard un-pins F within ~10ms instead of waiting out the periodic beat. Multiple commits
1298
+ * inside the window collapse onto one beat. */
1299
+ triggerCoalesced(): void;
1300
+ private beat;
1301
+ /** The most recent frontier reading, or null if no beat has completed yet. `lagMs` is recomputed
1302
+ * against the current clock so a caller between beats still sees a fresh age. */
1303
+ stats(): FrontierStats | null;
1304
+ stop(): void;
1305
+ }
1306
+ /**
1307
+ * D4 — a generic async-op serializer: returns a `run(op)` function where every queued `op` executes
1308
+ * only after the PREVIOUS one has settled (resolved OR rejected — `.then(op, op)`, so one op
1309
+ * rejecting never wedges the chain for the next). Callers that fire-and-forget a burst of overlapping
1310
+ * async calls (see `startFleetNode`'s `startDriversChained`/`stopDriversOnlyChained`) get a
1311
+ * deterministic EXECUTION order matching the ISSUE order, even when the callee itself is merely
1312
+ * idempotent (idempotence alone doesn't prevent a later-issued-but-faster call from resolving before
1313
+ * an earlier one, which is the race this closes). Each call to `createAsyncChain()` returns an
1314
+ * independent chain — callers needing one shared serialization point construct exactly one instance
1315
+ * and route every op through the same returned function.
1316
+ */
1317
+ declare function createAsyncChain(): (op: () => Promise<void>) => Promise<void>;
1318
+ /**
1319
+ * Bounded-writer-session timeouts (Fenced Frontier B1, D4) applied to every fleet node's pinned
1320
+ * Postgres connection — the single `NodePgClient` `prepareFleetNode` builds is the writer-capable one
1321
+ * (a sync node's same connection becomes the writer's on promotion via `pgStore.setWritable()`), so
1322
+ * it is bounded from boot. `idle_in_transaction=5s` kills a transaction a wedged writer leaves open
1323
+ * mid-commit (releasing the row lock a fencer's `evictExpired` needs); `statement=10s` caps any single
1324
+ * runaway statement. A NON-fleet single-node `serve`/`dev` (`makeStore` in `packages/cli`) constructs
1325
+ * `NodePgClient` WITHOUT this option and stays unbounded — see that call site.
1326
+ */
1327
+ declare const FLEET_WRITER_SESSION_TIMEOUTS: {
1328
+ idleInTransactionMs: number;
1329
+ statementMs: number;
1330
+ };
1331
+ declare function fleetApplicationName(advertiseUrl: string): string;
1332
+ /**
1333
+ * Whether this deployment runs in MULTI-WRITER mode (`HELIPOD_FLEET_MULTI_WRITER`), read in ONE
1334
+ * place so `prepareFleetNode` (which decides the runtime store/queryStore wiring) and
1335
+ * `startFleetNode` (which decides the tailer/promotion wiring) can never disagree about a node's
1336
+ * shape. Multi-writer IS the hybrid regime (Fleet B3): every writer-ish node keeps a local replica
1337
+ * and serves queries from it while committing to the primary. Off (the production default) → the
1338
+ * shipped single-writer topology: the sole writer holds every shard and reads the primary, additional
1339
+ * nodes are pure read-replica sync nodes — no hybrid machinery anywhere (byte-identical to B2b).
1340
+ */
1341
+ declare function fleetMultiWriterEnabled(): boolean;
1342
+ /**
1343
+ * Whether group commit (Fleet B4) is enabled (`HELIPOD_GROUP_COMMIT`), read the SAME way as
1344
+ * `fleetMultiWriterEnabled` above — one place, so every `runtimeOptions` construction site in
1345
+ * `prepareFleetNode` reads the identical value. Threaded straight into `createEmbeddedRuntime`
1346
+ * (`EmbeddedRuntimeOptions.groupCommit`), which routes every shard's commits through the two-buffer
1347
+ * stage-then-flush committer loop. Default OFF at Fleet B4/T4 — T5 owns flipping the production
1348
+ * default once the throughput win is E2E-proven; unset/anything else → the byte-identical
1349
+ * single-commit path every shard already runs.
1350
+ */
1351
+ declare function groupCommitEnabled(): boolean;
1352
+ interface FleetHandles {
1353
+ role(): "sync" | "writer";
1354
+ /** The current writer's URL — the proxy target for public httpActions handled on a sync node. */
1355
+ writerUrl(): Promise<string>;
1356
+ /** Register a callback fired once, when this node is promoted from sync to writer. */
1357
+ onPromoted(cb: () => void): void;
1358
+ /** The current frontier-lag reading (D5 health observability): `min(frontier_ts)` across shards,
1359
+ * how long it's been stuck (ms), and the pinning shard. Null before the first frontier beat, or if
1360
+ * no shard rows exist yet. */
1361
+ frontierStats(): FrontierStats | null;
1362
+ /**
1363
+ * Group-commit counters (Fleet B4, T4 health observability): `EmbeddedRuntime.groupCommitStats()`'s
1364
+ * aggregate reading plus a derived `flushesPerSec` — a rolling delta between successive calls
1365
+ * (`(flushCount now - flushCount at the prior call) / elapsed seconds`), the simplest honest
1366
+ * throughput signal without a dedicated timer. The first call after boot (or after any gap) has no
1367
+ * prior sample, so it reports 0 for that one call. Structurally all-zero when `HELIPOD_GROUP_
1368
+ * COMMIT` is off (the underlying counters are never touched on the single-commit path) — never
1369
+ * null, unlike `frontierStats` (no "before the first beat" gating applies here).
1370
+ */
1371
+ groupCommitStats(): {
1372
+ lastBatchSize: number;
1373
+ maxBatchSize: number;
1374
+ flushCount: number;
1375
+ flushesPerSec: number;
1376
+ };
1377
+ /** Per-shard ownership (B2b, D1): does THIS node currently hold `shardId`'s write lease? Backs
1378
+ * the `/_fleet/run` single-hop guard (`packages/cli`'s `http-handler.ts`) — delegates straight to
1379
+ * `WriteForwarder.isLocalWriter`, the same live held-set view the executor's own per-shard
1380
+ * router and `relinquish()` consult. */
1381
+ isLocalWriter(shardId: ShardId): boolean;
1382
+ /**
1383
+ * Effectively-once forwarding (Fleet B3, D3): read back `key`'s `fleet_idempotency` row for a
1384
+ * replay decision. Delegates to `LeaseManager.lookupIdempotency` — the SAME control table the
1385
+ * commit guard (`installCommitGuard`) writes into, on the SAME `client`, so a lookup immediately
1386
+ * after a catch always sees a just-committed sibling's row. `packages/cli`'s `/_fleet/run`
1387
+ * handler calls this BOTH before running (SELECT-first) and after catching a unique_violation on
1388
+ * this table (the concurrent-duplicate race's loser re-selecting the winner's row).
1389
+ */
1390
+ idempotencyLookup(key: string): Promise<IdempotencyReplay | null>;
1391
+ /** Best-effort post-run recording of a forwarded mutation's return value — see
1392
+ * `LeaseManager.recordIdempotencyValue`. */
1393
+ idempotencyRecordValue(key: string, value: JSONValue): Promise<void>;
1394
+ stop(): Promise<void>;
1395
+ }
1396
+ /** The createEmbeddedRuntime option deltas the caller threads through `bootProject`. `store` is the
1397
+ * node's RUNTIME (WRITE) store: the writable Postgres store for a writer, the `SwitchableDocStore`
1398
+ * (over the local replica, until promotion swaps in the Postgres store) for a single-writer sync
1399
+ * node, and — for a HYBRID (multi-writer) node — ALWAYS the Postgres store (mutations commit to the
1400
+ * primary), paired with `queryStore` for the replica-backed read path. */
1401
+ interface FleetRuntimeOptions {
1402
+ store: DocStore;
1403
+ writeRouter: WriteRouter;
1404
+ deferDrivers: boolean;
1405
+ fanoutAdapter?: EmbeddedWriteFanoutAdapter;
1406
+ /** Number of shards this node's runtime runs (B2a). Threaded to `createEmbeddedRuntime`, which
1407
+ * builds ONE `ShardedTransactor` (N per-shard mutexes) over the pooled store instead of the
1408
+ * single-shard `SingleWriterTransactor` when >1 — so cross-shard commits run in parallel. */
1409
+ numShards: number;
1410
+ /**
1411
+ * Fleet B3 hybrid nodes (D1) — the replica-backed QUERY store (the `SwitchableDocStore` over the
1412
+ * local replica). Set ONLY on a hybrid (multi-writer) node: `store` (above) is the primary, where
1413
+ * mutations commit, while queries/subscriptions read this replica store. `createEmbeddedRuntime`
1414
+ * builds a separate query-path transactor + QueryRuntime over it, and routes `observeTimestamp`
1415
+ * (fed by this node's ReplicaTailer post-apply) to the query oracle so a query snapshot never
1416
+ * exceeds the replica watermark. Absent → every query runs against `store` (single-writer/sync/
1417
+ * non-fleet), byte-identical to before B3.
1418
+ */
1419
+ queryStore?: DocStore;
1420
+ /**
1421
+ * Receipted Outbox (verdict §(c) placement) — the AUTHORITATIVE receipts store the Connect
1422
+ * handshake classifies/prunes against. Set to the PRIMARY (`pgStore`) on a SYNC node so the
1423
+ * handshake never reads the local replica (which carries no `client_mutations`/`client_floors`
1424
+ * receipts → spurious `known:false` resets / false kill-after-commit failures). Threaded straight
1425
+ * into `createEmbeddedRuntime`. Absent → the runtime uses `store` (writer boots, where `store` IS
1426
+ * the primary; and every non-fleet deployment). See `EmbeddedRuntimeOptions.receiptsStore`.
1427
+ */
1428
+ receiptsStore?: DocStore;
1429
+ /**
1430
+ * Fleet B3 hybrid RYOW (D2) — awaited in the runtime's serial fan-out `drain()` before a local
1431
+ * commit's subscription re-runs, so those re-runs (reading the replica) don't observe the commit's
1432
+ * absence on a replica that hasn't applied it yet. Wired to `forwarder.waitForReplica`. Set only on
1433
+ * a hybrid node. Absent → the drain is byte-identical to before B3.
1434
+ */
1435
+ beforeNotify?: (commitTs: bigint) => Promise<void>;
1436
+ /**
1437
+ * Fleet B4 (T4): group commit — resolved once via `groupCommitEnabled()` (above) and threaded
1438
+ * uniformly onto every boot shape (writer/sync × single-writer/hybrid). Unset/false →
1439
+ * `createEmbeddedRuntime` builds the byte-identical single-commit path, as shipped.
1440
+ */
1441
+ groupCommit?: boolean;
1442
+ /**
1443
+ * Triggers D1 — the stable-prefix accessor for `DriverContext.readLog`. In a fleet, N per-shard
1444
+ * commit connections land timestamps out of order, so the log tail is gap-free ONLY below
1445
+ * `min(shard_leases.frontier_ts)` (the fenced fleet frontier). Wired here to exactly that read (the
1446
+ * same `readAllFrontiers`-min the frontier-lag monitor uses), so `readLog` never surfaces — nor
1447
+ * skips — a change above an in-flight gap. Threaded onto every boot shape; consulted only when a
1448
+ * driver runs (post-promotion on a sync node), by which point `store` points at the primary.
1449
+ */
1450
+ stablePrefix?: () => Promise<bigint | null>;
1451
+ /**
1452
+ * Receipted Outbox — receipts-guard ownership handoff. Always `true` for a fleet node: fleet owns
1453
+ * installing the `clientReceiptsGuard()` exactly-once barrier on the CONCRETE Postgres store in
1454
+ * `armWriter` (before the epoch fence, release-on-re-arm), NOT on `store` — which for a sync node is
1455
+ * a `SwitchableDocStore` the runtime would register the guard on, only for it to silently vanish on
1456
+ * the promotion swapTo (leaving a promoted single-writer node's client mutations un-receipted → a
1457
+ * resent (clientId, seq) re-executes). Threaded to `createEmbeddedRuntime` so its `create()` skips
1458
+ * its own registration. See `EmbeddedRuntimeOptions.externalReceiptsGuard` and `armWriter`.
1459
+ */
1460
+ externalReceiptsGuard?: boolean;
1461
+ }
1462
+ interface FleetPrep {
1463
+ client: NodePgClient;
1464
+ /** The Postgres store. Writer: the runtime store (writable). Sync: the tail source + promotion
1465
+ * swap target (read-only until promoted) — NOT the runtime store (`runtimeOptions.store` is the
1466
+ * `SwitchableDocStore` over the local replica for a sync node). */
1467
+ pgStore: PostgresDocStore;
1468
+ /** The local file-backed replica the runtime reads queries through, and the switchable wrapper it
1469
+ * points at. Present for a single-writer SYNC boot (runtime store = the switchable) AND for a
1470
+ * HYBRID (multi-writer) boot of EITHER role (the switchable is `runtimeOptions.queryStore`, the
1471
+ * read path beside the primary write store). Absent only for a single-writer WRITER boot (no
1472
+ * replica — it reads the primary). */
1473
+ replica?: SqliteDocStore;
1474
+ switchable?: SwitchableDocStore;
1475
+ /** The on-disk path of `replica`, threaded through to `startFleetNode` so its C7
1476
+ * `reconcileReplicaIdentity` call (deferred there — see that function's doc comment) can rebuild
1477
+ * the file in place if needed. Present whenever `replica` is (sync boot, or any hybrid boot). */
1478
+ replicaPath?: string;
1479
+ lease: LeaseManager;
1480
+ forwarder: WriteForwarder;
1481
+ role: "sync" | "writer";
1482
+ /** The shard count decided at boot — threaded through to `startFleetNode` (acquire-all loop,
1483
+ * all-rows seed, per-shard commit guard, idle closer) and the tailer's `count(*) < N` ready gate. */
1484
+ numShards: number;
1485
+ runtimeOptions: FleetRuntimeOptions;
1486
+ }
1487
+ /**
1488
+ * Decide writer-vs-sync BEFORE the runtime is constructed: one `tryAcquire()` after the lease
1489
+ * table exists. This must run before `bootProject` because the acquire result determines the
1490
+ * store's writability, the fan-out adapter, whether drivers are deferred, and the forwarder's
1491
+ * starting role — all of which are `createEmbeddedRuntime` inputs.
1492
+ */
1493
+ declare function prepareFleetNode(deps: {
1494
+ databaseUrl: string;
1495
+ advertiseUrl: string;
1496
+ adminKey: string;
1497
+ /** The serve data dir — the sync node's local replica is created at `<dataDir>/fleet-replica.db`. */
1498
+ dataDir: string;
1499
+ /** Lease TTL in ms — the single knob the whole failover clock scales from (see `fleetProbeMs`/
1500
+ * `fleetAcquireRetryMs`). Threaded from serve's fleet config, which reads
1501
+ * `HELIPOD_FLEET_LEASE_TTL_MS` (ops/test tuning; the wedged-writer E2E uses 4000). Default 15000
1502
+ * reproduces the historical constants exactly. */
1503
+ leaseTtlMs?: number;
1504
+ /** Number of shards this fleet runs (B2a). This task threads it as a plain parameter (default 8);
1505
+ * T5 owns the persist-once/`HELIPOD_FLEET_SHARDS`/mismatch-fail-fast story. Drives the commit
1506
+ * pool's connection set, the acquire-all loop, and the runtime's `ShardedTransactor`. */
1507
+ numShards?: number;
1508
+ }): Promise<FleetPrep>;
1509
+ interface StartFleetNodeDeps {
1510
+ client: NodePgClient;
1511
+ /** The Postgres store — the tail source + promotion swap target (sync), or the live runtime store
1512
+ * (writer, in which case `startFleetNode` just returns handles). */
1513
+ pgStore: PostgresDocStore;
1514
+ runtime: EmbeddedRuntime;
1515
+ lease: LeaseManager;
1516
+ forwarder: WriteForwarder;
1517
+ /** Sync only: the local replica the tailer applies onto, and the switchable the runtime reads
1518
+ * through (swapped to the Postgres store on promotion). Absent/ignored for a writer boot. */
1519
+ replica?: SqliteDocStore;
1520
+ switchable?: SwitchableDocStore;
1521
+ /** Sync only: `replica`'s on-disk path — needed here (not just in `prepareFleetNode`) because the
1522
+ * C7 `reconcileReplicaIdentity` check now runs in THIS function, right before `tailer.start()`. */
1523
+ replicaPath?: string;
1524
+ /** Process-exit indirection (C4/C5). Injected so tests observe exits instead of killing the runner;
1525
+ * defaults to `console.error` + `process.exit(1)`. Fires on writer lease loss (the lease monitor)
1526
+ * and on a failed promotion step. */
1527
+ onExit?: (reason: string) => void;
1528
+ /** Number of shards this node runs (B2a) — from `prepareFleetNode`'s decision (`FleetPrep.numShards`).
1529
+ * The writer/promotion arming acquires all N shard leases (slots 1…N-1 beyond the default), seeds
1530
+ * all N frontiers before ready, and drives the idle-shard closer over them. Defaults to 1 so an
1531
+ * older single-shard call site (the B1 lifecycle tests that drive `startFleetNode` directly) is
1532
+ * byte-identical — one shard, one lease, one frontier. */
1533
+ numShards?: number;
1534
+ }
1535
+ /** The seam the C5 promotion wrap touches — narrow structural spies over the promotion sequence,
1536
+ * monitor start, promoted-callback fan-out, and the exit indirection. */
1537
+ interface PromotionRunDeps {
1538
+ /** The CRITICAL PROMOTION ORDER (`promoteFleetNode`), as a thunk. */
1539
+ promote: () => Promise<void>;
1540
+ /** Start the writer lease monitor — this node is now the writer and must self-exit on lease loss. */
1541
+ startMonitor: () => void;
1542
+ /** Notify the http layer to drop its writer proxy. */
1543
+ firePromoted: () => void;
1544
+ /** Exit indirection — any promotion-step failure routes here (a half-promoted node must not linger). */
1545
+ onExit: (reason: string) => void;
1546
+ }
1547
+ /**
1548
+ * C5 promotion error policy: wrap the promotion sequence so ANY step failure is caught, logged, and
1549
+ * turned into an exit rather than left as a silent unhandled rejection with the node stuck
1550
+ * half-promoted (writable pg store, un-swapped runtime, no drivers). On success the lease monitor is
1551
+ * started (writer self-exit is now armed) and the http layer is told to drop its proxy. Fires exit at
1552
+ * most once by construction — it's invoked once per node (guarded by the caller's `promoting` flag).
1553
+ */
1554
+ declare function runPromotion(deps: PromotionRunDeps): Promise<void>;
1555
+ /**
1556
+ * Install the epoch-fenced commit guard (Fenced Frontier B1, D3) on `pgStore`. Runs inside every
1557
+ * `commitWrite` transaction, after the row inserts, before COMMIT — the SAME row as the lease
1558
+ * (`shard_leases`) advances the durable-commit frontier chain (`prev_ts := frontier_ts, frontier_ts
1559
+ * := commitTs`) predicated on THIS node's epoch still being current, so frontier publication,
1560
+ * fencing, and the lease are one row with zero extra round-trips. `lease.currentEpoch()` is read
1561
+ * LIVE on every commit (not a snapshot taken at install time) so a later re-promotion's epoch bump
1562
+ * is honored automatically. Zero rows updated (a stale/superseded epoch) throws `FencedError`, which
1563
+ * aborts the whole transaction — AND calls `onFenced(shardId, reason)` before throwing so the caller
1564
+ * can react (B2b, D2: `startFleetNode` wires this straight to `relinquish(shardId, reason)` — drop
1565
+ * JUST this shard, keep serving everything else — never to a whole-node exit; see `relinquish` below
1566
+ * for why `LeaseMonitor.fenced()`/`onExit` are NOT reached from here anymore). Called at writer boot
1567
+ * AND on promotion success — see the two call sites below.
1568
+ *
1569
+ * Returns the `addCommitGuard` unregister handle (Receipted Outbox decision 2 — the guard
1570
+ * slot→chain migration). `armWriter` re-arms on EVERY promotion; a caller that calls
1571
+ * `installCommitGuard` again WITHOUT first calling the PRIOR returned unregister function stacks a
1572
+ * duplicate epoch-fence guard on the chain — duplicate frontier bumps + duplicate
1573
+ * `fleet_idempotency` INSERTs at the same ts, so every subsequent forwarded commit self-PK-collides
1574
+ * and aborts. See `armWriter`'s `unregisterCommitGuard` handling below for the required pattern.
1575
+ */
1576
+ declare function installCommitGuard(pgStore: PostgresDocStore, lease: LeaseManager, onFenced: (shardId: ShardId, reason: string) => void): () => void;
1577
+ /** The narrow seams `relinquish` needs — deliberately structural (not the concrete `NodePgClient`) so
1578
+ * it's unit-testable with a stub, and reusable by T4's balancer without depending on `startFleetNode`'s
1579
+ * internal closures. */
1580
+ interface RelinquishDeps {
1581
+ lease: LeaseManager;
1582
+ /** `releaseShardLock` is consulted defensively (`?.`) — absent on a poolless/PGlite client, though
1583
+ * every real B2b fleet node runs in pool mode and always has it. */
1584
+ client: {
1585
+ releaseShardLock?: (slot: number) => Promise<void>;
1586
+ };
1587
+ /** Ordered shard list (index = slot) — how `shardId`'s per-slot advisory lock is found for
1588
+ * `releaseShardLock`. The same list `prepareFleetNode`/`startFleetNode` derive via `shardIdList`. */
1589
+ shards: readonly ShardId[];
1590
+ /** Structured-log seam, defaults to `console.error`. Tests inject a spy. */
1591
+ log?: (msg: string) => void;
1592
+ /**
1593
+ * Invoked when the DEFAULT shard is relinquished (B2b, D5 — "drivers follow the default shard"):
1594
+ * `startFleetNode` wires this to `runtime.stopDriversOnly()`, so the moment this node loses the
1595
+ * default ring the scheduler/workflow/cron/reaper drivers go quiet (a different node now owns that
1596
+ * ring). Optional — the balancer/relinquish unit tests that don't care about drivers omit it, in
1597
+ * which case relinquishing the default shard just drops the shard with no driver side effect.
1598
+ */
1599
+ onDefaultRelinquished?: () => void;
1600
+ }
1601
+ /**
1602
+ * Per-shard relinquish dispatcher (Fenced Frontier B2b, D2): the reduction a `FencedError` on shard
1603
+ * `s` gets now — "drop `s`, keep serving everything else" — instead of B1/B2a's "kill the node".
1604
+ * Routed to from the commit guard's `onFenced` (a fence discovered mid-commit — that commit itself
1605
+ * still aborts and propagates `FencedError` to its caller, OCC-retryable; relinquish is the SIDE
1606
+ * EFFECT, not a swallow), the batched heartbeat's `fencedShardIds` (a fence discovered on the probe
1607
+ * beat), and a per-shard commit-connection loss.
1608
+ *
1609
+ * Idempotent per shard: `lease.currentEpoch(shardId) === null` means this shard is already forgotten
1610
+ * (a prior relinquish call, or a shard this node never held) — a silent no-op, so callers never need
1611
+ * to track "have I already relinquished this" themselves.
1612
+ *
1613
+ * 1. `lease.forgetShard(shardId)` — drops the held-epoch entry, so the commit guard's "no acquired
1614
+ * epoch" branch fences any straggler commit on `s` cleanly, and `heartbeatAll`/`closeIdleFrontiers`
1615
+ * stop touching it.
1616
+ * 2. Release `s`'s per-slot advisory lock (`PgClient.releaseShardLock`) — UNLESS `opts.connectionLost`:
1617
+ * a dead commit connection already released its session-scoped lock the instant the backend went
1618
+ * away, so there is nothing left to release (and no live connection to run the unlock query on).
1619
+ * 3. Log one structured line. Relinquishing the DEFAULT shard additionally WARNS that drivers keep
1620
+ * running (scheduler/workflow/cron/reaper stay armed on this node) until a later task wires
1621
+ * `stopDriversOnly` (D5) — driver stop/start is deliberately NOT built here.
1622
+ *
1623
+ * Deliberately does NOT touch `LeaseMonitor` and never calls `onExit` — the whole point of B2b is that
1624
+ * a per-shard fence must no longer escalate to a whole-node exit. `LeaseMonitor` stays reserved for
1625
+ * pinned-connection loss and probe exhaustion (definitive WHOLE-NODE loss); see `startFleetNode`.
1626
+ */
1627
+ declare function relinquish(deps: RelinquishDeps, shardId: ShardId, reason: string, opts?: {
1628
+ connectionLost?: boolean;
1629
+ }): void;
1630
+ /**
1631
+ * Acquire one non-default shard's lease as the writer (B2a acquire-all). `tryAcquire(shardId, slot)`
1632
+ * epoch-bumps that shard's row (fencing any prior holder) and takes its per-shard advisory lock. If
1633
+ * the lock is still held by a wedged prior holder AND that shard's lease has expired, fence + evict +
1634
+ * terminate its backend (the same fencing-first eviction the default-shard acquire loop uses) and
1635
+ * retry. Bounded by a generous deadline (~10 lease TTLs): the common failover frees every slot at once
1636
+ * when the default-shard loop terminates the whole wedged node, so this normally succeeds on the first
1637
+ * try (fresh boot) or right after one eviction. Exceeding the deadline throws → promotion/boot fails →
1638
+ * the node exits and rejoins fresh, never silently running as a partial-shard writer.
1639
+ */
1640
+ declare function acquireShardAsWriter(lease: LeaseManager, shardId: ShardId, slot: number, retryMs: number, seedFrontierFromDocuments?: boolean): Promise<void>;
1641
+ /**
1642
+ * Pure throughput derivation for group-commit health (Fleet B4, T4): a rolling delta of
1643
+ * `flushCount` between two `groupCommitStats()` reads, over the elapsed wall-clock time — the
1644
+ * simplest honest `flushesPerSec` signal without a dedicated timer/interval. `prevReadMs === null`
1645
+ * (no prior sample — the first read after boot, or after any gap) and a non-positive elapsed time
1646
+ * (a clock that hasn't advanced, or moved backward) both report 0 rather than a division artifact.
1647
+ * Extracted as a standalone pure function so it's unit-testable without booting a fleet node.
1648
+ */
1649
+ declare function deriveFlushesPerSec(prevReadMs: number | null, prevFlushCount: number, nowMs: number, nowFlushCount: number): number;
1650
+ /**
1651
+ * Wire the running fleet node. A writer node is already fully live (promoted in `prepareFleetNode`,
1652
+ * drivers started at `create()`) — it just gets handles. A sync node starts the replica tailer (its
1653
+ * `start()` catch-up is the node's ready gate) and the lease acquire loop, and promotes on acquire
1654
+ * via `promoteFleetNode`.
1655
+ */
1656
+ declare function startFleetNode(deps: StartFleetNodeDeps): Promise<FleetHandles>;
1657
+
1658
+ /**
1659
+ * `StablePrefixTs` (Fenced Frontier B1, D6) — a compile-time brand over `bigint` marking a
1660
+ * timestamp as a member of the FENCED FRONTIER: the stable prefix of the primary's MVCC log that
1661
+ * `ReplicaTailer` is allowed to pull from, per D5. A raw `bigint` (e.g. `primary.maxTimestamp()`,
1662
+ * which reports the log's live high-water mark INCLUDING any commit that raced past the last fence
1663
+ * check) is NOT a `StablePrefixTs` — only a value read straight off `shard_leases.frontier_ts`, or
1664
+ * a watermark previously derived from one, is.
1665
+ *
1666
+ * This is deliberately narrow: it brands ONLY `ReplicaTailer`'s internal pull target (`F`) and its
1667
+ * own watermark (`wm`), which must never advance past `F` (D5's "wm never exceeds F" invariant).
1668
+ * It does NOT brand `waitFor`'s public `ts` parameter — that's a caller-supplied `commitTs` (a raw
1669
+ * log position by nature, e.g. forwarded from `WriteForwarder`'s `/_fleet/run` response), and RYOW
1670
+ * correctness only needs `wm >= ts`, which is a plain bigint comparison regardless of which side is
1671
+ * branded. Branding that parameter too would just force every call site to launder a raw commitTs
1672
+ * through the constructor for no safety benefit.
1673
+ *
1674
+ * The sole constructor, `stablePrefixFromFrontier`, is intentionally the ONLY way to produce one —
1675
+ * assigning a raw `bigint` to a `StablePrefixTs`-typed binding without going through it is a
1676
+ * compile error (see `replica-tailer.test.ts`'s `@ts-expect-error` cases). At runtime this is a
1677
+ * plain `bigint`; the brand (`__stablePrefix`) exists purely in the type system and costs nothing.
1678
+ */
1679
+ type StablePrefixTs = bigint & {
1680
+ readonly __stablePrefix: unique symbol;
1681
+ };
1682
+ /**
1683
+ * The sole constructor. Call this ONLY where a value is actually a frontier — a fresh read of
1684
+ * `shard_leases.frontier_ts` (via the tailer's `CommitChannelClient`), or a watermark already
1685
+ * derived from a prior `StablePrefixTs` (e.g. the replica's own persisted `maxTimestamp()` on
1686
+ * restart, which IS this node's last-applied frontier). Do not use this to launder an arbitrary
1687
+ * `bigint` (e.g. `primary.maxTimestamp()`, or a document's raw `ts`) into the brand — that value is
1688
+ * NOT guaranteed to be a durably-fenced, dense prefix and defeats the whole point of the type.
1689
+ */
1690
+ declare function stablePrefixFromFrontier(raw: bigint): StablePrefixTs;
1691
+
1692
+ /**
1693
+ * The offline fleet reshard tool (B5 Part 1, Task 9.1): change a STOPPED Postgres fleet's shard
1694
+ * count N→M by updating the persist-once `fleet:numShards` global + creating/deleting
1695
+ * `shard_leases` lane rows — atomically, in one transaction — and NOTHING else. Because a shard is
1696
+ * a logical commit-serialization lane over one shared store (`documents.shard_id` is decorative;
1697
+ * routing always recomputes `shardIdForKeyValue(key, numShards)` fresh), resharding moves no rows:
1698
+ * only which lane FUTURE commits for a key serialize through changes. See
1699
+ * `docs/dev/research/write-sharding/b5-reshard-and-object-storage.md` (Part 1) and
1700
+ * `docs/superpowers/plans/2026-02-20-fleet-reshard-b5.md` for the full design.
1701
+ *
1702
+ * Safety rests on the STOPPED-FLEET precondition (no `--force`): a live node makes the old-N and
1703
+ * new-M epochs potentially overlap in time, which is out of scope (needs an epoch-fence vocabulary
1704
+ * this tool doesn't have — see the plan's "online reshard" scope boundary). A quiesced fleet makes
1705
+ * the two epochs strictly non-overlapping, so changing a key's lane between them is as safe as any
1706
+ * config change to a stopped system.
1707
+ *
1708
+ * `F = min(frontier_ts)` is monotone across a reshard: new lanes are seeded at `MAX(ts)` (never
1709
+ * below the true high-water mark — the same F1 seed `LeaseManager.setup()`/`tryAcquire()` use, see
1710
+ * `frontierSeedExpr` in `./lease.ts`), retained lanes are GREATEST-bumped up to that same floor
1711
+ * (never lowered — a documented, safe deviation from "retained rows untouched": raise-only can
1712
+ * never regress F, and it proactively heals a lane whose frontier legitimately lags `MAX(ts)` after
1713
+ * a crash stop — no `selfFence` ran, so nothing bumped it since the last idle-closer beat — the
1714
+ * same healing a node boot's `seedFrontierAll` would eventually do), and deleting a lane can only
1715
+ * raise or hold `min(frontier_ts)`, never lower it. Net effect: EVERY post-reshard lane is
1716
+ * `>= MAX(ts)`, making the post-verify's F1 check a true post-condition rather than a best-effort one.
1717
+ */
1718
+
1719
+ /** The `persistence_globals` key the resolved shard count is stamped under — byte-identical to
1720
+ * `packages/cli/src/boot.ts`'s `NUM_SHARDS_GLOBAL_KEY` (kept as an independent literal here: the
1721
+ * ee `@helipod/fleet` package has no dependency on core `packages/cli`, mirroring the same
1722
+ * independent-literal choice `boot.ts`'s own doc comment makes for `DEFAULT_NUM_SHARDS`). */
1723
+ declare const NUM_SHARDS_GLOBAL_KEY = "fleet:numShards";
1724
+ /** Thrown when `reshardFleet` is asked to run against a fleet with any live `fleet_nodes` or
1725
+ * `shard_leases.writer_url` row — the hard stopped-fleet precondition, no `--force` escape hatch.
1726
+ * `liveUrls` names every live node/writer URL found, so the operator error is actionable. */
1727
+ declare class ReshardFleetLiveError extends Error {
1728
+ readonly liveUrls: string[];
1729
+ constructor(liveUrls: string[]);
1730
+ }
1731
+ /** Thrown when the post-commit verification pass finds the reshard did not land as expected —
1732
+ * should be impossible (a guard against a logic bug, not an expected operational outcome). */
1733
+ declare class ReshardVerificationError extends Error {
1734
+ constructor(message: string);
1735
+ }
1736
+ /** Thrown when `reshardFleet` is pointed at a Postgres database that has never run a fleet on it —
1737
+ * no `shard_leases`/`fleet_nodes` tables. Without this pre-gate, the live-fleet gate query below
1738
+ * would throw a raw `relation "fleet_nodes" does not exist`, which reads like an internal SQL bug
1739
+ * rather than the operator-actionable "you haven't booted a fleet here yet" that it actually is. */
1740
+ declare class ReshardNotAFleetError extends Error {
1741
+ constructor();
1742
+ }
1743
+ interface ReshardResult {
1744
+ previousShards: number;
1745
+ newShards: number;
1746
+ /** Shard ids newly created (present in the target set, absent from the prior lane set). */
1747
+ created: string[];
1748
+ /** Shard ids removed (present in the prior lane set, absent from the target set). Never includes
1749
+ * `"default"` — it is a member of `shardIdList(M)` for every `M >= 1` and so can never be a
1750
+ * target-set omission. */
1751
+ deleted: string[];
1752
+ /** `min(frontier_ts)` across every post-reshard lane, as a decimal string (frontier_ts is a
1753
+ * Postgres BIGINT — carried as a string rather than risk a JS-number precision loss). */
1754
+ frontierFloor: string;
1755
+ }
1756
+ /**
1757
+ * Change a STOPPED Postgres fleet's shard count from whatever it currently is to `opts.targetShards`.
1758
+ * See the module doc comment for the full safety argument. Steps (verbatim to the plan, plus the
1759
+ * not-a-fleet pre-gate):
1760
+ * 1. validate `targetShards >= 1`
1761
+ * 2. probe that this is actually a fleet store (`shard_leases`/`fleet_nodes` exist) —
1762
+ * `ReshardNotAFleetError` otherwise, before the next step's raw SQL can throw an opaque
1763
+ * "relation does not exist"
1764
+ * 3. refuse if any `fleet_nodes`/`shard_leases` row is live (`ReshardFleetLiveError`)
1765
+ * 4. read the current lane set + the current `fleet:numShards` global
1766
+ * 5. one transaction: upsert the global, INSERT new lanes (seeded at `MAX(ts)`), DELETE surplus
1767
+ * lanes, then GREATEST-bump every remaining lane's frontier up to that same `MAX(ts)` floor
1768
+ * (heals a retained lane that crash-stopped below it — see the module doc comment)
1769
+ * 6. post-verify (count/set/frontier-floor/global) — `ReshardVerificationError` on any mismatch
1770
+ */
1771
+ declare function reshardFleet(client: PgClient, opts: {
1772
+ targetShards: number;
1773
+ }): Promise<ReshardResult>;
1774
+
1775
+ declare const FLEET_VERSION = "0.0.0";
1776
+
1777
+ export { type AppliedInvalidation, type BalancerLease, type CommitChannelClient, DEFAULT_NUM_SHARDS, DensityViolationError, FLEET_VERSION, FLEET_WRITER_SESSION_TIMEOUTS, FencedError, type FleetHandles, type FleetPrep, type FleetRuntimeOptions, FrontierMonitor, type FrontierStats, IDEMPOTENCY_VALUE_CAP_BYTES, type IdempotencyReplay, LeaseManager, type LeaseManagerOptions, LeaseMonitor, type LeaseMonitorDeps, type LeaseRow, type LeaseState, NUM_SHARDS_GLOBAL_KEY, NotifyingFanoutAdapter, type PromotionRunDeps, REPLICA_DB_FILENAME, type RelinquishDeps, ReplicaTailer, type ReplicaTailerOptions, type ReplicaWaiter, ReshardFleetLiveError, ReshardNotAFleetError, type ReshardResult, ReshardVerificationError, ShardLeaseBalancer, type ShardLeaseBalancerDeps, type StablePrefixTs, type StartFleetNodeDeps, SwitchableDocStore, WriteForwarder, type WriteForwarderOptions, acquireShardAsWriter, createAsyncChain, deriveFlushesPerSec, fleetApplicationName, fleetMultiWriterEnabled, groupCommitEnabled, installCommitGuard, prepareFleetNode, relinquish, rendezvousOwner, rendezvousWeight, reshardFleet, runPromotion, stablePrefixFromFrontier, startFleetNode };