@helipod/fleet 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +11 -0
- package/dist/index.d.ts +1777 -0
- package/dist/index.js +2514 -0
- package/dist/index.js.map +1 -0
- package/package.json +60 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,1777 @@
|
|
|
1
|
+
import { PgClient, PgQuerier, PostgresDocStore, NodePgClient } from '@helipod/docstore-postgres';
|
|
2
|
+
import { ShardId } from '@helipod/id-codec';
|
|
3
|
+
export { docKeyToPointRange, keyToPointRange } from '@helipod/id-codec';
|
|
4
|
+
import { JSONValue } from '@helipod/values';
|
|
5
|
+
import { EmbeddedWriteFanoutAdapter, EmbeddedWriteFanoutPayload, FanoutListener, WriteRouter, ClientReplay, EmbeddedRuntime } from '@helipod/runtime-embedded';
|
|
6
|
+
import { InternalDocumentId, DocStore, SchemaSetupOptions, DocumentLogEntry, IndexWrite, ConflictStrategy, ShardId as ShardId$1, CommitUnit, CommitGuardUnit, LatestDocument, Interval, Order, TimestampRange, PrevRevQuery, ClientVerdictRecord, ClientVerdictWrite } from '@helipod/docstore';
|
|
7
|
+
import { SqliteDocStore } from '@helipod/docstore-sqlite';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Non-blocking per-shard commit-mutex seam (see `Transactor.tryRunExclusiveOnShard` /
|
|
11
|
+
* `EmbeddedRuntime.tryRunExclusiveOnShard`): run `fn` under shard `shardId`'s commit mutex if that
|
|
12
|
+
* shard is IDLE right now (returns `true`), else skip without running it (returns `false`). "Idle"
|
|
13
|
+
* means BOTH the commit mutex is free AND no group-commit batch is staged/flushing — Fleet B4's group
|
|
14
|
+
* commit runs the flush I/O OFF the mutex, so mutex-freedom alone no longer implies "nothing in
|
|
15
|
+
* flight". `closeIdleFrontiers` takes this so each idle shard's frontier bump is mutually exclusive
|
|
16
|
+
* with that shard's own commits AND with any mid-flush batch — the fix for the frontier-inversion race
|
|
17
|
+
* (an idle-closer publishing a frontier ahead of an in-flight commit's or batch's not-yet-landed
|
|
18
|
+
* rows). The writer owns both the closer and every shard's commits, so this single-process check is
|
|
19
|
+
* airtight; cross-node closing of a held shard is impossible by design (only the lease holder closes
|
|
20
|
+
* its own held shards).
|
|
21
|
+
*/
|
|
22
|
+
type TryRunExclusiveOnShard = (shardId: ShardId, fn: () => Promise<void>) => Promise<boolean>;
|
|
23
|
+
/** The current fleet writer lease: which epoch is live and which node holds it. */
|
|
24
|
+
interface LeaseState {
|
|
25
|
+
epoch: bigint;
|
|
26
|
+
writerUrl: string;
|
|
27
|
+
/** The row's fenced-frontier high-water mark at acquisition time (Fleet B3). On an `ON CONFLICT`
|
|
28
|
+
* re-acquire the upsert leaves `frontier_ts` untouched, so this returns the value an INTERIM owner
|
|
29
|
+
* advanced it to while this node didn't hold the shard — the exact floor the caller feeds to
|
|
30
|
+
* `runtime.observeWriteTimestamp` so a re-acquired shard's write snapshot never sits stale. */
|
|
31
|
+
frontierTs: bigint;
|
|
32
|
+
}
|
|
33
|
+
/** `LeaseManager.read()`'s full row — every `shard_leases` column (Fenced Frontier B1, D2). The
|
|
34
|
+
* extra columns beyond `LeaseState` are consumed by the D4 eviction/D5 tailer-frontier work; this
|
|
35
|
+
* class only writes them (frontier_ts/prev_ts advance via the commit-guard SQL installed in
|
|
36
|
+
* `node.ts`, never through this class directly), so they're returned loosely typed for now. */
|
|
37
|
+
interface LeaseRow extends LeaseState {
|
|
38
|
+
writerAppName: string | null;
|
|
39
|
+
/** Raw Postgres value for the TIMESTAMPTZ column (a `Date` under `NodePgClient`, ISO-ish string
|
|
40
|
+
* under PGlite) — not a `LeaseManager` client's job to interpret; a heartbeat/fence caller only
|
|
41
|
+
* cares about row-count effects, not the wall-clock value. */
|
|
42
|
+
expiresAt: unknown;
|
|
43
|
+
frontierTs: bigint;
|
|
44
|
+
prevTs: bigint;
|
|
45
|
+
}
|
|
46
|
+
interface LeaseManagerOptions {
|
|
47
|
+
/** URL this node advertises as the writer, recorded into shard_leases on acquire. */
|
|
48
|
+
advertiseUrl: string;
|
|
49
|
+
/** This node's Postgres `application_name` (see `fleetApplicationName`), recorded on acquire so
|
|
50
|
+
* a D4 eviction fencer can `pg_terminate_backend` the exact wedged holder's connection. */
|
|
51
|
+
applicationName?: string;
|
|
52
|
+
/** Interval between tryAcquire() attempts inside acquireLoop(). Default 2000ms. */
|
|
53
|
+
retryMs?: number;
|
|
54
|
+
/** How long a fresh acquisition/heartbeat extends `expires_at` by, in ms. Default 15000ms. The
|
|
55
|
+
* whole failover clock scales with this: a wedged writer's lease expires this long after its last
|
|
56
|
+
* heartbeat, so a follower's eviction can't fire before then. Shortened by
|
|
57
|
+
* `HELIPOD_FLEET_LEASE_TTL_MS` for the wedged-writer E2E (and available to operators as tuning);
|
|
58
|
+
* the LeaseMonitor's probe cadence is derived from it so a live writer always renews in time. */
|
|
59
|
+
ttlMs?: number;
|
|
60
|
+
}
|
|
61
|
+
/** Effectively-once forwarding (Fleet B3, D3): the `fleet_idempotency.value_json` cap. A recorded
|
|
62
|
+
* mutation result larger than this is NOT stored — the row keeps `value_json = NULL` and
|
|
63
|
+
* `oversized = true` instead, so a replay reports `valueMissing: true` rather than growing this
|
|
64
|
+
* control table unboundedly on a large mutation return value. The WRITE itself is unaffected
|
|
65
|
+
* either way (this cap only governs the best-effort RESULT-VALUE cache, not the commit). */
|
|
66
|
+
declare const IDEMPOTENCY_VALUE_CAP_BYTES: number;
|
|
67
|
+
/** A `fleet_idempotency` row read back for replay (Fleet B3, D3). `hasValue` distinguishes a
|
|
68
|
+
* genuinely-recorded value (including a mutation that legitimately returned JSON `null`, which is
|
|
69
|
+
* still stored as the TEXT `"null"`) from `value_json` being SQL NULL — the crash-window
|
|
70
|
+
* (commit landed, the post-run value UPDATE never ran) and the oversized-cap cases both leave
|
|
71
|
+
* `value_json` SQL NULL, and both replay as `valueMissing: true` uniformly. */
|
|
72
|
+
interface IdempotencyReplay {
|
|
73
|
+
commitTs: bigint;
|
|
74
|
+
hasValue: boolean;
|
|
75
|
+
value: JSONValue | null;
|
|
76
|
+
oversized: boolean;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Coordinates the single-writer lease across a fleet of nodes sharing one Postgres database.
|
|
80
|
+
* The advisory lock (PgClient.tryAcquireWriterLock) is the FAST-PATH mutual-exclusion primitive;
|
|
81
|
+
* `shard_leases` is the fencing token + discovery row + frontier chain — one row per shard (B1:
|
|
82
|
+
* only `'default'`) so any node (including read replicas forwarding writes) can find the current
|
|
83
|
+
* writer's URL/epoch, and so `PostgresDocStore`'s installed commit guard can verify — inside every
|
|
84
|
+
* commit transaction — that the writer holding the advisory lock is STILL the epoch on file (see
|
|
85
|
+
* `node.ts`'s `installCommitGuard`). `epoch` bumps on every acquisition (D2); `frontier_ts`/
|
|
86
|
+
* `prev_ts` are the durable-commit chain the guard advances (D3) — this class never writes them
|
|
87
|
+
* itself beyond seeding them to 0 on first creation.
|
|
88
|
+
*
|
|
89
|
+
* Liveness: the LeaseMonitor's periodic probe IS `heartbeat()` (see `node.ts`) — one round-trip
|
|
90
|
+
* serves liveness-probe + TTL maintenance + fence verification, per D2. `heartbeat()` returning 0
|
|
91
|
+
* rows means this node's epoch has been superseded (fenced) even though its connection is still
|
|
92
|
+
* alive — a DEFINITIVE loss, distinct from the probe-miss tolerance used for transient blips.
|
|
93
|
+
*/
|
|
94
|
+
declare class LeaseManager {
|
|
95
|
+
private readonly client;
|
|
96
|
+
/** This node's advertised URL, recorded onto `shard_leases`/`fleet_nodes`. Public so the balancer
|
|
97
|
+
* can use it as this node's rendezvous identity (`ShardLeaseBalancer.myUrl`) without threading it
|
|
98
|
+
* separately through `startFleetNode`. */
|
|
99
|
+
readonly advertiseUrl: string;
|
|
100
|
+
private readonly applicationName;
|
|
101
|
+
private readonly retryMs;
|
|
102
|
+
/** The lease TTL in ms this manager stamps onto `expires_at`. Public so the LeaseMonitor can
|
|
103
|
+
* derive its probe cadence from the SAME knob (see `startFleetNode`) — a live writer must renew
|
|
104
|
+
* well within the TTL. */
|
|
105
|
+
readonly ttlMs: number;
|
|
106
|
+
/** `ttlMs` as a whole-ms integer, ready to interpolate into the `interval '<n> milliseconds'` SQL
|
|
107
|
+
* below. Integer-coerced (never a fraction/NaN) so the interpolation can't produce invalid SQL. */
|
|
108
|
+
private readonly ttlMsSql;
|
|
109
|
+
private timer;
|
|
110
|
+
private stopped;
|
|
111
|
+
/** Per-shard epoch map: `shardId → the epoch this node most recently acquired for it`. B1 tracked
|
|
112
|
+
* ONE `lastEpoch` (default shard only); B2a's writer holds N shards, each fenced against its own
|
|
113
|
+
* epoch, so the commit guard/heartbeat/idle-closer look up `currentEpoch(shardId)`. Updated on
|
|
114
|
+
* every successful per-shard `tryAcquire(shardId)` (including re-promotion's epoch bumps) so the
|
|
115
|
+
* guard always fences against the CURRENT epoch, not a boot snapshot. A shard absent from the
|
|
116
|
+
* map is one this node has never acquired. */
|
|
117
|
+
private readonly lastEpochByShard;
|
|
118
|
+
constructor(client: PgClient, opts: LeaseManagerOptions);
|
|
119
|
+
/**
|
|
120
|
+
* Idempotent DDL: creates shard_leases if it doesn't already exist, and — when `shardIds` is
|
|
121
|
+
* non-empty — pre-seeds a row for every shard in the list (Fleet B3, D4: the concurrent-boot
|
|
122
|
+
* count-gate fix, reversing B2a's "no pre-seeding" call).
|
|
123
|
+
*
|
|
124
|
+
* B2a originally left this deliberately EMPTY (no pre-seeded rows): a bare `frontier_ts = 0` row
|
|
125
|
+
* created at DDL time, on a database that already held documents (a pre-`--fleet` upgrade), would
|
|
126
|
+
* be momentarily visible at `frontier_ts = 0` to a concurrently-booting sync node — `count(*) = N`
|
|
127
|
+
* AND `min(frontier_ts) = 0` would fake-report ready with an EMPTY replica (the F1×N hole,
|
|
128
|
+
* recurring ×N). That is why row existence used to be tied to a REAL, frontier-seeded acquisition
|
|
129
|
+
* (`tryAcquire`) rather than to `setup()`.
|
|
130
|
+
*
|
|
131
|
+
* The reversal here is safe ONLY because the seed now travels WITH the row: `documentsExist` (the
|
|
132
|
+
* caller's `documentsTableExists` probe, run BEFORE calling this) selects the exact same
|
|
133
|
+
* `frontierSeedExpr` fragment `tryAcquire` uses — `0` on a fresh database (no `documents` table
|
|
134
|
+
* yet: correct, there is no history to protect against), or `(SELECT COALESCE(MAX(ts), 0) FROM
|
|
135
|
+
* documents)` on an upgrade (`documents` already exists: correct, the row is born at the true
|
|
136
|
+
* high-water mark, never momentarily at 0). A pre-seeded row is created UNACQUIRED — `epoch = 0`,
|
|
137
|
+
* `writer_url = NULL`, already-expired `expires_at` — so the FIRST real `tryAcquire` still lands on
|
|
138
|
+
* its `ON CONFLICT` branch and bumps `epoch` from 0 to 1, byte-identical to a fresh INSERT's
|
|
139
|
+
* `epoch = 1`; every existing epoch/ownership assertion in the test suite is unaffected. Idempotent
|
|
140
|
+
* (`ON CONFLICT (shard_id) DO NOTHING`): a row already created by a peer's concurrent `setup()` or
|
|
141
|
+
* by a real acquisition is left untouched — this can only ever RAISE a row into existence, never
|
|
142
|
+
* clobber one. With every shard row present immediately, the tailer's `count(*) < NUM_SHARDS →
|
|
143
|
+
* not-ready` gate is satisfied the instant `setup()` returns — a concurrent multi-writer boot no
|
|
144
|
+
* longer stalls waiting for a writer to finish its acquire-all pass. See `node.ts`'s
|
|
145
|
+
* `prepareFleetNode` for the `documentsTableExists` probe + call site, and `replica-tailer.ts`'s
|
|
146
|
+
* `readFrontier` for the count gate.
|
|
147
|
+
*/
|
|
148
|
+
/** Run one DDL statement, swallowing the concurrent-boot duplicate-object race family
|
|
149
|
+
* (23505 unique_violation / 42P07 duplicate_table / 42710 duplicate_object — the third face
|
|
150
|
+
* is "type <table> already exists", the table's composite rowtype, which is what two nodes
|
|
151
|
+
* racing CREATE TABLE IF NOT EXISTS actually surface). The statement's objective — the
|
|
152
|
+
* object exists — is achieved when any of these fires; anything else propagates. Mirrors
|
|
153
|
+
* docstore-postgres setupSchema's discipline; added when the SIMULTANEOUS multi-writer boot
|
|
154
|
+
* E2E caught the unguarded race here (Plan A's extra boot DDL widened the window). */
|
|
155
|
+
private ddl;
|
|
156
|
+
setup(shardIds?: readonly ShardId[], documentsExist?: boolean): Promise<void>;
|
|
157
|
+
/**
|
|
158
|
+
* The `frontier_ts` seed SQL fragment shared by `setup()`'s row pre-seed and `tryAcquire()`'s
|
|
159
|
+
* first-creation INSERT (Fleet B3, D4 — the one place this reasoning is written down): `0` when the
|
|
160
|
+
* caller has verified `documents` does not exist yet (a fresh database — no history to protect
|
|
161
|
+
* against), else `(SELECT COALESCE(MAX(ts), 0) FROM documents)` (an upgrade — the row is born at the
|
|
162
|
+
* store's true high-water mark, never momentarily visible below it). Both call sites gate this on
|
|
163
|
+
* the SAME `documentsTableExists` probe (`node.ts`), so a row is never seeded incorrectly regardless
|
|
164
|
+
* of which of the two code paths creates it first.
|
|
165
|
+
*/
|
|
166
|
+
private frontierSeedExpr;
|
|
167
|
+
/**
|
|
168
|
+
* Upsert this node's `fleet_nodes` presence row (B2b, D3), extending `expires_at` by the lease TTL.
|
|
169
|
+
* Called at boot (BEFORE the writer-election `tryAcquire`, so a losing node is already visible when
|
|
170
|
+
* it boots sync) and on every balancer beat / writer probe — the node's liveness signal now. A node
|
|
171
|
+
* whose process dies stops heartbeating; its row expires after the TTL and it drops out of every
|
|
172
|
+
* survivor's `liveNodes()`, re-converging the rendezvous assignment. `epoch` increments per beat —
|
|
173
|
+
* carried only for observability/debugging (which incarnation of a node the row belongs to); the
|
|
174
|
+
* live-set membership check keys off `expires_at`, not `epoch`.
|
|
175
|
+
*/
|
|
176
|
+
heartbeatPresence(): Promise<void>;
|
|
177
|
+
/**
|
|
178
|
+
* The live fleet node set (B2b, D3): distinct unexpired `fleet_nodes` advertise URLs, UNION the live
|
|
179
|
+
* `shard_leases` writer URLs (belt-and-braces — a writer whose presence heartbeat momentarily lapsed
|
|
180
|
+
* but whose shard lease is still live is unambiguously alive). Every node computes rendezvous over
|
|
181
|
+
* this set, so they all derive the same shard→owner assignment. Expiry is `expires_at >= now()` per
|
|
182
|
+
* the DB's OWN clock, authoritative against host clock skew.
|
|
183
|
+
*/
|
|
184
|
+
liveNodes(): Promise<string[]>;
|
|
185
|
+
/**
|
|
186
|
+
* Per-shard ownership snapshot for the balancer (B2b, D3): for each EXISTING `shard_leases` row, its
|
|
187
|
+
* `writer_url` (null = orphaned) and whether it has expired per the DB clock. A shard with no row is
|
|
188
|
+
* simply absent from the map — the balancer treats an absent shard as acquirable (never created, or
|
|
189
|
+
* fully reaped), the same as an orphaned one. Read-only; the balancer decides acquire/release from it.
|
|
190
|
+
*/
|
|
191
|
+
readShardOwnership(): Promise<Map<ShardId, {
|
|
192
|
+
writerUrl: string | null;
|
|
193
|
+
expired: boolean;
|
|
194
|
+
}>>;
|
|
195
|
+
/**
|
|
196
|
+
* Self-fence a shard this node currently holds, for a GRACEFUL balancer release (B2b, D3): bump the
|
|
197
|
+
* epoch (so this node's own subsequent commit/heartbeat on the now-stale epoch fences cleanly),
|
|
198
|
+
* clear `writer_url` (so the shard reads as orphaned and its rightful rendezvous owner acquires it),
|
|
199
|
+
* and GREATEST-bump `frontier_ts` from the shared sequence (so F never regresses across the handoff).
|
|
200
|
+
* Predicated on this node's currently-held epoch — a silent no-op if the shard is already fenced/
|
|
201
|
+
* relinquished (`currentEpoch === null`). The caller runs this only when the shard is fully IDLE —
|
|
202
|
+
* `runtime.tryRunExclusiveOnShard` grants the mutex only if no commit holds it AND no group-commit
|
|
203
|
+
* batch is staged/flushing (Fleet B4), so no in-flight commit's OR batch's frontier write races the
|
|
204
|
+
* GREATEST-bump here; the release is deferred a beat if the shard is busy rather than fencing
|
|
205
|
+
* mid-flush (see `releaseShard` in `node.ts`). A mutation that begins mid-execute after the fence
|
|
206
|
+
* simply hits `FencedError` at its own commit (OCC-retryable, the forwarder re-routes the retry to
|
|
207
|
+
* the new owner). Mirrors `evictExpired`'s frontier-bump SQL, but unconditional on expiry (this is a
|
|
208
|
+
* voluntary handoff, not an eviction of a wedged peer).
|
|
209
|
+
*/
|
|
210
|
+
selfFence(shardId: ShardId): Promise<void>;
|
|
211
|
+
/**
|
|
212
|
+
* The epoch this node most recently acquired for `shardId` (via `tryAcquire`), or `null` if it
|
|
213
|
+
* never has. Read live — not a boot-time snapshot — by the commit guard and the heartbeat probe,
|
|
214
|
+
* so a re-promotion's epoch bump (another `tryAcquire` call) is picked up automatically with no
|
|
215
|
+
* extra threading between `prepareFleetNode`/`startFleetNode`/`promoteFleetNode`. Defaults to the
|
|
216
|
+
* default shard so B1 call sites (`currentEpoch()`) are unchanged.
|
|
217
|
+
*/
|
|
218
|
+
currentEpoch(shardId?: ShardId): bigint | null;
|
|
219
|
+
/** The (shard, epoch) pairs this node currently holds — the input to the batched heartbeat,
|
|
220
|
+
* all-rows seed, and idle-shard closer (all fence per-row against the epoch on file). A snapshot
|
|
221
|
+
* of the live per-shard epoch map, so it reflects any re-promotion's epoch bumps. */
|
|
222
|
+
heldPairs(): Array<{
|
|
223
|
+
shardId: ShardId;
|
|
224
|
+
epoch: bigint;
|
|
225
|
+
}>;
|
|
226
|
+
/**
|
|
227
|
+
* Drop `shardId` from this node's held-epoch map (Fenced Frontier B2b, D2 — per-shard relinquish).
|
|
228
|
+
* After this call `currentEpoch(shardId)` returns `null`, so:
|
|
229
|
+
* - the commit guard's "no acquired epoch" branch fences any straggler commit on `shardId`
|
|
230
|
+
* cleanly (rather than a stale epoch happening to still match a row some other node has since
|
|
231
|
+
* re-fenced and moved on from);
|
|
232
|
+
* - `heldPairs()` (and therefore `heartbeatAll()`/`closeIdleFrontiers()`/`seedFrontierAll()`) stop
|
|
233
|
+
* including this shard.
|
|
234
|
+
* Idempotent: forgetting a shard this node doesn't currently hold is a silent no-op — the
|
|
235
|
+
* relinquish dispatcher (`node.ts`) relies on exactly this for ITS OWN idempotency, via
|
|
236
|
+
* `currentEpoch(shardId) === null` as its "already relinquished" check. Deliberately does NOT touch
|
|
237
|
+
* the `shard_leases` ROW — only the caller's advisory-lock release (`PgClient.releaseShardLock`) and
|
|
238
|
+
* this in-memory forget are relinquish's job; the row itself was already epoch-bumped by whoever
|
|
239
|
+
* fenced this node (or will lapse via TTL), and a future re-acquire (`tryAcquire`) is a fresh INSERT/
|
|
240
|
+
* ON CONFLICT UPDATE regardless of what this map remembers.
|
|
241
|
+
*/
|
|
242
|
+
forgetShard(shardId: ShardId): void;
|
|
243
|
+
/**
|
|
244
|
+
* One non-blocking attempt: takes the advisory lock (fast path); on success, runs the fencing
|
|
245
|
+
* upsert against `shard_leases` (bumping `epoch`, recording this node's URL/app-name, extending
|
|
246
|
+
* `expires_at`) and returns the new state. On failure to take the lock, returns null.
|
|
247
|
+
* `prev_ts` is seeded to 0 on first creation only, and `frontier_ts` to either 0 or the store's
|
|
248
|
+
* current `MAX(ts)` (see `seedFrontierFromDocuments`) — an `ON CONFLICT` re-acquisition (including
|
|
249
|
+
* promotion) leaves BOTH untouched, so the durable-commit chain survives across epochs (D3 depends
|
|
250
|
+
* on this: frontier must never reset just because the writer changed).
|
|
251
|
+
*
|
|
252
|
+
* `seedFrontierFromDocuments` (the F1×N residual-window fix): when a row is FIRST created, seed its
|
|
253
|
+
* `frontier_ts` to `MAX(ts)` from the `documents` log inside the same INSERT — atomically, so the
|
|
254
|
+
* row is NEVER momentarily visible at `frontier_ts = 0` on a pre-loaded store (which would let a
|
|
255
|
+
* concurrently-booting sync node pass its `count == N ∧ min-F` ready gate with an empty replica).
|
|
256
|
+
* Pass `true` only when the `documents` table is known to exist (post-`setupSchema`, or a pre-loaded
|
|
257
|
+
* store) — the caller guards this; `false` (the default, and the only safe value pre-DDL on a fresh
|
|
258
|
+
* database) creates the row at `frontier_ts = 0`, which is correct precisely because a fresh store
|
|
259
|
+
* holds no data. `seedFrontierAll` remains the idempotent belt-and-braces second pass.
|
|
260
|
+
*/
|
|
261
|
+
tryAcquire(shardId?: ShardId, slot?: number, seedFrontierFromDocuments?: boolean): Promise<LeaseState | null>;
|
|
262
|
+
/**
|
|
263
|
+
* Extend `expires_at` for this node's `epoch` — the LeaseMonitor's periodic probe (D2: one
|
|
264
|
+
* round-trip serves liveness-probe + TTL maintenance + fence verification). Returns the number
|
|
265
|
+
* of rows updated: 1 = still this node's epoch (TTL extended), 0 = fenced — some other node has
|
|
266
|
+
* bumped the epoch (a D4 eviction) and this node no longer holds the lease, even though its
|
|
267
|
+
* connection never dropped. Callers (see `node.ts`) treat 0 as definitive lease loss.
|
|
268
|
+
*/
|
|
269
|
+
heartbeat(epoch: bigint, shardId?: ShardId): Promise<number>;
|
|
270
|
+
/**
|
|
271
|
+
* Batched heartbeat over EVERY (shard, epoch) pair this node holds — one UPDATE per beat (B2a):
|
|
272
|
+
* the writer holds N leases and must renew all of them in a single round-trip, not N. Returns
|
|
273
|
+
* `{ updated, expected, fencedShardIds }` where `expected` is how many pairs this node believes it
|
|
274
|
+
* holds, `updated` is how many rows actually matched, and `fencedShardIds` (B2b, D2) is PRECISELY
|
|
275
|
+
* which held shards did NOT match — diffing the `RETURNING shard_id` rows against the held set —
|
|
276
|
+
* so the caller can relinquish exactly those shards rather than treat `updated < expected` as an
|
|
277
|
+
* undifferentiated whole-node signal. `updated < expected` (equivalently `fencedShardIds.length >
|
|
278
|
+
* 0`) means at least one shard's epoch was superseded; `fencedShardIds` is always `[]` when this
|
|
279
|
+
* node holds nothing (never a writer) — see the early return. The `(shard_id, epoch) IN ((..),(..))`
|
|
280
|
+
* tuple form fences per row.
|
|
281
|
+
*/
|
|
282
|
+
heartbeatAll(): Promise<{
|
|
283
|
+
updated: number;
|
|
284
|
+
expected: number;
|
|
285
|
+
fencedShardIds: ShardId[];
|
|
286
|
+
}>;
|
|
287
|
+
/**
|
|
288
|
+
* Read-only expiry check: does the lease row exist AND is `expires_at` in the past, per the DB's
|
|
289
|
+
* OWN clock? Used as the acquire loop's per-tick pre-gate before the heavier `evictExpired`
|
|
290
|
+
* transaction — a healthy (live) lease must NOT make every follower take the row lock every tick and
|
|
291
|
+
* contend with the writer's commit guard, so the common case stays a single cheap SELECT. The
|
|
292
|
+
* comparison is `now()` in SQL (not `read().expiresAt` in JS) so it's authoritative against clock
|
|
293
|
+
* skew between a follower's host and Postgres.
|
|
294
|
+
*/
|
|
295
|
+
isExpired(shardId?: ShardId): Promise<boolean>;
|
|
296
|
+
/**
|
|
297
|
+
* Fencing-first eviction of an EXPIRED lease (Fenced Frontier B1, D4). Bumps `epoch` (fencing the
|
|
298
|
+
* wedged holder — its next commit guard / heartbeat now matches 0 rows), clears `writer_url`/
|
|
299
|
+
* `writer_app_name`, and advances `frontier_ts`, all predicated on the lease actually being
|
|
300
|
+
* expired. Returns `{ fenced: true, oldAppName }` capturing the evicted holder's `writer_app_name`
|
|
301
|
+
* (so the acquire loop can `pg_terminate_backend` its lingering connection); `{ fenced: false }`
|
|
302
|
+
* when the row is live (no-op) OR the row lock couldn't be taken within `lock_timeout` (retry next
|
|
303
|
+
* tick — never throws that to the loop).
|
|
304
|
+
*
|
|
305
|
+
* Why a `SELECT ... FOR UPDATE` then a separate `UPDATE`, NOT a single `UPDATE ... RETURNING (SELECT
|
|
306
|
+
* writer_app_name FROM cte)`: on Postgres, RETURNING — and the inlined CTE subquery it evaluates —
|
|
307
|
+
* sees the NEW (already-nulled) row, so a single statement reads back NULL for the old app name
|
|
308
|
+
* (verified on PGlite; a `MATERIALIZED` CTE with its own `FOR UPDATE` instead collides with the
|
|
309
|
+
* same-statement UPDATE's row lock and yields zero rows). The two-statement form captures the old
|
|
310
|
+
* value cleanly, and the `FOR UPDATE` takes the very row lock a concurrent commit-guard UPDATE
|
|
311
|
+
* contends on — serializing eviction against an in-flight commit. That contention is single-
|
|
312
|
+
* connection-untestable and covered E2E only (see the test header).
|
|
313
|
+
*/
|
|
314
|
+
evictExpired(shardId?: ShardId): Promise<{
|
|
315
|
+
fenced: boolean;
|
|
316
|
+
oldAppName: string | null;
|
|
317
|
+
}>;
|
|
318
|
+
/**
|
|
319
|
+
* Kill the evicted holder's lingering Postgres backend(s) by `application_name` so its session-level
|
|
320
|
+
* advisory writer lock is released and the NEXT acquire tick can win. Matches the fleet E2E's query
|
|
321
|
+
* shape exactly, including the `pid <> pg_backend_pid()` self-exclusion (the fencer never terminates
|
|
322
|
+
* its own connection — its app name differs anyway, so this is belt-and-braces). Best-effort: if the
|
|
323
|
+
* backend is already gone, the query simply affects zero rows.
|
|
324
|
+
*/
|
|
325
|
+
terminateBackend(appName: string): Promise<void>;
|
|
326
|
+
/**
|
|
327
|
+
* A tick observed the advisory-lock try FAIL — another backend still holds the writer lock. If the
|
|
328
|
+
* lease has ALSO expired, that holder is wedged (its heartbeat stopped) while its connection lingers
|
|
329
|
+
* (a hung/paused process still owns the advisory lock, so no other node can acquire). Fence it
|
|
330
|
+
* (`evictExpired` bumps the epoch) then terminate its backend to release the lock; the next tick's
|
|
331
|
+
* advisory try then succeeds → normal acquisition → promotion. No-op when the lease is still live.
|
|
332
|
+
*/
|
|
333
|
+
private maybeEvictWedged;
|
|
334
|
+
/** Loop tryAcquire() every retryMs until it succeeds, then invoke onAcquired once. stop() cancels.
|
|
335
|
+
* On a tick where the advisory try fails AND the lease has expired, the wedged holder is fenced +
|
|
336
|
+
* its backend terminated (`maybeEvictWedged`) so a later tick can take over. */
|
|
337
|
+
acquireLoop(onAcquired: (s: LeaseState) => void): void;
|
|
338
|
+
/** Cancels any pending acquireLoop() retry. */
|
|
339
|
+
stop(): void;
|
|
340
|
+
/**
|
|
341
|
+
* F1 fix (Fenced Frontier B1 whole-branch review, BLOCKER): seed `frontier_ts` up to at least
|
|
342
|
+
* `maxTs` — the writer-BOOT step that closes the "pre-loaded database" hole. `tryAcquire()` only
|
|
343
|
+
* seeds `frontier_ts` to 0 on the row's FIRST creation (see above); a single-node Postgres `serve`
|
|
344
|
+
* that accumulates real data BEFORE `--fleet` is ever turned on has no `shard_leases` row at all,
|
|
345
|
+
* so the first `tryAcquire()` after enabling `--fleet` creates one at `frontier_ts=0` even though
|
|
346
|
+
* the store already holds history. Because the tailer targets `F=frontier_ts` (not
|
|
347
|
+
* `primary.maxTimestamp()`, see `replica-tailer.ts`'s D5), a fresh sync node's ready gate
|
|
348
|
+
* (`wm=0 < target=0`) is then a silent no-op — it reports ready with an EMPTY replica while the
|
|
349
|
+
* primary holds everything. Callers must invoke this at WRITER BOOT, after this node's own
|
|
350
|
+
* `setupSchema()` has definitely run (the `documents` table this reads may not exist before
|
|
351
|
+
* that — see `prepareFleetNode`/`startFleetNode`'s boot ordering) and BEFORE the node is reported
|
|
352
|
+
* ready; a later promotion does NOT need to call this — by then `frontier_ts` is already live,
|
|
353
|
+
* tracking every commit via the guard installed by `installCommitGuard`.
|
|
354
|
+
*
|
|
355
|
+
* `GREATEST` makes this a no-op on an already-live fleet (frontier already tracks every commit)
|
|
356
|
+
* and on re-acquisition — never regresses `frontier_ts`. Epoch-fenced like every other
|
|
357
|
+
* `shard_leases` write: a stale `epoch` here (this node lost the writer race to someone else
|
|
358
|
+
* between its `tryAcquire()` and this call) affects 0 rows rather than clobbering the new
|
|
359
|
+
* writer's state — that writer's own commit guard is the source of truth for `frontier_ts` going
|
|
360
|
+
* forward regardless. Deliberately does NOT touch `prev_ts`: this is a seed of the high-water
|
|
361
|
+
* mark, not a commit, so there is no new "previous commit" to record — the next REAL commit's
|
|
362
|
+
* guard sets `prev_ts := frontier_ts` (now the seeded value) exactly as it always does.
|
|
363
|
+
*/
|
|
364
|
+
seedFrontier(epoch: bigint, maxTs: bigint, shardId?: ShardId): Promise<void>;
|
|
365
|
+
/**
|
|
366
|
+
* All-rows frontier seed (B2a — the F1×N fix): seed EVERY shard this node holds up to `maxTs` in
|
|
367
|
+
* ONE batched, per-row-epoch-fenced UPDATE, run at writer boot BEFORE the node reports ready. The
|
|
368
|
+
* same reasoning as single-shard `seedFrontier` applies to every row at once: any future commit on
|
|
369
|
+
* any shard takes a later `nextval`, so seeding all held frontiers to the store's current max can
|
|
370
|
+
* never over-shoot a real commit, and `GREATEST` keeps it a no-op on an already-live fleet.
|
|
371
|
+
* Epoch-fenced per pair (`(shard_id, epoch) IN (...)`), so a shard this node lost between acquiring
|
|
372
|
+
* it and this call is simply skipped rather than clobbered. See `node.ts`'s writer boot.
|
|
373
|
+
*/
|
|
374
|
+
seedFrontierAll(maxTs: bigint): Promise<void>;
|
|
375
|
+
/**
|
|
376
|
+
* Per-shard idle-shard frontier closing (B2a, D5 — needed the moment N>1: an idle shard pins the
|
|
377
|
+
* fleet's `F = min(frontier_ts)`). Allocate a single fresh `nextval` `N` from the shared
|
|
378
|
+
* `helipod_ts` sequence per beat, then for EACH held shard advance its frontier up to `N` —
|
|
379
|
+
* `UPDATE ... SET frontier_ts = GREATEST(frontier_ts, $N) WHERE shard_id=$s AND epoch=$e AND
|
|
380
|
+
* frontier_ts < $N` — but run each shard's UPDATE UNDER THAT SHARD'S COMMIT MUTEX
|
|
381
|
+
* (`runExclusiveOnShard`), skipping any shard currently mid-commit (mutex busy → left for the next
|
|
382
|
+
* beat). Returns the `nextval` used (the ceiling this beat closed idle shards up to).
|
|
383
|
+
*
|
|
384
|
+
* Why skip-if-busy, not the old bare batched UPDATE (the frontier-inversion fix): the commit
|
|
385
|
+
* guard writes `frontier_ts := commitTs` for a commit that drew its ts `T` from the SAME sequence,
|
|
386
|
+
* *inside* the commit transaction — which runs under the shard's commit mutex (single-commit) or at
|
|
387
|
+
* FLUSH time for a group-commit batch (Fleet B4). If the closer drew `N > T` and bare-wrote
|
|
388
|
+
* `frontier_ts = N` while that commit's/batch's rows had not yet landed, a tailer could read `F ≥
|
|
389
|
+
* N`, pull `(watermark, N]`, and MISS `T`'s rows when they land afterward (silent replica miss), or
|
|
390
|
+
* the guard's later write of `T` would trip a frontier-regression assert. `runExclusiveOnShard`
|
|
391
|
+
* treats a shard as available only when it is fully IDLE — mutex free AND no staged/flushing
|
|
392
|
+
* group-commit batch (`ShardWriter.hasInFlightWork()`): group commit runs the flush OFF the mutex,
|
|
393
|
+
* so a mid-flush batch (ts's already drawn, rows not yet landed) reads BUSY even though the mutex is
|
|
394
|
+
* free, and is skipped for the next beat. Given that, the closer and that shard's commits are
|
|
395
|
+
* mutually exclusive on this (single, writer-owned) node: any commit/batch STARTED before the draw
|
|
396
|
+
* either holds the mutex or is staged/flushing → skipped this beat; any commit/batch that STARTS
|
|
397
|
+
* after the closer's `nextval` draws its `T` at commit/flush time, AFTER `N`, so `T > N` and
|
|
398
|
+
* `GREATEST` keeps `frontier_ts ≥ N` with no regression and nothing in-flight ever sits below `N`.
|
|
399
|
+
* Cross-node closing of a held shard is impossible by design (only
|
|
400
|
+
* the lease holder closes its own held shards; orphan bumping via `evictExpired` targets
|
|
401
|
+
* writer-less rows, where no commit can be in flight). Epoch-fenced per pair. No-op (returns the
|
|
402
|
+
* allocated ts anyway) when this node holds nothing.
|
|
403
|
+
*/
|
|
404
|
+
closeIdleFrontiers(runExclusiveOnShard: TryRunExclusiveOnShard): Promise<bigint>;
|
|
405
|
+
/**
|
|
406
|
+
* Orphan frontier bumping (B2b, D4): advance the frontier of every WRITER-LESS shard row
|
|
407
|
+
* (`writer_url IS NULL`) that lags `ceiling`, so an unassigned/relinquished/expired shard never
|
|
408
|
+
* pins the fleet's `F = min(frontier_ts)` below the live commit position. Runs on the WRITER beat
|
|
409
|
+
* alongside `closeIdleFrontiers` (which only ever touches THIS node's OWN held rows via
|
|
410
|
+
* `heldPairs()` and so structurally cannot un-pin a shard nobody holds).
|
|
411
|
+
*
|
|
412
|
+
* Safe with NO commit mutex, unlike `closeIdleFrontiers`: a writer-less shard has no in-flight
|
|
413
|
+
* commit by construction — its last writer was fenced (epoch-bumped, `writer_url` nulled) BEFORE
|
|
414
|
+
* the row became orphaned, so nothing can be mid-commit writing `frontier_ts` on it. The `UPDATE`'s
|
|
415
|
+
* own row lock serializes any two writers racing this bump, and `GREATEST` + `frontier_ts < ceiling`
|
|
416
|
+
* keep it monotone regardless. Reuses the SAME `ceiling` (one `nextval` per beat) the idle closer
|
|
417
|
+
* drew, so a beat costs one sequence draw, not two.
|
|
418
|
+
*/
|
|
419
|
+
bumpOrphanFrontiers(ceiling: bigint): Promise<void>;
|
|
420
|
+
/** Reads the current lease row for `shardId` (discovery for forwarding, plus the full fencing/
|
|
421
|
+
* frontier state); null if none exists yet. Defaults to the default shard (B1 call sites). */
|
|
422
|
+
read(shardId?: ShardId): Promise<LeaseRow | null>;
|
|
423
|
+
/** All shard rows' `(shard_id, frontier_ts)` — the fleet-wide frontier picture the writer's
|
|
424
|
+
* frontier-lag monitor reads to compute `min(frontier_ts)` + which shard is pinning it (D5's
|
|
425
|
+
* health observability). Ordered by frontier ascending so `rows[0]` is the pinning shard. */
|
|
426
|
+
readAllFrontiers(): Promise<Array<{
|
|
427
|
+
shardId: ShardId;
|
|
428
|
+
frontierTs: bigint;
|
|
429
|
+
}>>;
|
|
430
|
+
/**
|
|
431
|
+
* Read back `key`'s `fleet_idempotency` row for a replay decision (Fleet B3, D3), or `null` if no
|
|
432
|
+
* such key has ever committed (a genuine miss — proceed to run). Called by `packages/cli`'s
|
|
433
|
+
* `/_fleet/run` handler BOTH before running (the SELECT-first check) and after catching a
|
|
434
|
+
* unique_violation on this table (the concurrent-duplicate race's loser re-selecting the
|
|
435
|
+
* winner's row). See `IdempotencyReplay` for what `hasValue`/`oversized` distinguish.
|
|
436
|
+
*/
|
|
437
|
+
lookupIdempotency(key: string): Promise<IdempotencyReplay | null>;
|
|
438
|
+
/**
|
|
439
|
+
* Best-effort post-run recording of a forwarded mutation's return VALUE onto its already-committed
|
|
440
|
+
* `fleet_idempotency` row (Fleet B3, D3) — the value isn't known inside the commit transaction (the
|
|
441
|
+
* guard only sees `commitTs`), so this runs AFTER `runtime.run`/`runAction`/`runSystem` returns,
|
|
442
|
+
* from `packages/cli`'s `/_fleet/run` handler. Over `IDEMPOTENCY_VALUE_CAP_BYTES` → `value_json`
|
|
443
|
+
* stays/goes NULL and `oversized = true` instead of storing it (a replay then reports
|
|
444
|
+
* `valueMissing: true`, same as the crash-window case where this UPDATE never ran at all). A
|
|
445
|
+
* missing row (no matching `key` — e.g. a no-op mutation that staged nothing, so the guard never
|
|
446
|
+
* ran) silently affects 0 rows; the caller doesn't need to check for that itself.
|
|
447
|
+
*/
|
|
448
|
+
recordIdempotencyValue(key: string, value: JSONValue): Promise<void>;
|
|
449
|
+
/**
|
|
450
|
+
* Reclaim `fleet_idempotency` rows older than the TTL (Fleet B3, D3) — a cheap indexed (PK-only
|
|
451
|
+
* table, no index needed at this scale) delete run on the balancer beat of every WRITER-ish node
|
|
452
|
+
* (see `ShardLeaseBalancerDeps.sweepIdempotency` / `node.ts`'s wiring). A retry arriving after a
|
|
453
|
+
* row has been swept simply re-executes (documented boundary — retries are seconds-scale, the
|
|
454
|
+
* sweep window is 1h).
|
|
455
|
+
*/
|
|
456
|
+
sweepIdempotency(): Promise<void>;
|
|
457
|
+
/**
|
|
458
|
+
* Build a `(shard_id, epoch) IN ((...),(...))` VALUES-list clause + its flat params array for a
|
|
459
|
+
* batched per-row-fenced statement, with placeholder numbers starting AFTER `offset` positional
|
|
460
|
+
* params the caller already consumed (e.g. a leading `$1 = maxTs`). Returns `{ clause, params }`
|
|
461
|
+
* where `params` is `[shardA, epochA, shardB, epochB, …]` in placeholder order.
|
|
462
|
+
*/
|
|
463
|
+
private tupleInClause;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Fleet writer liveness monitor (Task 4, C4). Runs ONLY while a node is the writer. Its job: notice
|
|
468
|
+
* when this node has lost its exclusive writer lease and exit the process so it can rejoin the fleet
|
|
469
|
+
* as a fresh sync node (exit-and-rejoin) — rather than lingering as a zombie writer that believes it
|
|
470
|
+
* still owns the lock while some other node has taken over.
|
|
471
|
+
*
|
|
472
|
+
* Three independent loss signals feed it:
|
|
473
|
+
*
|
|
474
|
+
* - `connectionLost()` — DEFINITIVE. The advisory lock lives on `NodePgClient`'s single pinned
|
|
475
|
+
* connection, which never reconnects (see `NodePgClient.ensure()`); if that connection closes or
|
|
476
|
+
* errors, the session-level `pg_advisory_lock` is released by Postgres the instant the backend
|
|
477
|
+
* goes away. So a closed pinned connection == lease definitely lost → exit immediately.
|
|
478
|
+
*
|
|
479
|
+
* - `fenced()` — DEFINITIVE (Fenced Frontier B1, D2/D3). The connection is still alive, but this
|
|
480
|
+
* node's epoch has been superseded — either the heartbeat probe found 0 rows for `(shard_id,
|
|
481
|
+
* epoch)`, or an epoch-fenced commit guard's UPDATE matched 0 rows. Both mean some OTHER node
|
|
482
|
+
* now holds the lease, so — exactly like `connectionLost()` — this bypasses the probe-miss
|
|
483
|
+
* tolerance below (which exists only for transient/ambiguous blips; a fenced epoch never is)
|
|
484
|
+
* and exits immediately.
|
|
485
|
+
*
|
|
486
|
+
* - `probe()` misses — a BACKSTOP for a silently-wedged connection that isn't emitting error/end
|
|
487
|
+
* events (e.g. a half-open TCP connection). The probe is a caller-supplied liveness round-trip
|
|
488
|
+
* (`() => client.query("SELECT 1")`); it must NEVER be `pg_try_advisory_lock` — that is re-entrant
|
|
489
|
+
* on the session already holding the lock and would leak a lock count on every probe. We tolerate
|
|
490
|
+
* `maxMisses` consecutive failures (transient blips) and exit on the NEXT one; any success resets
|
|
491
|
+
* the counter. Each probe is itself bounded by `probeTimeoutMs` (default `probeMs`) via
|
|
492
|
+
* `Promise.race` — a silently-wedged connection (e.g. a half-open TCP socket) can leave
|
|
493
|
+
* `SELECT 1` hanging forever with no error/end event; without a per-probe timeout `inFlight`
|
|
494
|
+
* would stay true and every subsequent tick would early-return, so misses would never accrue and
|
|
495
|
+
* this exact backstop would never fire. A timed-out probe counts as a miss; if the underlying
|
|
496
|
+
* query later settles anyway, that late settlement is discarded (`Promise.race` only ever acts on
|
|
497
|
+
* whichever of the probe/timeout settles first) — no reset, no double-count. `inFlight` still
|
|
498
|
+
* clears once the race settles, so the next tick starts its own fresh attempt regardless of what
|
|
499
|
+
* the abandoned probe call eventually does.
|
|
500
|
+
*
|
|
501
|
+
* `onExit` is injected: production passes `(reason) => { console.error(...); process.exit(1); }`;
|
|
502
|
+
* tests pass a spy. It fires AT MOST ONCE — the first of (connectionLost, probe-exhaustion) wins,
|
|
503
|
+
* and `stop()` (graceful shutdown / promotion hand-off) halts everything so nothing fires after.
|
|
504
|
+
*/
|
|
505
|
+
interface LeaseMonitorDeps {
|
|
506
|
+
/** Liveness round-trip against the pinned writer connection. MUST be a plain query (e.g.
|
|
507
|
+
* `SELECT 1`), NEVER `pg_try_advisory_lock` (re-entrant → leaks a lock count per probe). */
|
|
508
|
+
probe: () => Promise<void>;
|
|
509
|
+
/** Fired at most once when the lease is deemed lost. Production: `console.error` + `process.exit(1)`. */
|
|
510
|
+
onExit: (reason: string) => void;
|
|
511
|
+
/** Interval between probes. Default 5000ms. */
|
|
512
|
+
probeMs?: number;
|
|
513
|
+
/** Consecutive probe failures tolerated before exit — the NEXT (maxMisses+1-th) miss exits. Default 3. */
|
|
514
|
+
maxMisses?: number;
|
|
515
|
+
/** Per-probe timeout — a probe that hasn't settled by this point counts as a miss (the wedged
|
|
516
|
+
* connection is abandoned, not awaited further). Default `probeMs`. */
|
|
517
|
+
probeTimeoutMs?: number;
|
|
518
|
+
}
|
|
519
|
+
declare class LeaseMonitor {
|
|
520
|
+
private readonly probe;
|
|
521
|
+
private readonly onExit;
|
|
522
|
+
private readonly probeMs;
|
|
523
|
+
private readonly maxMisses;
|
|
524
|
+
private readonly probeTimeoutMs;
|
|
525
|
+
private timer;
|
|
526
|
+
private misses;
|
|
527
|
+
private inFlight;
|
|
528
|
+
private exited;
|
|
529
|
+
private stopped;
|
|
530
|
+
constructor(deps: LeaseMonitorDeps);
|
|
531
|
+
/** Begin probing every `probeMs`. Idempotent; a no-op once stopped or after exit. */
|
|
532
|
+
start(): void;
|
|
533
|
+
private tick;
|
|
534
|
+
private recordMiss;
|
|
535
|
+
/** DEFINITIVE lease loss (pinned connection closed/errored) → exit immediately. */
|
|
536
|
+
connectionLost(): void;
|
|
537
|
+
/** DEFINITIVE lease loss (epoch superseded — heartbeat or commit guard found 0 rows for this
|
|
538
|
+
* node's epoch) → exit immediately, mirroring `connectionLost()`. Idempotent/at-most-once via
|
|
539
|
+
* the same `fireExit` guard — safe to call from both the heartbeat probe and a commit guard. */
|
|
540
|
+
fenced(reason?: string): void;
|
|
541
|
+
private fireExit;
|
|
542
|
+
/** Halt all probing and disarm exit — graceful shutdown or promotion hand-off. */
|
|
543
|
+
stop(): void;
|
|
544
|
+
private clearTimer;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Thrown when this node's epoch has been superseded by another writer (Fenced Frontier B1, D2/D3).
|
|
549
|
+
* Two sources:
|
|
550
|
+
*
|
|
551
|
+
* - `LeaseManager.heartbeat(epoch)` finds zero rows matching `(shard_id, epoch)` — some other
|
|
552
|
+
* node has fenced this one (bumped the epoch, e.g. the D4 wedged-writer eviction path) and
|
|
553
|
+
* this node's lease is gone even though it never lost its Postgres connection.
|
|
554
|
+
* - `PostgresDocStore`'s installed commit guard's epoch-predicated `UPDATE` matches zero rows —
|
|
555
|
+
* a straggler commit that raced a fencer and lost; the whole transaction aborts.
|
|
556
|
+
*
|
|
557
|
+
* Both are DEFINITIVE lease loss — exactly like a dropped connection — and are routed to
|
|
558
|
+
* `LeaseMonitor.fenced()` for immediate exit (bypassing the probe-miss tolerance, which exists
|
|
559
|
+
* only for transient/ambiguous blips; a fenced epoch is never ambiguous).
|
|
560
|
+
*/
|
|
561
|
+
declare class FencedError extends Error {
|
|
562
|
+
constructor(message: string);
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Cross-process commit propagation for a Postgres-backed fleet (scale-seam #4, multi-process
|
|
567
|
+
* variant): the in-memory `EmbeddedWriteFanoutAdapter` only fans a commit out within ONE process.
|
|
568
|
+
* `NotifyingFanoutAdapter` (writer side) wraps it so every publish ALSO does
|
|
569
|
+
* `pg_notify('helipod_commits', …)`; each follower's `ReplicaTailer` (`replica-tailer.ts`)
|
|
570
|
+
* LISTENs for that channel (plus a wall-clock poll fallback, since NOTIFY delivery is not
|
|
571
|
+
* guaranteed while a listener connection is reconnecting) and, on each wake, verbatim-applies the
|
|
572
|
+
* primary's MVCC log onto its local replica and derives what to invalidate from that same batch —
|
|
573
|
+
* no in-process `OplogDelta` ever crosses a process boundary.
|
|
574
|
+
*
|
|
575
|
+
* The `CommitChannelClient` seam below (the LISTEN/NOTIFY + parameterized-query slice of
|
|
576
|
+
* `NodePgClient`) is shared by both the writer-side adapter here and the follower-side
|
|
577
|
+
* `ReplicaTailer`. The slice-1 `CommitTailer` (which derived-only, never applied to a replica)
|
|
578
|
+
* was removed in slice 2 once `ReplicaTailer` subsumed it.
|
|
579
|
+
*/
|
|
580
|
+
|
|
581
|
+
/**
|
|
582
|
+
* The narrow slice of `NodePgClient` this module (and `ReplicaTailer`) depends on (LISTEN/NOTIFY
|
|
583
|
+
* plus a plain parameterized query) — kept as a structural interface, matching the `PgClient` seam's
|
|
584
|
+
* own philosophy of never tying engine/fleet logic to a concrete driver class. A `NodePgClient`
|
|
585
|
+
* instance satisfies this; so does any test double that implements the same shape.
|
|
586
|
+
*/
|
|
587
|
+
interface CommitChannelClient extends PgQuerier {
|
|
588
|
+
/** LISTEN on `channel`; returns a closer. Rejecting (e.g. no LISTEN support) is tolerated by
|
|
589
|
+
* `ReplicaTailer.start()`, which falls back to poll-only. */
|
|
590
|
+
listen(channel: string, onNotify: (payload: string) => void): Promise<() => Promise<void>>;
|
|
591
|
+
notify(channel: string, payload: string): Promise<void>;
|
|
592
|
+
}
|
|
593
|
+
/** Writer side: wraps the in-memory adapter; every publish ALSO does
|
|
594
|
+
* `pg_notify('helipod_commits', String(commitTs))` so followers wake promptly instead of
|
|
595
|
+
* waiting out the poll interval. NOTIFY is a latency optimization only — the poll fallback in
|
|
596
|
+
* `CommitTailer` is the correctness path if it's ever dropped or a listener misses it. */
|
|
597
|
+
declare class NotifyingFanoutAdapter implements EmbeddedWriteFanoutAdapter {
|
|
598
|
+
private readonly inner;
|
|
599
|
+
private readonly client;
|
|
600
|
+
constructor(inner: EmbeddedWriteFanoutAdapter, client: CommitChannelClient);
|
|
601
|
+
publish(payload: EmbeddedWriteFanoutPayload): void;
|
|
602
|
+
subscribe(listener: FanoutListener): () => void;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* `ReplicaTailer` (Fleet slice 2, Task 2; Fenced Frontier B1, D5/D6) — tails the Postgres
|
|
607
|
+
* primary's MVCC log VERBATIM onto a local embedded replica `DocStore` (an in-process SQLite
|
|
608
|
+
* instance, in the intended deployment), deriving invalidation ranges straight from the SAME
|
|
609
|
+
* batch it just applied — not a separate query — so a follower's reactive fan-out is provably
|
|
610
|
+
* consistent with what actually landed on the replica.
|
|
611
|
+
*
|
|
612
|
+
* Evolves `CommitTailer` (`commit-notifier.ts`, slice 1)'s LISTEN + poll wake posture, the
|
|
613
|
+
* `draining` re-entrancy guard, and watermark-advances-only-after-`onInvalidation`-resolves
|
|
614
|
+
* semantics — but diverges in structural ways `CommitTailer` never needed:
|
|
615
|
+
*
|
|
616
|
+
* 1. VERBATIM APPLY — `CommitTailer` only ever derived ranges to wake a LIVE in-process runtime;
|
|
617
|
+
* it never touched a second store. This class additionally re-materializes the primary's
|
|
618
|
+
* actual `DocumentLogEntry`/`IndexWrite` rows onto a real replica `DocStore` via `write(...,
|
|
619
|
+
* "Overwrite")`, so the replica is a byte-for-byte MVCC mirror (historical reads included),
|
|
620
|
+
* not just a wake signal.
|
|
621
|
+
* 2. BOOTSTRAP CATCH-UP — a fresh replica starts at watermark 0 (or wherever it last left off),
|
|
622
|
+
* which can be arbitrarily far behind the primary. `start()` doesn't resolve until the
|
|
623
|
+
* replica has caught up to the FENCED FRONTIER `F` AT CALL TIME (the ready gate), batching
|
|
624
|
+
* the catch-up in `batchSize`-sized ticks instead of one unbounded pull.
|
|
625
|
+
* 3. FENCED-FRONTIER TARGETING (D5) — the pull target is no longer `primary.maxTimestamp()` (the
|
|
626
|
+
* log's live high-water mark, which can include a commit that raced past the last fence
|
|
627
|
+
* check under contention) but `F = shard_leases.frontier_ts` — the durably-fenced, dense
|
|
628
|
+
* prefix the epoch-fenced commit guard (`node.ts`'s `installCommitGuard`) advances inside
|
|
629
|
+
* every commit transaction. At one shard F advances exactly with commits, so this is
|
|
630
|
+
* behavior-identical to the old target in the steady state; the difference only matters
|
|
631
|
+
* under a wedged-writer fencing race (B1 D3/D4), where `maxTimestamp()` could momentarily
|
|
632
|
+
* run ahead of what's actually safe to consider durable. See `stable-prefix.ts` for the
|
|
633
|
+
* `StablePrefixTs` brand this introduces.
|
|
634
|
+
* 4. DENSITY ASSERTIONS (D5) — defense-in-depth: the construction (D1 store-allocated ts + D3
|
|
635
|
+
* epoch-fenced commits) is what actually PREVENTS a skipped commit from ever landing on a
|
|
636
|
+
* replica, but the apply loop additionally verifies it, per document entry, before applying:
|
|
637
|
+
* the replica's current head for that doc must chain from `prev_ts` exactly (or be absent,
|
|
638
|
+
* for an insert). A violation throws `DensityViolationError` — crash loudly rather than serve
|
|
639
|
+
* silently corrupted state; the operator remedy is the already-shipped delete-and-re-bootstrap
|
|
640
|
+
* (a replica file is a rebuildable mirror).
|
|
641
|
+
*
|
|
642
|
+
* `CommitTailer` was the slice-1 derive-only precursor; slice 2 (Task 4) deleted it once this class
|
|
643
|
+
* subsumed its wake/derive posture with verbatim replica apply. `AppliedInvalidation` below carries
|
|
644
|
+
* the invalidation shape (identical members to what `CommitTailer.DerivedInvalidation` had).
|
|
645
|
+
*
|
|
646
|
+
* Per-tick pipeline (see the class body for the full step-by-step):
|
|
647
|
+
* 1. `F = await readFrontier()` (D5: `shard_leases.frontier_ts`, branded `StablePrefixTs`); no-op
|
|
648
|
+
* if `<= watermark`. F is asserted non-decreasing across reads (D5's other defense-in-depth
|
|
649
|
+
* invariant) — a regression means `shard_leases` itself was corrupted/hand-tampered.
|
|
650
|
+
* 2. Pull `DocumentLogEntry` rows for `(watermark, F]` via `primary.load_documents`, capped at
|
|
651
|
+
* `batchSize` — but never splitting a single commit's ts group across ticks (a transaction
|
|
652
|
+
* shares exactly one commit `ts` across all its writes, see `postgres-docstore.ts`'s
|
|
653
|
+
* `write()` doc comment; if the batch fills mid-way through a ts group, the remaining
|
|
654
|
+
* same-ts rows are drained too before capping, so a partial transaction's writes are never
|
|
655
|
+
* applied on the replica).
|
|
656
|
+
* 3. Pull the matching `indexes` rows for the SAME `(watermark, cappedMax]` via raw SQL and
|
|
657
|
+
* invert `postgres-docstore.ts`'s `write()` serialization exactly: `deleted=true` → the
|
|
658
|
+
* `Deleted` variant, else `NonClustered` carrying the decoded `docId`.
|
|
659
|
+
* 4. Density-assert the batch (D5) against the replica's PRE-apply state, THEN
|
|
660
|
+
* `replica.write(docs, indexWrites, "Overwrite")` — verbatim, idempotent re-apply.
|
|
661
|
+
* 5. Build `AppliedInvalidation` from the SAME in-memory batch (index-derived writtenTables/
|
|
662
|
+
* writtenKeys, DISTINCT-by-(tableId,internalId) writtenDocs from the doc entries).
|
|
663
|
+
* 6. `await onInvalidation(inv)`, THEN advance the watermark (branded `StablePrefixTs`), THEN
|
|
664
|
+
* resolve any satisfied `waitFor()`s — a throwing/slow handler must not cause a range to be
|
|
665
|
+
* silently skipped.
|
|
666
|
+
*/
|
|
667
|
+
|
|
668
|
+
/**
|
|
669
|
+
* Thrown by the apply loop's density assertion (D5, defense-in-depth — the construction, D1 store-
|
|
670
|
+
* allocated ts + D3 epoch-fenced commits, is the actual guarantee that prevents this) when a
|
|
671
|
+
* document entry's `prev_ts` doesn't chain from the replica's actual current head: either a
|
|
672
|
+
* non-null `prev_ts` that doesn't match (or is missing entirely — no live head), or a null
|
|
673
|
+
* `prev_ts` (an insert) where the replica already has a live head. Either shape means a commit
|
|
674
|
+
* touching this document was skipped somewhere between the primary and this replica — serving
|
|
675
|
+
* reads off it from here on would be silent corruption, so the tailer crashes loudly instead. The
|
|
676
|
+
* shipped operator remedy is deleting the local replica file and letting it re-bootstrap from the
|
|
677
|
+
* primary (a replica is always a rebuildable mirror).
|
|
678
|
+
*/
|
|
679
|
+
declare class DensityViolationError extends Error {
|
|
680
|
+
readonly docId: InternalDocumentId;
|
|
681
|
+
readonly expectedPrevTs: bigint | null;
|
|
682
|
+
readonly actualHeadTs: bigint | null;
|
|
683
|
+
constructor(docId: InternalDocumentId, expectedPrevTs: bigint | null, actualHeadTs: bigint | null);
|
|
684
|
+
}
|
|
685
|
+
/** Mirrors slice 1's `DerivedInvalidation` shape byte-for-byte (see the module doc comment for why
|
|
686
|
+
* this is a deliberate parallel type, not an import). `newMaxTs` is the tick's APPLIED ceiling,
|
|
687
|
+
* which may be less than the primary's live `maxTimestamp()` when a batch was capped. */
|
|
688
|
+
interface AppliedInvalidation {
|
|
689
|
+
newMaxTs: bigint;
|
|
690
|
+
/** DISTINCT `table_id` values touched, as strings (the storage-encoded table id). */
|
|
691
|
+
writtenTables: string[];
|
|
692
|
+
/** Raw written index keys — point invalidation input, NOT yet point ranges. */
|
|
693
|
+
writtenKeys: Array<{
|
|
694
|
+
indexId: string;
|
|
695
|
+
key: Uint8Array;
|
|
696
|
+
}>;
|
|
697
|
+
/** DISTINCT `(table_id, internal_id)` pairs written in this batch, deduped from the applied
|
|
698
|
+
* `DocumentLogEntry` rows themselves (one entry per doc regardless of how many revisions or
|
|
699
|
+
* index rows accompanied it in this batch). Point invalidation input for the DOCUMENT
|
|
700
|
+
* keyspace, NOT yet point ranges — same split as `writtenKeys` above. */
|
|
701
|
+
writtenDocs: Array<{
|
|
702
|
+
tableId: string;
|
|
703
|
+
internalId: Uint8Array;
|
|
704
|
+
}>;
|
|
705
|
+
}
|
|
706
|
+
interface ReplicaTailerOptions {
|
|
707
|
+
/**
|
|
708
|
+
* Tailer mode (B2b, D5/T5-c):
|
|
709
|
+
* - `"replica"` (default, sync node): verbatim-apply the primary's MVCC log onto the local replica
|
|
710
|
+
* `DocStore` AND derive invalidation from the applied batch — the shipped follower behavior.
|
|
711
|
+
* - `"invalidateOnly"` (writer-ish node in MULTI-WRITER mode): DERIVE-ONLY. The node already has
|
|
712
|
+
* every shard's data (it reads the primary directly), so it never applies to a replica and never
|
|
713
|
+
* runs the density assertions — it only pulls `(watermark, F]` to derive what a FOREIGN writer's
|
|
714
|
+
* commit touched, then fires `onInvalidation` so its own live subscriptions re-run and its oracle
|
|
715
|
+
* observes the new ts. Watermark seeds at boot F (only invalidate commits from here on, not
|
|
716
|
+
* re-derive all history). In this mode the `replica` constructor arg is NEVER dereferenced.
|
|
717
|
+
*/
|
|
718
|
+
mode?: "replica" | "invalidateOnly";
|
|
719
|
+
/** Wall-clock poll fallback interval, in ms. Default 1000. */
|
|
720
|
+
pollMs?: number;
|
|
721
|
+
/** Max `DocumentLogEntry` rows pulled per tick (bootstrap catch-up + steady state). Default
|
|
722
|
+
* 1000. A tick may apply slightly more than this to avoid splitting a commit's ts group. */
|
|
723
|
+
batchSize?: number;
|
|
724
|
+
/** Number of shards the fleet runs (B2a). The tailer's target is `F = min(frontier_ts)` over ALL
|
|
725
|
+
* shard rows, and it refuses to treat a partial `shard_leases` (`count(*) < numShards`) as ready —
|
|
726
|
+
* a half-created lease table must not fake a min (belt-and-braces against the F1×N hole). Default
|
|
727
|
+
* 1 → B1's single-shard behavior. */
|
|
728
|
+
numShards?: number;
|
|
729
|
+
/** Invoked once per non-empty applied batch, in watermark order, AFTER the batch has already
|
|
730
|
+
* been written to the replica. The watermark only advances after this resolves. */
|
|
731
|
+
onInvalidation: (inv: AppliedInvalidation) => Promise<void>;
|
|
732
|
+
}
|
|
733
|
+
type WaitOutcome = "reached" | "timeout" | "released";
|
|
734
|
+
/** Verbatim log-apply tailer: primary Postgres MVCC log -> local replica `DocStore`, plus the
|
|
735
|
+
* batch-derived invalidation feed and the `waitFor` read-your-own-writes primitive Task 3 needs. */
|
|
736
|
+
declare class ReplicaTailer {
|
|
737
|
+
private readonly client;
|
|
738
|
+
private readonly primary;
|
|
739
|
+
private readonly replica;
|
|
740
|
+
private readonly mode;
|
|
741
|
+
private readonly pollMs;
|
|
742
|
+
private readonly batchSize;
|
|
743
|
+
private readonly numShards;
|
|
744
|
+
private readonly onInvalidation;
|
|
745
|
+
/** The tailer's own applied high-water mark — a `StablePrefixTs` (D6): only ever seeded from the
|
|
746
|
+
* replica's own persisted `maxTimestamp()` (a prior run's watermark, on restart) or advanced to
|
|
747
|
+
* a freshly-read/capped `F`. Never assigned a raw, un-branded `bigint` directly. */
|
|
748
|
+
private wm;
|
|
749
|
+
/** The last-observed fenced frontier (D5) — tracked purely to assert F never regresses across
|
|
750
|
+
* reads (`null` = no read yet, so the first `readFrontier()` establishes the baseline without
|
|
751
|
+
* asserting anything). */
|
|
752
|
+
private lastF;
|
|
753
|
+
private timer;
|
|
754
|
+
private unlisten;
|
|
755
|
+
private stopped;
|
|
756
|
+
/** Reentrancy guard: a NOTIFY wake and a poll tick can land back-to-back — only one
|
|
757
|
+
* pull-apply-invalidate walk runs at a time, so `onInvalidation` never sees overlapping ranges,
|
|
758
|
+
* and the bootstrap loop in `start()` never races a concurrent LISTEN-triggered tick. */
|
|
759
|
+
private draining;
|
|
760
|
+
private readonly waiters;
|
|
761
|
+
constructor(client: CommitChannelClient, primary: PostgresDocStore, replica: DocStore, opts: ReplicaTailerOptions);
|
|
762
|
+
/** Plain `bigint` on purpose (not the branded `StablePrefixTs`) — external callers (tests, the
|
|
763
|
+
* `PromotionDeps` seam) only ever compare/print this value; widening away the internal brand at
|
|
764
|
+
* the public boundary costs nothing and avoids forcing every caller to know about the brand. */
|
|
765
|
+
watermark(): bigint;
|
|
766
|
+
/**
|
|
767
|
+
* Reads the fenced frontier `F` (Fenced Frontier B1 D5, generalized to N shards in B2a) —
|
|
768
|
+
* `F = min(frontier_ts)` over ALL `shard_leases` rows — via the same `CommitChannelClient` already
|
|
769
|
+
* threaded in for LISTEN/NOTIFY. The min is the dense prefix the WHOLE fleet has durably fenced: a
|
|
770
|
+
* replica may serve reads up to it and never past a commit that hasn't been fenced on every shard.
|
|
771
|
+
*
|
|
772
|
+
* Belt-and-braces (B2a — the F1×N hole otherwise recurs ×N): treat `count(*) < numShards` as
|
|
773
|
+
* F=0-equivalent (NOT ready). A half-created `shard_leases` — some shards claimed+seeded, others not
|
|
774
|
+
* yet — must not let `min` over the present rows fake a ready frontier; the writer's acquire-all +
|
|
775
|
+
* seed-all completes (all N rows present, each seeded ≥ max) BEFORE it reports ready, so a real min
|
|
776
|
+
* only appears once every shard is genuinely fenced. `count(*) = 0` (fresh fleet, pre-acquisition)
|
|
777
|
+
* is the same F=0.
|
|
778
|
+
*
|
|
779
|
+
* Asserts F is monotonically non-decreasing across reads once the count gate is satisfied (D5's
|
|
780
|
+
* defense-in-depth invariant) — a regression can only mean `shard_leases` was corrupted/hand-
|
|
781
|
+
* tampered. The assertion is skipped while `count(*) < numShards` (F is a placeholder 0 there, so a
|
|
782
|
+
* later real min is legitimately larger, not a regression).
|
|
783
|
+
*/
|
|
784
|
+
private readFrontier;
|
|
785
|
+
/**
|
|
786
|
+
* @param seedWm ONLY consulted in `"invalidateOnly"` mode (ignored in `"replica"` mode, which always
|
|
787
|
+
* seeds from the replica's own persisted high-water mark — see below). When provided, seeds the
|
|
788
|
+
* watermark from THIS value instead of a fresh `readFrontier()` read.
|
|
789
|
+
*
|
|
790
|
+
* Fixes the PROMOTION-HANDOFF INVALIDATION GAP (B2b whole-branch review): a promoting node stops
|
|
791
|
+
* its `ReplicaTailer` (sync mode) at some watermark `W`, then starts this derive-only listener.
|
|
792
|
+
* Between the tailer's stop and the listener's start, a co-writer's commit can land and bump the
|
|
793
|
+
* fenced frontier `F` past `W` — if the listener then seeds from a FRESH `readFrontier()` (i.e.
|
|
794
|
+
* that same later `F`), it treats everything up to `F` as already-known and never derives/
|
|
795
|
+
* invalidates the `(W, F]` range, even though nothing ever actually invalidated it: the sync
|
|
796
|
+
* tailer was already stopped, and this listener starts past it. A subscription open across the
|
|
797
|
+
* promotion goes silently stale for exactly those commits.
|
|
798
|
+
*
|
|
799
|
+
* The caller (`node.ts`'s `startWriterInvalidationListener`) closes this gap by passing the
|
|
800
|
+
* OUTGOING tailer's own final `watermark()` (captured right after `tailer.stop()`) as `seedWm` on
|
|
801
|
+
* promotion, so the listener picks up exactly where the tailer left off — contiguous coverage,
|
|
802
|
+
* with the tailer's own last-applied range harmlessly re-derived (idempotent double-invalidation).
|
|
803
|
+
* Writer BOOT (no prior tailer, so no gap by construction) omits `seedWm` and keeps the original
|
|
804
|
+
* fresh-`readFrontier()` behavior.
|
|
805
|
+
*/
|
|
806
|
+
start(seedWm?: bigint): Promise<void>;
|
|
807
|
+
stop(): Promise<void>;
|
|
808
|
+
/** Resolves when `watermark() >= ts` (immediately if already true), or when `release()` fires.
|
|
809
|
+
* Task 3's read-your-own-writes primitive: a client that just committed at `ts` can wait for
|
|
810
|
+
* a follower's replica to have caught up before serving that client's next read from it. */
|
|
811
|
+
waitFor(ts: bigint, timeoutMs: number): Promise<WaitOutcome>;
|
|
812
|
+
/** Releases ALL pending `waitFor()`s with `"released"` (e.g. this node was just promoted to
|
|
813
|
+
* writer, or is shutting down, and callers should stop waiting on replica catch-up). */
|
|
814
|
+
release(): void;
|
|
815
|
+
private wakeSatisfiedWaiters;
|
|
816
|
+
private tick;
|
|
817
|
+
/** One pull-apply-invalidate walk. Split out from `tick()` so `tick()` can wrap it with the
|
|
818
|
+
* DensityViolationError halt handling without that catch swallowing the `draining` bookkeeping. */
|
|
819
|
+
private drainOnce;
|
|
820
|
+
/**
|
|
821
|
+
* Density assertion (D5, defense-in-depth): for EVERY document entry in this batch, in
|
|
822
|
+
* `DocumentLogEntry` order, the replica's head for that document IMMEDIATELY BEFORE this entry
|
|
823
|
+
* lands must chain from `entry.prev_ts` exactly — `prev_ts !== null` requires a live head equal
|
|
824
|
+
* to it; `prev_ts === null` (an insert) requires no live head at all. Throws `DensityViolationError`
|
|
825
|
+
* on the first mismatch.
|
|
826
|
+
*
|
|
827
|
+
* IDEMPOTENT RE-APPLY exception: if the replica's head is ALREADY at this exact entry's own
|
|
828
|
+
* `ts`, that's not a violation — it's this exact revision being re-applied (the whole class is
|
|
829
|
+
* built around `"Overwrite"` idempotency, e.g. a restart re-walking an already-applied range).
|
|
830
|
+
* Since `ts` is the log's globally unique per-shard commit position, a head landing on it can
|
|
831
|
+
* only mean this precise commit already landed here, never an unrelated collision.
|
|
832
|
+
*
|
|
833
|
+
* A batch can carry more than one revision of the SAME document (it was written more than once
|
|
834
|
+
* inside the pulled `(wm, F]` range), so "the head immediately before this entry" is a running
|
|
835
|
+
* per-batch simulation — seeded from the replica's REAL state (`replica.get`) the first time a
|
|
836
|
+
* doc is encountered, then advanced in-memory to each entry's own `ts` as it's validated —
|
|
837
|
+
* rather than re-reading the replica for every entry (which would see later, not-yet-applied
|
|
838
|
+
* entries' predecessor as still absent).
|
|
839
|
+
*/
|
|
840
|
+
private assertDensity;
|
|
841
|
+
/**
|
|
842
|
+
* Pull `DocumentLogEntry` rows for `(after, upTo]` from `load_documents`, capped at
|
|
843
|
+
* `this.batchSize` — but if the cap lands mid-way through a group of rows sharing the same
|
|
844
|
+
* commit `ts`, keep draining until that ts group is fully collected before stopping. A single
|
|
845
|
+
* transaction shares exactly one commit `ts` across every document it wrote (see
|
|
846
|
+
* `postgres-docstore.ts`'s `write()`), so cutting a batch mid-group would apply a transaction's
|
|
847
|
+
* writes partially, which `load_documents`'s `(after, cap]`-shaped next-tick range would then
|
|
848
|
+
* never revisit (the excluded rows share `ts === cap`, which the next tick's range excludes as
|
|
849
|
+
* its own lower bound).
|
|
850
|
+
*
|
|
851
|
+
* `TimestampRange` is `{minInclusive, maxExclusive}` (see `packages/docstore/src/types.ts`) —
|
|
852
|
+
* the half-open opposite skew from the `(after, upTo]` shape this tailer needs, so the bounds
|
|
853
|
+
* are translated with a `+1n` offset on both sides (safe: `ts` is a monotonic integer log
|
|
854
|
+
* position, never a real number needing density guarantees).
|
|
855
|
+
*/
|
|
856
|
+
private pullDocs;
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
/**
|
|
860
|
+
* `WriteForwarder` — the non-owner side of the fleet write path. Implements the engine's
|
|
861
|
+
* per-shard `WriteRouter` seam (B2b, D1): for any shard this node does NOT currently hold,
|
|
862
|
+
* `forward()` POSTs the call to that shard's owner (`/_fleet/run`, discovered per-shard from the
|
|
863
|
+
* `shard_leases` row) and returns its JSON result. `isLocalWriter(shardId)` is a live view of the
|
|
864
|
+
* node's held-shard set (`LeaseManager.currentEpoch(shardId) !== null` — the SAME source of truth
|
|
865
|
+
* `relinquish()` uses for its own idempotency check), so a shard acquired/relinquished mid-flight
|
|
866
|
+
* (balancer rebalancing, failover) takes effect on the very next call — no caching, no stale role.
|
|
867
|
+
*
|
|
868
|
+
* The forwarder learns each shard's writer URL from that shard's `shard_leases` discovery row (via
|
|
869
|
+
* `LeaseManager.read(shardId)`), never from static config — so a failover to a new owner is picked
|
|
870
|
+
* up by re-reading the lease. `writerUrlFor(shardId)` caches per shard (one node normally forwards
|
|
871
|
+
* the same shard repeatedly) and refreshes on a POST failure, retrying once — same shape as the
|
|
872
|
+
* shipped single-shard `writerUrl()` this replaces, generalized per shard.
|
|
873
|
+
*
|
|
874
|
+
* Single-hop guard (B2b, D1 spec-review edit): every forward body carries `forwarded: true`. If it
|
|
875
|
+
* lands on a node that is ALSO not the shard's owner (a point-in-time race during rebalance
|
|
876
|
+
* convergence), the receiver's `/_fleet/run` handler rejects with a typed, retryable
|
|
877
|
+
* `NotShardOwnerError` instead of re-forwarding itself (which would let a forward chase a moving
|
|
878
|
+
* target unboundedly). `forward()` treats that ONE error shape like a transport failure — refresh
|
|
879
|
+
* the shard's cached URL and retry once — then surfaces whatever the second attempt does. Any OTHER
|
|
880
|
+
* typed `HelipodError` (an OCC conflict, a validation failure, …) means a LIVE owner answered
|
|
881
|
+
* DEFINITIVELY and is re-thrown unchanged, exactly as before.
|
|
882
|
+
*
|
|
883
|
+
* Task 3 (read-your-own-writes): `/_fleet/run`'s response carries the write's `commitTs`
|
|
884
|
+
* (stringified — bigints don't survive `JSON.stringify`). If a `ReplicaTailer` has been attached
|
|
885
|
+
* via `attachTailer()` (this node is a fleet SYNC node reading off a local replica), `forward()`
|
|
886
|
+
* waits for that replica's watermark to reach `commitTs` before resolving — otherwise a client that
|
|
887
|
+
* just wrote through this node could immediately read its own write's absence off a replica that
|
|
888
|
+
* hasn't caught up yet. `promote()` also releases any pending wait: once this node becomes the
|
|
889
|
+
* writer, replica catch-up is no longer the right thing to block on.
|
|
890
|
+
*/
|
|
891
|
+
|
|
892
|
+
interface WriteForwarderOptions {
|
|
893
|
+
/** Admin bearer token — the `/_fleet/run` endpoint authenticates with the deployment admin key. */
|
|
894
|
+
adminKey: string;
|
|
895
|
+
/** This node's own advertised URL (recorded on the lease when/if it becomes the writer). */
|
|
896
|
+
selfUrl: string;
|
|
897
|
+
}
|
|
898
|
+
/**
|
|
899
|
+
* The narrow seam `WriteForwarder` needs from `ReplicaTailer` — declared locally (rather than
|
|
900
|
+
* typing `attachTailer` as `ReplicaTailer` itself) so tests can pass a lightweight structural stub
|
|
901
|
+
* instead of standing up a real tailer (which needs a live Postgres primary + replica store).
|
|
902
|
+
* A real `ReplicaTailer` satisfies this trivially since it's a strict subset of its public API.
|
|
903
|
+
*/
|
|
904
|
+
type ReplicaWaiter = Pick<ReplicaTailer, "waitFor" | "release">;
|
|
905
|
+
declare class WriteForwarder implements WriteRouter {
|
|
906
|
+
private readonly lease;
|
|
907
|
+
private readonly opts;
|
|
908
|
+
private tailer;
|
|
909
|
+
/** Guard each distinct malformed-response warning to once per process (independently — an absent
|
|
910
|
+
* commitTs and an unparseable one are different failure modes and each deserves its own log
|
|
911
|
+
* line) so a bad/old writer response doesn't spam the log on every subsequent forwarded write. */
|
|
912
|
+
private warnedMissingCommitTs;
|
|
913
|
+
private warnedUnparseableCommitTs;
|
|
914
|
+
/** Per-shard writer-URL cache (B2b, T2): populated on first forward for a shard, refreshed on a
|
|
915
|
+
* POST failure (transport error OR a not-the-owner rejection) and retried once. A node normally
|
|
916
|
+
* forwards the same shard repeatedly, so caching avoids a `shard_leases` read on every call. */
|
|
917
|
+
private readonly writerUrlCache;
|
|
918
|
+
constructor(lease: LeaseManager, opts: WriteForwarderOptions);
|
|
919
|
+
/** Attach the local `ReplicaTailer` this node reads off, enabling the read-your-own-writes wait
|
|
920
|
+
* in `forward()`. Fleet WRITER nodes never call this — they have no replica to wait on. */
|
|
921
|
+
attachTailer(t: ReplicaWaiter): void;
|
|
922
|
+
/** Called on promotion (sync → writer-ish). Releases any read-your-own-writes wait in flight —
|
|
923
|
+
* this node no longer serves reads off a replica for its own forwarded writes, so waiting on
|
|
924
|
+
* catch-up would only add latency. `isLocalWriter` itself needs no flip here: it already reads
|
|
925
|
+
* the live `LeaseManager` state directly (see below), which promotion's `tryAcquire` calls
|
|
926
|
+
* update as a side effect.
|
|
927
|
+
*
|
|
928
|
+
* NOTE (Fleet B3): a HYBRID promotion (multi-writer) does NOT call this — a hybrid keeps its
|
|
929
|
+
* replica + tailer running past promotion and STILL serves reads (and forwarded writes to shards
|
|
930
|
+
* it doesn't own) off the replica, so releasing the RYOW waits would be wrong. Only the
|
|
931
|
+
* single-writer failover promotion (`promoteFleetNode`, no replica read path afterward) calls it. */
|
|
932
|
+
promote(): void;
|
|
933
|
+
/**
|
|
934
|
+
* Fleet B3 (D2) — the HYBRID own-commit RYOW gate. Wired as the runtime's `beforeNotify` drain
|
|
935
|
+
* hook: awaited before a LOCALLY-committed mutation's subscription re-runs fire, so those re-runs
|
|
936
|
+
* (which read this node's replica, via the hybrid query path) don't observe the commit's absence on
|
|
937
|
+
* a replica that hasn't applied it yet. Delegates to the SAME attached `ReplicaTailer.waitFor` the
|
|
938
|
+
* forwarded-write RYOW uses (`attachTailer`), so local and forwarded writes share one catch-up
|
|
939
|
+
* primitive. No-op when no tailer is attached (a single-writer node with no replica, or before the
|
|
940
|
+
* tailer exists) or when nothing committed (`0n`). A timeout is swallowed — reactivity is
|
|
941
|
+
* best-effort; the read path stays correct regardless (the re-run just fires against a replica that
|
|
942
|
+
* is still catching up, no worse than the pre-gate behavior).
|
|
943
|
+
*/
|
|
944
|
+
waitForReplica(commitTs: bigint): Promise<void>;
|
|
945
|
+
/**
|
|
946
|
+
* True iff this node currently holds `shardId`'s write lease (B2b, D1: per-shard membership, not
|
|
947
|
+
* a whole-node binary role). `LeaseManager.currentEpoch(shardId) !== null` is the SAME live
|
|
948
|
+
* held-set accessor `relinquish()` uses as its own idempotency check (`node.ts`) — so this can
|
|
949
|
+
* never disagree with what the commit guard / relinquish dispatcher consider "held". Consulted
|
|
950
|
+
* fresh on every call (never cached): a shard acquired or relinquished mid-flight (balancer
|
|
951
|
+
* rebalance, failover, promotion) takes effect on the very next mutation.
|
|
952
|
+
*/
|
|
953
|
+
isLocalWriter(shardId?: ShardId): boolean;
|
|
954
|
+
forward(kind: "mutation" | "action", path: string, args: JSONValue, identity: string | null, shardId?: ShardId, dedup?: {
|
|
955
|
+
clientId: string;
|
|
956
|
+
seq: number;
|
|
957
|
+
}): Promise<{
|
|
958
|
+
value: JSONValue;
|
|
959
|
+
commitTs?: number;
|
|
960
|
+
shardId?: string;
|
|
961
|
+
replay?: ClientReplay;
|
|
962
|
+
}>;
|
|
963
|
+
/** Discover shard `shardId`'s current writer URL — cached after the first read. */
|
|
964
|
+
private writerUrlFor;
|
|
965
|
+
/** Force a fresh `shard_leases` read for `shardId`, overwriting any cached URL — the refresh half
|
|
966
|
+
* of `writerUrlFor`'s cache, also used directly by `forward()`'s retry-once path. */
|
|
967
|
+
private refreshWriterUrlFor;
|
|
968
|
+
private post;
|
|
969
|
+
/** Waits for the local replica to observe `commitTsStr`, when a tailer is attached. No-op on a
|
|
970
|
+
* fleet WRITER node (no tailer attached) or when the write committed nothing (`0`/absent). */
|
|
971
|
+
private waitForReplicaCatchUp;
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
/**
|
|
975
|
+
* `ShardLeaseBalancer` — the deterministic, coordinator-free shard placement loop (B2b, D3).
|
|
976
|
+
*
|
|
977
|
+
* Every fleet node (writer OR sync) runs one. Each ~2s beat:
|
|
978
|
+
* 1. HEARTBEAT this node's `fleet_nodes` presence row (its liveness signal — the balancer is the
|
|
979
|
+
* presence heartbeat for a shardless SYNC node, which has no `shard_leases` row and no
|
|
980
|
+
* LeaseMonitor; a writer node's LeaseMonitor probe also heartbeats presence, redundantly and
|
|
981
|
+
* harmlessly).
|
|
982
|
+
* 2. READ the live node set (`fleet_nodes` ∪ live `shard_leases` holders) and compute this node's
|
|
983
|
+
* rendezvous TARGET shards — the shards HRW assigns to `myUrl` over that set. Every node derives
|
|
984
|
+
* the same assignment from the same rows (`rendezvous.ts`), so there is nothing to negotiate.
|
|
985
|
+
* 3. If this node is a pure SYNC node (`!isWriterish()`) and any NON-default target is currently
|
|
986
|
+
* acquirable (orphaned/expired/missing) — a peer released it, or its holder died — request the
|
|
987
|
+
* shipped whole-node PROMOTION (sync → writer-ish); a promoted node then acquires on the next
|
|
988
|
+
* steps. (The DEFAULT shard has its own shipped election path — `LeaseManager.acquireLoop` — so
|
|
989
|
+
* it is not a promotion trigger here; that avoids a redundant second promotion driver racing the
|
|
990
|
+
* election.)
|
|
991
|
+
* 4. Writer-ish node: ACQUIRE any target shard it does not hold that is acquirable (orphaned/
|
|
992
|
+
* expired/missing). This is failover — un-damped and prompt; it NEVER touches a live non-target
|
|
993
|
+
* holder's row (advisory locks + fencing already guarantee at most one holder, so acquisition is
|
|
994
|
+
* always safe).
|
|
995
|
+
* 5. Writer-ish node, under DAMPING (live set identical for ≥2 consecutive beats): gracefully
|
|
996
|
+
* RELEASE any held shard NOT in its target set — a point-in-time self-fence (`releaseShard`, wired
|
|
997
|
+
* in `node.ts`) that hands the shard to its rightful owner, which acquires it on its own next
|
|
998
|
+
* beat (~2s), no TTL wait, no failover event. Damping keeps a flapping membership from thrashing
|
|
999
|
+
* placement; acquisition (step 4) is intentionally NOT damped so a real death recovers promptly.
|
|
1000
|
+
*
|
|
1001
|
+
* The balancer owns ZERO SQL beyond the three lease reads — all acquire/release/promote effects are
|
|
1002
|
+
* injected thunks (`node.ts` wires them over `LeaseManager`/the runtime), so this class is a pure
|
|
1003
|
+
* orchestrator and unit-testable with lightweight fakes.
|
|
1004
|
+
*/
|
|
1005
|
+
|
|
1006
|
+
/** The narrow slice of `LeaseManager` the balancer reads — a structural interface so tests can pass a
|
|
1007
|
+
* fake, and so the balancer never reaches past these three reads into the lease's write surface. */
|
|
1008
|
+
interface BalancerLease {
|
|
1009
|
+
/** Upsert this node's `fleet_nodes` presence row, extending `expires_at` by the lease TTL. */
|
|
1010
|
+
heartbeatPresence(): Promise<void>;
|
|
1011
|
+
/** Distinct unexpired advertise URLs — `fleet_nodes` ∪ live `shard_leases.writer_url` holders. */
|
|
1012
|
+
liveNodes(): Promise<string[]>;
|
|
1013
|
+
/** Per-shard ownership snapshot: for each existing `shard_leases` row, its `writer_url` (null =
|
|
1014
|
+
* orphaned) and whether it has expired per the DB clock. A shard with NO row is absent from the map
|
|
1015
|
+
* (treated as acquirable). */
|
|
1016
|
+
readShardOwnership(): Promise<Map<ShardId, {
|
|
1017
|
+
writerUrl: string | null;
|
|
1018
|
+
expired: boolean;
|
|
1019
|
+
}>>;
|
|
1020
|
+
}
|
|
1021
|
+
interface ShardLeaseBalancerDeps {
|
|
1022
|
+
lease: BalancerLease;
|
|
1023
|
+
/** This node's advertised URL — its identity in the rendezvous candidate set. */
|
|
1024
|
+
myUrl: string;
|
|
1025
|
+
/** Shard count (the fleet's fixed NUM_SHARDS). Drives the `shardIdList` this balancer ranges over. */
|
|
1026
|
+
numShards: number;
|
|
1027
|
+
/** Live per-shard held-set membership (`LeaseManager.currentEpoch(shardId) !== null`). */
|
|
1028
|
+
isHeld(shardId: ShardId): boolean;
|
|
1029
|
+
/** Is this node writer-ish (has completed promotion / booted as the writer)? A pure sync node only
|
|
1030
|
+
* heartbeats presence and may request promotion; it never acquires/releases. */
|
|
1031
|
+
isWriterish(): boolean;
|
|
1032
|
+
/** Acquire `shardId`'s lease (+ its per-slot advisory lock), evicting a wedged expired holder if
|
|
1033
|
+
* needed. Returns whether the shard is now held. One tick's attempt — a miss retries next beat. */
|
|
1034
|
+
tryAcquireShard(shardId: ShardId): Promise<boolean>;
|
|
1035
|
+
/** Gracefully release a held shard (point-in-time self-fence under the shard's commit mutex, then
|
|
1036
|
+
* the T3 relinquish-unwind) so its rightful owner can acquire it. */
|
|
1037
|
+
releaseShard(shardId: ShardId): Promise<void>;
|
|
1038
|
+
/** Run the shipped whole-node promotion (sync → writer-ish). Idempotent at the node level. */
|
|
1039
|
+
requestPromotion(): Promise<void>;
|
|
1040
|
+
/**
|
|
1041
|
+
* Fleet B3, D3 (effectively-once forwarding): reclaim `fleet_idempotency` rows older than the
|
|
1042
|
+
* TTL — a cheap indexed delete, run on every WRITER-ish beat (see `LeaseManager.sweepIdempotency`).
|
|
1043
|
+
* Optional so an older/stub `ShardLeaseBalancerDeps` (and every existing balancer test) needs no
|
|
1044
|
+
* change; a pure sync node never has this called regardless (see `tick()`).
|
|
1045
|
+
*/
|
|
1046
|
+
sweepIdempotency?(): Promise<void>;
|
|
1047
|
+
/**
|
|
1048
|
+
* Multi-writer scale-out mode (B2b, D3 — the "writer nodes vs sync nodes" scaling knob, off by
|
|
1049
|
+
* default). When OFF (the default, and the shipped single-writer behavior): this node's target set is
|
|
1050
|
+
* EVERY shard (the sole writer owns them all), the balancer NEVER gracefully releases a live-held
|
|
1051
|
+
* shard to a peer, and a pure sync node is NEVER auto-promoted by rendezvous — additional nodes stay
|
|
1052
|
+
* READ REPLICAS (sync), reading off their local replica whose tailer sees every shard's commits.
|
|
1053
|
+
* Failover acquisition (picking up an orphaned/expired shard) and presence heartbeating are ON
|
|
1054
|
+
* regardless. When ON: full rendezvous distribution — each live node owns and writes its HRW share,
|
|
1055
|
+
* peers gracefully release a newcomer's share, and a sync node promotes to claim an orphaned share.
|
|
1056
|
+
* (Enabling it makes a joining node a CO-WRITER, whose own subscriptions do not yet receive
|
|
1057
|
+
* cross-writer reactivity for shards it doesn't hold — that is T6's proving ground.)
|
|
1058
|
+
*/
|
|
1059
|
+
multiWriter?: boolean;
|
|
1060
|
+
/** Beat interval (ms). Default 2000. */
|
|
1061
|
+
beatMs?: number;
|
|
1062
|
+
/** Structured-log seam for a tick error, defaults to `console.error`. */
|
|
1063
|
+
log?: (msg: string) => void;
|
|
1064
|
+
}
|
|
1065
|
+
declare class ShardLeaseBalancer {
|
|
1066
|
+
private readonly deps;
|
|
1067
|
+
private readonly shards;
|
|
1068
|
+
private readonly multiWriter;
|
|
1069
|
+
private readonly beatMs;
|
|
1070
|
+
private readonly log;
|
|
1071
|
+
private timer;
|
|
1072
|
+
private stopped;
|
|
1073
|
+
private running;
|
|
1074
|
+
/** Canonical (sorted, comma-joined) live set from the PREVIOUS beat — damping compares against it:
|
|
1075
|
+
* a RELEASE only fires when the current live set equals this (identical for ≥2 consecutive beats). */
|
|
1076
|
+
private previousLiveSet;
|
|
1077
|
+
constructor(deps: ShardLeaseBalancerDeps);
|
|
1078
|
+
/** Begin the periodic beat. The FIRST beat fires after `beatMs` (never synchronously) — prompt boot
|
|
1079
|
+
* acquisition is done explicitly via `acquireTargetsNow()`, not the periodic loop. Idempotent. */
|
|
1080
|
+
start(): void;
|
|
1081
|
+
stop(): void;
|
|
1082
|
+
/** Resolve the live candidate set for a beat: the DB's live nodes ∪ this node itself (a node is
|
|
1083
|
+
* always live to itself — belt-and-braces so its own boot never depends on its presence row having
|
|
1084
|
+
* landed/propagated yet). Returns both the URL array (rendezvous input) and its canonical form. */
|
|
1085
|
+
private liveCandidates;
|
|
1086
|
+
/** This node's target shards. In single-writer mode (the default) the sole writer's targets are ALL
|
|
1087
|
+
* shards — byte-identical to B2a's acquire-all — so a joining node never steals a share. In
|
|
1088
|
+
* multi-writer mode it is this node's rendezvous (HRW) share over `liveUrls`. */
|
|
1089
|
+
private targetsFor;
|
|
1090
|
+
/** A shard is acquirable iff it has no row (absent from the map), is orphaned (`writer_url NULL`), or
|
|
1091
|
+
* has expired — i.e. NOT held live by a peer. Acquisition only ever targets these; a live non-target
|
|
1092
|
+
* holder is never fenced by the balancer (that's failover's job). */
|
|
1093
|
+
private acquirable;
|
|
1094
|
+
/**
|
|
1095
|
+
* Un-damped acquire pass over this node's CURRENT rendezvous targets — called at writer boot and on
|
|
1096
|
+
* promotion so the node holds its share PROMPTLY (before the ready line / before the first periodic
|
|
1097
|
+
* beat), rather than waiting out a 2s tick. In a single-node fleet the target set is EVERY shard, so
|
|
1098
|
+
* this acquires all N — byte-identical steady state to B2a's acquire-all. Seeds `previousLiveSet` so
|
|
1099
|
+
* the first periodic beat treats the boot membership as the damping baseline.
|
|
1100
|
+
*/
|
|
1101
|
+
acquireTargetsNow(): Promise<void>;
|
|
1102
|
+
/** One balancer beat (see the class doc for the five steps). Never throws — a tick error is logged
|
|
1103
|
+
* and the loop continues; the next beat retries against fresh state. */
|
|
1104
|
+
tick(): Promise<void>;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
/**
|
|
1108
|
+
* Rendezvous (highest-random-weight) hashing for the shard balancer (B2b, D3).
|
|
1109
|
+
*
|
|
1110
|
+
* Every fleet node runs the SAME pure function over the SAME live-node set (discovered from
|
|
1111
|
+
* `fleet_nodes` ∪ live `shard_leases` holders — see `LeaseManager.liveNodes`), so all nodes derive
|
|
1112
|
+
* an identical shard→owner assignment with NOTHING to negotiate. For each shard we compute a stable
|
|
1113
|
+
* 64-bit weight `hash(shardId, advertiseUrl)` per candidate node and pick the max — the owner. When
|
|
1114
|
+
* the membership set changes, HRW moves ONLY the shards whose max-weight node changed (minimality),
|
|
1115
|
+
* so a join/leave reshuffles a bounded, minimal set of shards rather than the whole ring.
|
|
1116
|
+
*
|
|
1117
|
+
* The weight is FNV-1a-64 over the UTF-8 bytes of `${shardId}${advertiseUrl}` — the same
|
|
1118
|
+
* hashing discipline `@helipod/id-codec`'s jump-hash router uses (FNV-1a-64 → a well-mixed 64-bit
|
|
1119
|
+
* key), reimplemented locally rather than importing that package's private helper so the rendezvous
|
|
1120
|
+
* weight function is self-contained and its exact bytes are pinned here. It MUST stay a pure function
|
|
1121
|
+
* of the two strings and identical across nodes/releases — every node's placement decision depends on
|
|
1122
|
+
* byte-for-byte agreement.
|
|
1123
|
+
*/
|
|
1124
|
+
|
|
1125
|
+
/**
|
|
1126
|
+
* The rendezvous weight of `advertiseUrl` for `shardId` — a stable 64-bit value. A space separates
|
|
1127
|
+
* the two fields so `("s1","x")` and `("s","1x")` can never collide into the same input — safe
|
|
1128
|
+
* because neither `shardId` nor `advertiseUrl` ever contains a space (shard ids are short identifiers
|
|
1129
|
+
* like `"default"`/`"s1"`; advertise URLs are `scheme://host:port` with no whitespace).
|
|
1130
|
+
*/
|
|
1131
|
+
declare function rendezvousWeight(shardId: ShardId, advertiseUrl: string): bigint;
|
|
1132
|
+
/**
|
|
1133
|
+
* The owner of `shardId` under rendezvous hashing over `liveUrls`: the node with the maximum weight,
|
|
1134
|
+
* ties broken by the lexicographically-smaller URL (so the choice is fully deterministic even in the
|
|
1135
|
+
* astronomically unlikely event of a 64-bit weight collision). Returns `""` for an empty candidate
|
|
1136
|
+
* set (no live node — the caller always includes at least its own URL, so this is defensive only).
|
|
1137
|
+
* Pure and allocation-light; identical output on every node given the same `liveUrls` (order-
|
|
1138
|
+
* independent — the max/tie-break does not depend on the input array's order).
|
|
1139
|
+
*/
|
|
1140
|
+
declare function rendezvousOwner(shardId: ShardId, liveUrls: readonly string[]): string;
|
|
1141
|
+
|
|
1142
|
+
/**
|
|
1143
|
+
* A `DocStore` that forwards every call to a swappable inner delegate, so a replica-to-primary
|
|
1144
|
+
* promotion (Task 4) can become a single atomic pointer swap instead of a coordinated teardown of
|
|
1145
|
+
* every in-flight caller.
|
|
1146
|
+
*
|
|
1147
|
+
* Atomicity contract: each method reads `this.delegate` exactly once, at call entry, into a local
|
|
1148
|
+
* binding, and completes its whole operation against that snapshot. A call already in progress when
|
|
1149
|
+
* `swapTo()` runs finishes against the OLD delegate; a call started after `swapTo()` returns sees
|
|
1150
|
+
* the NEW one. There is no instant where a single call is torn between two delegates. This holds for
|
|
1151
|
+
* the async generators too (`index_scan`/`load_documents`): the delegate is captured before the
|
|
1152
|
+
* first `yield`, so a scan already draining rows from the old store keeps draining the old store to
|
|
1153
|
+
* completion even if `swapTo()` runs concurrently.
|
|
1154
|
+
*
|
|
1155
|
+
* `close()` semantics: closes only the CURRENT delegate (whatever `this.delegate` is at the moment
|
|
1156
|
+
* `close()` is called). It does NOT close a delegate that was swapped out by an earlier `swapTo()` —
|
|
1157
|
+
* this class has no memory of former delegates once replaced. Closing a swapped-out (e.g. demoted
|
|
1158
|
+
* primary, or promoted-from replica) delegate is the lifecycle owner's responsibility (Task 4
|
|
1159
|
+
* decides when that's safe — e.g. after draining its in-flight callers), not this wrapper's.
|
|
1160
|
+
*/
|
|
1161
|
+
declare class SwitchableDocStore implements DocStore {
|
|
1162
|
+
private delegate;
|
|
1163
|
+
constructor(initial: DocStore);
|
|
1164
|
+
/** Atomically repoint all future calls at `next`. Does not touch the outgoing delegate. */
|
|
1165
|
+
swapTo(next: DocStore): void;
|
|
1166
|
+
/** The delegate current calls are being forwarded to. */
|
|
1167
|
+
current(): DocStore;
|
|
1168
|
+
setupSchema(options?: SchemaSetupOptions): Promise<void>;
|
|
1169
|
+
write(documents: readonly DocumentLogEntry[], indexUpdates: readonly IndexWrite[], conflictStrategy: ConflictStrategy, shardId?: ShardId$1): Promise<void>;
|
|
1170
|
+
commitWrite(documents: readonly DocumentLogEntry[], indexUpdates: readonly IndexWrite[], shardId?: ShardId$1, opts?: {
|
|
1171
|
+
meta?: Record<string, string>;
|
|
1172
|
+
}): Promise<bigint>;
|
|
1173
|
+
commitWriteBatch(units: readonly CommitUnit[], shardId?: ShardId$1): Promise<bigint[]>;
|
|
1174
|
+
/** Forwards to the CURRENT delegate at call entry, same atomicity contract as every other
|
|
1175
|
+
* method here — but note the resulting registration is NOT itself re-forwarded on a later
|
|
1176
|
+
* `swapTo()`: the guard lands on whichever concrete store was `this.delegate` at the moment
|
|
1177
|
+
* `addCommitGuard` was called, and stays registered there even after the switchable repoints
|
|
1178
|
+
* elsewhere. In practice fleet code (`node.ts`) always calls `addCommitGuard` on the concrete
|
|
1179
|
+
* `PostgresDocStore` directly (never through this wrapper), so this pass-through exists purely
|
|
1180
|
+
* for `DocStore` interface conformance. */
|
|
1181
|
+
addCommitGuard(guard: (q: any, units: readonly CommitGuardUnit[], shardId: ShardId$1) => void | Promise<void>): () => void;
|
|
1182
|
+
get(id: InternalDocumentId, readTimestamp?: bigint): Promise<LatestDocument | null>;
|
|
1183
|
+
index_scan(indexId: string, tableId: string, readTimestamp: bigint, interval: Interval, order: Order, limit?: number): AsyncGenerator<readonly [Uint8Array, LatestDocument]>;
|
|
1184
|
+
load_documents(range: TimestampRange, order: Order, limit?: number): AsyncGenerator<DocumentLogEntry>;
|
|
1185
|
+
previous_revisions(queries: readonly PrevRevQuery[]): Promise<Map<string, DocumentLogEntry>>;
|
|
1186
|
+
scan(tableId: string, readTimestamp?: bigint): Promise<LatestDocument[]>;
|
|
1187
|
+
count(tableId: string): Promise<number>;
|
|
1188
|
+
maxTimestamp(): Promise<bigint>;
|
|
1189
|
+
getGlobal(key: string): Promise<JSONValue | null>;
|
|
1190
|
+
writeGlobal(key: string, value: JSONValue): Promise<void>;
|
|
1191
|
+
writeGlobalIfAbsent(key: string, value: JSONValue): Promise<boolean>;
|
|
1192
|
+
getClientVerdict(identity: string, clientId: string, seq: number): Promise<ClientVerdictRecord | null>;
|
|
1193
|
+
getClientFloor(identity: string, clientId: string): Promise<number | null>;
|
|
1194
|
+
recordClientVerdict(identity: string, clientId: string, seq: number, record: ClientVerdictWrite): Promise<void>;
|
|
1195
|
+
updateClientVerdictValue(identity: string, clientId: string, seq: number, value: JSONValue): Promise<void>;
|
|
1196
|
+
pruneClientMutations(identity: string, clientId: string, opts: {
|
|
1197
|
+
ackedThrough?: number;
|
|
1198
|
+
ttlBeforeMs?: number;
|
|
1199
|
+
}): Promise<{
|
|
1200
|
+
prunedThroughSeq: number;
|
|
1201
|
+
}>;
|
|
1202
|
+
sweepExpiredClientMutations(beforeMs: number): Promise<{
|
|
1203
|
+
deletedCount: number;
|
|
1204
|
+
}>;
|
|
1205
|
+
close(): void | Promise<void>;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
/** The filename of a sync node's local embedded replica, under the serve data dir. */
|
|
1209
|
+
declare const REPLICA_DB_FILENAME = "fleet-replica.db";
|
|
1210
|
+
/** Default number of shards a fleet node runs (B2a). T5 owns the persist-once/env/`HELIPOD_FLEET_
|
|
1211
|
+
* SHARDS` story; this task threads it as a plain parameter (`prepareFleetNode`'s `numShards` dep),
|
|
1212
|
+
* defaulting here. The one node holds ALL N shard leases, commits in parallel across them (per-shard
|
|
1213
|
+
* commit-connection pool + per-shard `ShardedTransactor` mutexes), and F = min over the N frontiers. */
|
|
1214
|
+
declare const DEFAULT_NUM_SHARDS = 8;
|
|
1215
|
+
/** A point-in-time frontier-lag reading for the health endpoint (D5): the fleet-wide fenced frontier
|
|
1216
|
+
* (`min(frontier_ts)` across all shard rows), how long (wall-clock ms) it has been stuck at that
|
|
1217
|
+
* value, and which shard is holding it there. */
|
|
1218
|
+
interface FrontierStats {
|
|
1219
|
+
frontier: bigint;
|
|
1220
|
+
lagMs: number;
|
|
1221
|
+
pinningShard: ShardId;
|
|
1222
|
+
}
|
|
1223
|
+
/**
|
|
1224
|
+
* Fleet frontier-lag monitor + (writer-only) idle-shard closer (B2a, D5). One runs per fleet node:
|
|
1225
|
+
*
|
|
1226
|
+
* - On the WRITER (`closeIdle: true`) each beat first CLOSES idle shards — `lease.closeIdleFrontiers`
|
|
1227
|
+
* allocates one `nextval` and advances every held shard whose frontier lags it — so an idle shard
|
|
1228
|
+
* can't pin `F = min(frontier_ts)` below the live commit position for more than a beat. A local
|
|
1229
|
+
* commit also schedules a coalesced beat (~10ms) so F reacts promptly to writes on sibling shards.
|
|
1230
|
+
* - On EITHER role each beat then READS all shard frontiers to compute `min` + the pinning shard and
|
|
1231
|
+
* track how long `min` has been stuck (wall-clock since it last advanced) → the `FrontierStats`
|
|
1232
|
+
* the health endpoint reports. A lag past `lagWarnMs` logs a warning naming the pinning shard
|
|
1233
|
+
* (once per stall, reset when it recovers), the console signal D5 asks for.
|
|
1234
|
+
*
|
|
1235
|
+
* A sync node runs it read-only (`closeIdle: false`) — it holds no leases, so it must NOT allocate
|
|
1236
|
+
* `nextval` (only the writer drives the frontier); it just observes whether the fleet frontier is
|
|
1237
|
+
* advancing.
|
|
1238
|
+
*/
|
|
1239
|
+
declare class FrontierMonitor {
|
|
1240
|
+
private readonly lease;
|
|
1241
|
+
private readonly opts;
|
|
1242
|
+
private timer;
|
|
1243
|
+
private coalesceTimer;
|
|
1244
|
+
private stopped;
|
|
1245
|
+
private running;
|
|
1246
|
+
private cached;
|
|
1247
|
+
/** The last-observed `min(frontier_ts)` and the wall-clock time it was first seen — lag is the age
|
|
1248
|
+
* of the CURRENT min (how long it's been stuck), so both reset whenever min advances. */
|
|
1249
|
+
private lastMin;
|
|
1250
|
+
private minSinceMs;
|
|
1251
|
+
private warned;
|
|
1252
|
+
/** D4 — replica-lag tracking: wall-clock time this node's OWN tailer watermark was first observed
|
|
1253
|
+
* BEHIND the fleet frontier F, or `null` while it's caught up. Mirrors `minSinceMs`/`lastMin`'s
|
|
1254
|
+
* "age of the current stuck condition" shape, but for the replica catch-up gap rather than F's own
|
|
1255
|
+
* advance. Reset the instant `wm` reaches `F` again — a transient blip re-arms silently. */
|
|
1256
|
+
private replicaLagSinceMs;
|
|
1257
|
+
private replicaLagWarned;
|
|
1258
|
+
constructor(lease: LeaseManager, opts: {
|
|
1259
|
+
closeIdle: boolean;
|
|
1260
|
+
/** Per-shard commit-mutex seam the idle-frontier closer runs each bump under (see
|
|
1261
|
+
* `closeIdleFrontiers`). Required when `closeIdle` is true; ignored otherwise. */
|
|
1262
|
+
runExclusiveOnShard?: TryRunExclusiveOnShard;
|
|
1263
|
+
beatMs?: number;
|
|
1264
|
+
coalesceMs?: number;
|
|
1265
|
+
lagWarnMs?: number;
|
|
1266
|
+
now?: () => number;
|
|
1267
|
+
warn?: (msg: string) => void;
|
|
1268
|
+
/**
|
|
1269
|
+
* D4 — this node's own local replica-tailer watermark, when it has one: a sync node's
|
|
1270
|
+
* `ReplicaTailer`, or (in the hybrid/multi-writer regime) a writer's own read-replica tailer.
|
|
1271
|
+
* Omitted for a non-hybrid writer (it reads the primary directly — no replica, no lag to track).
|
|
1272
|
+
* Compared against the fleet frontier `F = min(frontier_ts)` each beat; `wm` falling more than
|
|
1273
|
+
* `lagWarnMs` behind `F` logs ONE operator warning naming this node's replica (`lease.advertiseUrl`
|
|
1274
|
+
* — there is only ever one replica per node, so naming the NODE identifies it), silent below the
|
|
1275
|
+
* threshold and while caught up. Same shape/once-ness as the pinning-shard warning above; applies
|
|
1276
|
+
* identically whether this monitor is a sync node's or a hybrid writer's (same `beat()` code path).
|
|
1277
|
+
*/
|
|
1278
|
+
tailerWatermark?: () => bigint;
|
|
1279
|
+
/**
|
|
1280
|
+
* T3.5 (the fleet-connections bench's notify-diagnosis fix): fired at most once per beat, only
|
|
1281
|
+
* when this beat's `closeIdleFrontiers`/`bumpOrphanFrontiers` (or a concurrent commit) actually
|
|
1282
|
+
* pushed the fleet ceiling `F = min(frontier_ts)` past what the PRIOR beat observed. Neither
|
|
1283
|
+
* closer NOTIFYs on its own bare `UPDATE` — without this, an idle-shard close that finally
|
|
1284
|
+
* un-pins F is invisible to every `ReplicaTailer`'s LISTEN, and delivery silently falls through
|
|
1285
|
+
* to the tailer's `DEFAULT_POLL_MS` (1000ms) fallback even though NOTIFY itself is wired and
|
|
1286
|
+
* fires within ~1ms of the real commit (see `.superpowers/sdd/notify-diagnosis.md`). Omitted
|
|
1287
|
+
* entirely on a sync node (`closeIdle: false`) — only the writer's own beat performs the
|
|
1288
|
+
* advancing work, so only it needs to announce it. The callback owns its own failure handling
|
|
1289
|
+
* (fire-and-forget, matching `NotifyingFanoutAdapter.publish`'s own NOTIFY) — a rejected notify
|
|
1290
|
+
* must not, and structurally cannot, break the beat.
|
|
1291
|
+
*/
|
|
1292
|
+
notifyOnAdvance?: (ceiling: bigint) => void;
|
|
1293
|
+
});
|
|
1294
|
+
private get now();
|
|
1295
|
+
start(): void;
|
|
1296
|
+
/** Schedule a single coalesced beat `coalesceMs` out — called on each local commit so an idle
|
|
1297
|
+
* sibling shard un-pins F within ~10ms instead of waiting out the periodic beat. Multiple commits
|
|
1298
|
+
* inside the window collapse onto one beat. */
|
|
1299
|
+
triggerCoalesced(): void;
|
|
1300
|
+
private beat;
|
|
1301
|
+
/** The most recent frontier reading, or null if no beat has completed yet. `lagMs` is recomputed
|
|
1302
|
+
* against the current clock so a caller between beats still sees a fresh age. */
|
|
1303
|
+
stats(): FrontierStats | null;
|
|
1304
|
+
stop(): void;
|
|
1305
|
+
}
|
|
1306
|
+
/**
|
|
1307
|
+
* D4 — a generic async-op serializer: returns a `run(op)` function where every queued `op` executes
|
|
1308
|
+
* only after the PREVIOUS one has settled (resolved OR rejected — `.then(op, op)`, so one op
|
|
1309
|
+
* rejecting never wedges the chain for the next). Callers that fire-and-forget a burst of overlapping
|
|
1310
|
+
* async calls (see `startFleetNode`'s `startDriversChained`/`stopDriversOnlyChained`) get a
|
|
1311
|
+
* deterministic EXECUTION order matching the ISSUE order, even when the callee itself is merely
|
|
1312
|
+
* idempotent (idempotence alone doesn't prevent a later-issued-but-faster call from resolving before
|
|
1313
|
+
* an earlier one, which is the race this closes). Each call to `createAsyncChain()` returns an
|
|
1314
|
+
* independent chain — callers needing one shared serialization point construct exactly one instance
|
|
1315
|
+
* and route every op through the same returned function.
|
|
1316
|
+
*/
|
|
1317
|
+
declare function createAsyncChain(): (op: () => Promise<void>) => Promise<void>;
|
|
1318
|
+
/**
|
|
1319
|
+
* Bounded-writer-session timeouts (Fenced Frontier B1, D4) applied to every fleet node's pinned
|
|
1320
|
+
* Postgres connection — the single `NodePgClient` `prepareFleetNode` builds is the writer-capable one
|
|
1321
|
+
* (a sync node's same connection becomes the writer's on promotion via `pgStore.setWritable()`), so
|
|
1322
|
+
* it is bounded from boot. `idle_in_transaction=5s` kills a transaction a wedged writer leaves open
|
|
1323
|
+
* mid-commit (releasing the row lock a fencer's `evictExpired` needs); `statement=10s` caps any single
|
|
1324
|
+
* runaway statement. A NON-fleet single-node `serve`/`dev` (`makeStore` in `packages/cli`) constructs
|
|
1325
|
+
* `NodePgClient` WITHOUT this option and stays unbounded — see that call site.
|
|
1326
|
+
*/
|
|
1327
|
+
declare const FLEET_WRITER_SESSION_TIMEOUTS: {
|
|
1328
|
+
idleInTransactionMs: number;
|
|
1329
|
+
statementMs: number;
|
|
1330
|
+
};
|
|
1331
|
+
declare function fleetApplicationName(advertiseUrl: string): string;
|
|
1332
|
+
/**
|
|
1333
|
+
* Whether this deployment runs in MULTI-WRITER mode (`HELIPOD_FLEET_MULTI_WRITER`), read in ONE
|
|
1334
|
+
* place so `prepareFleetNode` (which decides the runtime store/queryStore wiring) and
|
|
1335
|
+
* `startFleetNode` (which decides the tailer/promotion wiring) can never disagree about a node's
|
|
1336
|
+
* shape. Multi-writer IS the hybrid regime (Fleet B3): every writer-ish node keeps a local replica
|
|
1337
|
+
* and serves queries from it while committing to the primary. Off (the production default) → the
|
|
1338
|
+
* shipped single-writer topology: the sole writer holds every shard and reads the primary, additional
|
|
1339
|
+
* nodes are pure read-replica sync nodes — no hybrid machinery anywhere (byte-identical to B2b).
|
|
1340
|
+
*/
|
|
1341
|
+
declare function fleetMultiWriterEnabled(): boolean;
|
|
1342
|
+
/**
|
|
1343
|
+
* Whether group commit (Fleet B4) is enabled (`HELIPOD_GROUP_COMMIT`), read the SAME way as
|
|
1344
|
+
* `fleetMultiWriterEnabled` above — one place, so every `runtimeOptions` construction site in
|
|
1345
|
+
* `prepareFleetNode` reads the identical value. Threaded straight into `createEmbeddedRuntime`
|
|
1346
|
+
* (`EmbeddedRuntimeOptions.groupCommit`), which routes every shard's commits through the two-buffer
|
|
1347
|
+
* stage-then-flush committer loop. Default OFF at Fleet B4/T4 — T5 owns flipping the production
|
|
1348
|
+
* default once the throughput win is E2E-proven; unset/anything else → the byte-identical
|
|
1349
|
+
* single-commit path every shard already runs.
|
|
1350
|
+
*/
|
|
1351
|
+
declare function groupCommitEnabled(): boolean;
|
|
1352
|
+
interface FleetHandles {
|
|
1353
|
+
role(): "sync" | "writer";
|
|
1354
|
+
/** The current writer's URL — the proxy target for public httpActions handled on a sync node. */
|
|
1355
|
+
writerUrl(): Promise<string>;
|
|
1356
|
+
/** Register a callback fired once, when this node is promoted from sync to writer. */
|
|
1357
|
+
onPromoted(cb: () => void): void;
|
|
1358
|
+
/** The current frontier-lag reading (D5 health observability): `min(frontier_ts)` across shards,
|
|
1359
|
+
* how long it's been stuck (ms), and the pinning shard. Null before the first frontier beat, or if
|
|
1360
|
+
* no shard rows exist yet. */
|
|
1361
|
+
frontierStats(): FrontierStats | null;
|
|
1362
|
+
/**
|
|
1363
|
+
* Group-commit counters (Fleet B4, T4 health observability): `EmbeddedRuntime.groupCommitStats()`'s
|
|
1364
|
+
* aggregate reading plus a derived `flushesPerSec` — a rolling delta between successive calls
|
|
1365
|
+
* (`(flushCount now - flushCount at the prior call) / elapsed seconds`), the simplest honest
|
|
1366
|
+
* throughput signal without a dedicated timer. The first call after boot (or after any gap) has no
|
|
1367
|
+
* prior sample, so it reports 0 for that one call. Structurally all-zero when `HELIPOD_GROUP_
|
|
1368
|
+
* COMMIT` is off (the underlying counters are never touched on the single-commit path) — never
|
|
1369
|
+
* null, unlike `frontierStats` (no "before the first beat" gating applies here).
|
|
1370
|
+
*/
|
|
1371
|
+
groupCommitStats(): {
|
|
1372
|
+
lastBatchSize: number;
|
|
1373
|
+
maxBatchSize: number;
|
|
1374
|
+
flushCount: number;
|
|
1375
|
+
flushesPerSec: number;
|
|
1376
|
+
};
|
|
1377
|
+
/** Per-shard ownership (B2b, D1): does THIS node currently hold `shardId`'s write lease? Backs
|
|
1378
|
+
* the `/_fleet/run` single-hop guard (`packages/cli`'s `http-handler.ts`) — delegates straight to
|
|
1379
|
+
* `WriteForwarder.isLocalWriter`, the same live held-set view the executor's own per-shard
|
|
1380
|
+
* router and `relinquish()` consult. */
|
|
1381
|
+
isLocalWriter(shardId: ShardId): boolean;
|
|
1382
|
+
/**
|
|
1383
|
+
* Effectively-once forwarding (Fleet B3, D3): read back `key`'s `fleet_idempotency` row for a
|
|
1384
|
+
* replay decision. Delegates to `LeaseManager.lookupIdempotency` — the SAME control table the
|
|
1385
|
+
* commit guard (`installCommitGuard`) writes into, on the SAME `client`, so a lookup immediately
|
|
1386
|
+
* after a catch always sees a just-committed sibling's row. `packages/cli`'s `/_fleet/run`
|
|
1387
|
+
* handler calls this BOTH before running (SELECT-first) and after catching a unique_violation on
|
|
1388
|
+
* this table (the concurrent-duplicate race's loser re-selecting the winner's row).
|
|
1389
|
+
*/
|
|
1390
|
+
idempotencyLookup(key: string): Promise<IdempotencyReplay | null>;
|
|
1391
|
+
/** Best-effort post-run recording of a forwarded mutation's return value — see
|
|
1392
|
+
* `LeaseManager.recordIdempotencyValue`. */
|
|
1393
|
+
idempotencyRecordValue(key: string, value: JSONValue): Promise<void>;
|
|
1394
|
+
stop(): Promise<void>;
|
|
1395
|
+
}
|
|
1396
|
+
/** The createEmbeddedRuntime option deltas the caller threads through `bootProject`. `store` is the
|
|
1397
|
+
* node's RUNTIME (WRITE) store: the writable Postgres store for a writer, the `SwitchableDocStore`
|
|
1398
|
+
* (over the local replica, until promotion swaps in the Postgres store) for a single-writer sync
|
|
1399
|
+
* node, and — for a HYBRID (multi-writer) node — ALWAYS the Postgres store (mutations commit to the
|
|
1400
|
+
* primary), paired with `queryStore` for the replica-backed read path. */
|
|
1401
|
+
interface FleetRuntimeOptions {
|
|
1402
|
+
store: DocStore;
|
|
1403
|
+
writeRouter: WriteRouter;
|
|
1404
|
+
deferDrivers: boolean;
|
|
1405
|
+
fanoutAdapter?: EmbeddedWriteFanoutAdapter;
|
|
1406
|
+
/** Number of shards this node's runtime runs (B2a). Threaded to `createEmbeddedRuntime`, which
|
|
1407
|
+
* builds ONE `ShardedTransactor` (N per-shard mutexes) over the pooled store instead of the
|
|
1408
|
+
* single-shard `SingleWriterTransactor` when >1 — so cross-shard commits run in parallel. */
|
|
1409
|
+
numShards: number;
|
|
1410
|
+
/**
|
|
1411
|
+
* Fleet B3 hybrid nodes (D1) — the replica-backed QUERY store (the `SwitchableDocStore` over the
|
|
1412
|
+
* local replica). Set ONLY on a hybrid (multi-writer) node: `store` (above) is the primary, where
|
|
1413
|
+
* mutations commit, while queries/subscriptions read this replica store. `createEmbeddedRuntime`
|
|
1414
|
+
* builds a separate query-path transactor + QueryRuntime over it, and routes `observeTimestamp`
|
|
1415
|
+
* (fed by this node's ReplicaTailer post-apply) to the query oracle so a query snapshot never
|
|
1416
|
+
* exceeds the replica watermark. Absent → every query runs against `store` (single-writer/sync/
|
|
1417
|
+
* non-fleet), byte-identical to before B3.
|
|
1418
|
+
*/
|
|
1419
|
+
queryStore?: DocStore;
|
|
1420
|
+
/**
|
|
1421
|
+
* Receipted Outbox (verdict §(c) placement) — the AUTHORITATIVE receipts store the Connect
|
|
1422
|
+
* handshake classifies/prunes against. Set to the PRIMARY (`pgStore`) on a SYNC node so the
|
|
1423
|
+
* handshake never reads the local replica (which carries no `client_mutations`/`client_floors`
|
|
1424
|
+
* receipts → spurious `known:false` resets / false kill-after-commit failures). Threaded straight
|
|
1425
|
+
* into `createEmbeddedRuntime`. Absent → the runtime uses `store` (writer boots, where `store` IS
|
|
1426
|
+
* the primary; and every non-fleet deployment). See `EmbeddedRuntimeOptions.receiptsStore`.
|
|
1427
|
+
*/
|
|
1428
|
+
receiptsStore?: DocStore;
|
|
1429
|
+
/**
|
|
1430
|
+
* Fleet B3 hybrid RYOW (D2) — awaited in the runtime's serial fan-out `drain()` before a local
|
|
1431
|
+
* commit's subscription re-runs, so those re-runs (reading the replica) don't observe the commit's
|
|
1432
|
+
* absence on a replica that hasn't applied it yet. Wired to `forwarder.waitForReplica`. Set only on
|
|
1433
|
+
* a hybrid node. Absent → the drain is byte-identical to before B3.
|
|
1434
|
+
*/
|
|
1435
|
+
beforeNotify?: (commitTs: bigint) => Promise<void>;
|
|
1436
|
+
/**
|
|
1437
|
+
* Fleet B4 (T4): group commit — resolved once via `groupCommitEnabled()` (above) and threaded
|
|
1438
|
+
* uniformly onto every boot shape (writer/sync × single-writer/hybrid). Unset/false →
|
|
1439
|
+
* `createEmbeddedRuntime` builds the byte-identical single-commit path, as shipped.
|
|
1440
|
+
*/
|
|
1441
|
+
groupCommit?: boolean;
|
|
1442
|
+
/**
|
|
1443
|
+
* Triggers D1 — the stable-prefix accessor for `DriverContext.readLog`. In a fleet, N per-shard
|
|
1444
|
+
* commit connections land timestamps out of order, so the log tail is gap-free ONLY below
|
|
1445
|
+
* `min(shard_leases.frontier_ts)` (the fenced fleet frontier). Wired here to exactly that read (the
|
|
1446
|
+
* same `readAllFrontiers`-min the frontier-lag monitor uses), so `readLog` never surfaces — nor
|
|
1447
|
+
* skips — a change above an in-flight gap. Threaded onto every boot shape; consulted only when a
|
|
1448
|
+
* driver runs (post-promotion on a sync node), by which point `store` points at the primary.
|
|
1449
|
+
*/
|
|
1450
|
+
stablePrefix?: () => Promise<bigint | null>;
|
|
1451
|
+
/**
|
|
1452
|
+
* Receipted Outbox — receipts-guard ownership handoff. Always `true` for a fleet node: fleet owns
|
|
1453
|
+
* installing the `clientReceiptsGuard()` exactly-once barrier on the CONCRETE Postgres store in
|
|
1454
|
+
* `armWriter` (before the epoch fence, release-on-re-arm), NOT on `store` — which for a sync node is
|
|
1455
|
+
* a `SwitchableDocStore` the runtime would register the guard on, only for it to silently vanish on
|
|
1456
|
+
* the promotion swapTo (leaving a promoted single-writer node's client mutations un-receipted → a
|
|
1457
|
+
* resent (clientId, seq) re-executes). Threaded to `createEmbeddedRuntime` so its `create()` skips
|
|
1458
|
+
* its own registration. See `EmbeddedRuntimeOptions.externalReceiptsGuard` and `armWriter`.
|
|
1459
|
+
*/
|
|
1460
|
+
externalReceiptsGuard?: boolean;
|
|
1461
|
+
}
|
|
1462
|
+
interface FleetPrep {
|
|
1463
|
+
client: NodePgClient;
|
|
1464
|
+
/** The Postgres store. Writer: the runtime store (writable). Sync: the tail source + promotion
|
|
1465
|
+
* swap target (read-only until promoted) — NOT the runtime store (`runtimeOptions.store` is the
|
|
1466
|
+
* `SwitchableDocStore` over the local replica for a sync node). */
|
|
1467
|
+
pgStore: PostgresDocStore;
|
|
1468
|
+
/** The local file-backed replica the runtime reads queries through, and the switchable wrapper it
|
|
1469
|
+
* points at. Present for a single-writer SYNC boot (runtime store = the switchable) AND for a
|
|
1470
|
+
* HYBRID (multi-writer) boot of EITHER role (the switchable is `runtimeOptions.queryStore`, the
|
|
1471
|
+
* read path beside the primary write store). Absent only for a single-writer WRITER boot (no
|
|
1472
|
+
* replica — it reads the primary). */
|
|
1473
|
+
replica?: SqliteDocStore;
|
|
1474
|
+
switchable?: SwitchableDocStore;
|
|
1475
|
+
/** The on-disk path of `replica`, threaded through to `startFleetNode` so its C7
|
|
1476
|
+
* `reconcileReplicaIdentity` call (deferred there — see that function's doc comment) can rebuild
|
|
1477
|
+
* the file in place if needed. Present whenever `replica` is (sync boot, or any hybrid boot). */
|
|
1478
|
+
replicaPath?: string;
|
|
1479
|
+
lease: LeaseManager;
|
|
1480
|
+
forwarder: WriteForwarder;
|
|
1481
|
+
role: "sync" | "writer";
|
|
1482
|
+
/** The shard count decided at boot — threaded through to `startFleetNode` (acquire-all loop,
|
|
1483
|
+
* all-rows seed, per-shard commit guard, idle closer) and the tailer's `count(*) < N` ready gate. */
|
|
1484
|
+
numShards: number;
|
|
1485
|
+
runtimeOptions: FleetRuntimeOptions;
|
|
1486
|
+
}
|
|
1487
|
+
/**
|
|
1488
|
+
* Decide writer-vs-sync BEFORE the runtime is constructed: one `tryAcquire()` after the lease
|
|
1489
|
+
* table exists. This must run before `bootProject` because the acquire result determines the
|
|
1490
|
+
* store's writability, the fan-out adapter, whether drivers are deferred, and the forwarder's
|
|
1491
|
+
* starting role — all of which are `createEmbeddedRuntime` inputs.
|
|
1492
|
+
*/
|
|
1493
|
+
declare function prepareFleetNode(deps: {
|
|
1494
|
+
databaseUrl: string;
|
|
1495
|
+
advertiseUrl: string;
|
|
1496
|
+
adminKey: string;
|
|
1497
|
+
/** The serve data dir — the sync node's local replica is created at `<dataDir>/fleet-replica.db`. */
|
|
1498
|
+
dataDir: string;
|
|
1499
|
+
/** Lease TTL in ms — the single knob the whole failover clock scales from (see `fleetProbeMs`/
|
|
1500
|
+
* `fleetAcquireRetryMs`). Threaded from serve's fleet config, which reads
|
|
1501
|
+
* `HELIPOD_FLEET_LEASE_TTL_MS` (ops/test tuning; the wedged-writer E2E uses 4000). Default 15000
|
|
1502
|
+
* reproduces the historical constants exactly. */
|
|
1503
|
+
leaseTtlMs?: number;
|
|
1504
|
+
/** Number of shards this fleet runs (B2a). This task threads it as a plain parameter (default 8);
|
|
1505
|
+
* T5 owns the persist-once/`HELIPOD_FLEET_SHARDS`/mismatch-fail-fast story. Drives the commit
|
|
1506
|
+
* pool's connection set, the acquire-all loop, and the runtime's `ShardedTransactor`. */
|
|
1507
|
+
numShards?: number;
|
|
1508
|
+
}): Promise<FleetPrep>;
|
|
1509
|
+
interface StartFleetNodeDeps {
|
|
1510
|
+
client: NodePgClient;
|
|
1511
|
+
/** The Postgres store — the tail source + promotion swap target (sync), or the live runtime store
|
|
1512
|
+
* (writer, in which case `startFleetNode` just returns handles). */
|
|
1513
|
+
pgStore: PostgresDocStore;
|
|
1514
|
+
runtime: EmbeddedRuntime;
|
|
1515
|
+
lease: LeaseManager;
|
|
1516
|
+
forwarder: WriteForwarder;
|
|
1517
|
+
/** Sync only: the local replica the tailer applies onto, and the switchable the runtime reads
|
|
1518
|
+
* through (swapped to the Postgres store on promotion). Absent/ignored for a writer boot. */
|
|
1519
|
+
replica?: SqliteDocStore;
|
|
1520
|
+
switchable?: SwitchableDocStore;
|
|
1521
|
+
/** Sync only: `replica`'s on-disk path — needed here (not just in `prepareFleetNode`) because the
|
|
1522
|
+
* C7 `reconcileReplicaIdentity` check now runs in THIS function, right before `tailer.start()`. */
|
|
1523
|
+
replicaPath?: string;
|
|
1524
|
+
/** Process-exit indirection (C4/C5). Injected so tests observe exits instead of killing the runner;
|
|
1525
|
+
* defaults to `console.error` + `process.exit(1)`. Fires on writer lease loss (the lease monitor)
|
|
1526
|
+
* and on a failed promotion step. */
|
|
1527
|
+
onExit?: (reason: string) => void;
|
|
1528
|
+
/** Number of shards this node runs (B2a) — from `prepareFleetNode`'s decision (`FleetPrep.numShards`).
|
|
1529
|
+
* The writer/promotion arming acquires all N shard leases (slots 1…N-1 beyond the default), seeds
|
|
1530
|
+
* all N frontiers before ready, and drives the idle-shard closer over them. Defaults to 1 so an
|
|
1531
|
+
* older single-shard call site (the B1 lifecycle tests that drive `startFleetNode` directly) is
|
|
1532
|
+
* byte-identical — one shard, one lease, one frontier. */
|
|
1533
|
+
numShards?: number;
|
|
1534
|
+
}
|
|
1535
|
+
/** The seam the C5 promotion wrap touches — narrow structural spies over the promotion sequence,
|
|
1536
|
+
* monitor start, promoted-callback fan-out, and the exit indirection. */
|
|
1537
|
+
interface PromotionRunDeps {
|
|
1538
|
+
/** The CRITICAL PROMOTION ORDER (`promoteFleetNode`), as a thunk. */
|
|
1539
|
+
promote: () => Promise<void>;
|
|
1540
|
+
/** Start the writer lease monitor — this node is now the writer and must self-exit on lease loss. */
|
|
1541
|
+
startMonitor: () => void;
|
|
1542
|
+
/** Notify the http layer to drop its writer proxy. */
|
|
1543
|
+
firePromoted: () => void;
|
|
1544
|
+
/** Exit indirection — any promotion-step failure routes here (a half-promoted node must not linger). */
|
|
1545
|
+
onExit: (reason: string) => void;
|
|
1546
|
+
}
|
|
1547
|
+
/**
|
|
1548
|
+
* C5 promotion error policy: wrap the promotion sequence so ANY step failure is caught, logged, and
|
|
1549
|
+
* turned into an exit rather than left as a silent unhandled rejection with the node stuck
|
|
1550
|
+
* half-promoted (writable pg store, un-swapped runtime, no drivers). On success the lease monitor is
|
|
1551
|
+
* started (writer self-exit is now armed) and the http layer is told to drop its proxy. Fires exit at
|
|
1552
|
+
* most once by construction — it's invoked once per node (guarded by the caller's `promoting` flag).
|
|
1553
|
+
*/
|
|
1554
|
+
declare function runPromotion(deps: PromotionRunDeps): Promise<void>;
|
|
1555
|
+
/**
|
|
1556
|
+
* Install the epoch-fenced commit guard (Fenced Frontier B1, D3) on `pgStore`. Runs inside every
|
|
1557
|
+
* `commitWrite` transaction, after the row inserts, before COMMIT — the SAME row as the lease
|
|
1558
|
+
* (`shard_leases`) advances the durable-commit frontier chain (`prev_ts := frontier_ts, frontier_ts
|
|
1559
|
+
* := commitTs`) predicated on THIS node's epoch still being current, so frontier publication,
|
|
1560
|
+
* fencing, and the lease are one row with zero extra round-trips. `lease.currentEpoch()` is read
|
|
1561
|
+
* LIVE on every commit (not a snapshot taken at install time) so a later re-promotion's epoch bump
|
|
1562
|
+
* is honored automatically. Zero rows updated (a stale/superseded epoch) throws `FencedError`, which
|
|
1563
|
+
* aborts the whole transaction — AND calls `onFenced(shardId, reason)` before throwing so the caller
|
|
1564
|
+
* can react (B2b, D2: `startFleetNode` wires this straight to `relinquish(shardId, reason)` — drop
|
|
1565
|
+
* JUST this shard, keep serving everything else — never to a whole-node exit; see `relinquish` below
|
|
1566
|
+
* for why `LeaseMonitor.fenced()`/`onExit` are NOT reached from here anymore). Called at writer boot
|
|
1567
|
+
* AND on promotion success — see the two call sites below.
|
|
1568
|
+
*
|
|
1569
|
+
* Returns the `addCommitGuard` unregister handle (Receipted Outbox decision 2 — the guard
|
|
1570
|
+
* slot→chain migration). `armWriter` re-arms on EVERY promotion; a caller that calls
|
|
1571
|
+
* `installCommitGuard` again WITHOUT first calling the PRIOR returned unregister function stacks a
|
|
1572
|
+
* duplicate epoch-fence guard on the chain — duplicate frontier bumps + duplicate
|
|
1573
|
+
* `fleet_idempotency` INSERTs at the same ts, so every subsequent forwarded commit self-PK-collides
|
|
1574
|
+
* and aborts. See `armWriter`'s `unregisterCommitGuard` handling below for the required pattern.
|
|
1575
|
+
*/
|
|
1576
|
+
declare function installCommitGuard(pgStore: PostgresDocStore, lease: LeaseManager, onFenced: (shardId: ShardId, reason: string) => void): () => void;
|
|
1577
|
+
/** The narrow seams `relinquish` needs — deliberately structural (not the concrete `NodePgClient`) so
|
|
1578
|
+
* it's unit-testable with a stub, and reusable by T4's balancer without depending on `startFleetNode`'s
|
|
1579
|
+
* internal closures. */
|
|
1580
|
+
interface RelinquishDeps {
|
|
1581
|
+
lease: LeaseManager;
|
|
1582
|
+
/** `releaseShardLock` is consulted defensively (`?.`) — absent on a poolless/PGlite client, though
|
|
1583
|
+
* every real B2b fleet node runs in pool mode and always has it. */
|
|
1584
|
+
client: {
|
|
1585
|
+
releaseShardLock?: (slot: number) => Promise<void>;
|
|
1586
|
+
};
|
|
1587
|
+
/** Ordered shard list (index = slot) — how `shardId`'s per-slot advisory lock is found for
|
|
1588
|
+
* `releaseShardLock`. The same list `prepareFleetNode`/`startFleetNode` derive via `shardIdList`. */
|
|
1589
|
+
shards: readonly ShardId[];
|
|
1590
|
+
/** Structured-log seam, defaults to `console.error`. Tests inject a spy. */
|
|
1591
|
+
log?: (msg: string) => void;
|
|
1592
|
+
/**
|
|
1593
|
+
* Invoked when the DEFAULT shard is relinquished (B2b, D5 — "drivers follow the default shard"):
|
|
1594
|
+
* `startFleetNode` wires this to `runtime.stopDriversOnly()`, so the moment this node loses the
|
|
1595
|
+
* default ring the scheduler/workflow/cron/reaper drivers go quiet (a different node now owns that
|
|
1596
|
+
* ring). Optional — the balancer/relinquish unit tests that don't care about drivers omit it, in
|
|
1597
|
+
* which case relinquishing the default shard just drops the shard with no driver side effect.
|
|
1598
|
+
*/
|
|
1599
|
+
onDefaultRelinquished?: () => void;
|
|
1600
|
+
}
|
|
1601
|
+
/**
|
|
1602
|
+
* Per-shard relinquish dispatcher (Fenced Frontier B2b, D2): the reduction a `FencedError` on shard
|
|
1603
|
+
* `s` gets now — "drop `s`, keep serving everything else" — instead of B1/B2a's "kill the node".
|
|
1604
|
+
* Routed to from the commit guard's `onFenced` (a fence discovered mid-commit — that commit itself
|
|
1605
|
+
* still aborts and propagates `FencedError` to its caller, OCC-retryable; relinquish is the SIDE
|
|
1606
|
+
* EFFECT, not a swallow), the batched heartbeat's `fencedShardIds` (a fence discovered on the probe
|
|
1607
|
+
* beat), and a per-shard commit-connection loss.
|
|
1608
|
+
*
|
|
1609
|
+
* Idempotent per shard: `lease.currentEpoch(shardId) === null` means this shard is already forgotten
|
|
1610
|
+
* (a prior relinquish call, or a shard this node never held) — a silent no-op, so callers never need
|
|
1611
|
+
* to track "have I already relinquished this" themselves.
|
|
1612
|
+
*
|
|
1613
|
+
* 1. `lease.forgetShard(shardId)` — drops the held-epoch entry, so the commit guard's "no acquired
|
|
1614
|
+
* epoch" branch fences any straggler commit on `s` cleanly, and `heartbeatAll`/`closeIdleFrontiers`
|
|
1615
|
+
* stop touching it.
|
|
1616
|
+
* 2. Release `s`'s per-slot advisory lock (`PgClient.releaseShardLock`) — UNLESS `opts.connectionLost`:
|
|
1617
|
+
* a dead commit connection already released its session-scoped lock the instant the backend went
|
|
1618
|
+
* away, so there is nothing left to release (and no live connection to run the unlock query on).
|
|
1619
|
+
* 3. Log one structured line. Relinquishing the DEFAULT shard additionally WARNS that drivers keep
|
|
1620
|
+
* running (scheduler/workflow/cron/reaper stay armed on this node) until a later task wires
|
|
1621
|
+
* `stopDriversOnly` (D5) — driver stop/start is deliberately NOT built here.
|
|
1622
|
+
*
|
|
1623
|
+
* Deliberately does NOT touch `LeaseMonitor` and never calls `onExit` — the whole point of B2b is that
|
|
1624
|
+
* a per-shard fence must no longer escalate to a whole-node exit. `LeaseMonitor` stays reserved for
|
|
1625
|
+
* pinned-connection loss and probe exhaustion (definitive WHOLE-NODE loss); see `startFleetNode`.
|
|
1626
|
+
*/
|
|
1627
|
+
declare function relinquish(deps: RelinquishDeps, shardId: ShardId, reason: string, opts?: {
|
|
1628
|
+
connectionLost?: boolean;
|
|
1629
|
+
}): void;
|
|
1630
|
+
/**
|
|
1631
|
+
* Acquire one non-default shard's lease as the writer (B2a acquire-all). `tryAcquire(shardId, slot)`
|
|
1632
|
+
* epoch-bumps that shard's row (fencing any prior holder) and takes its per-shard advisory lock. If
|
|
1633
|
+
* the lock is still held by a wedged prior holder AND that shard's lease has expired, fence + evict +
|
|
1634
|
+
* terminate its backend (the same fencing-first eviction the default-shard acquire loop uses) and
|
|
1635
|
+
* retry. Bounded by a generous deadline (~10 lease TTLs): the common failover frees every slot at once
|
|
1636
|
+
* when the default-shard loop terminates the whole wedged node, so this normally succeeds on the first
|
|
1637
|
+
* try (fresh boot) or right after one eviction. Exceeding the deadline throws → promotion/boot fails →
|
|
1638
|
+
* the node exits and rejoins fresh, never silently running as a partial-shard writer.
|
|
1639
|
+
*/
|
|
1640
|
+
declare function acquireShardAsWriter(lease: LeaseManager, shardId: ShardId, slot: number, retryMs: number, seedFrontierFromDocuments?: boolean): Promise<void>;
|
|
1641
|
+
/**
|
|
1642
|
+
* Pure throughput derivation for group-commit health (Fleet B4, T4): a rolling delta of
|
|
1643
|
+
* `flushCount` between two `groupCommitStats()` reads, over the elapsed wall-clock time — the
|
|
1644
|
+
* simplest honest `flushesPerSec` signal without a dedicated timer/interval. `prevReadMs === null`
|
|
1645
|
+
* (no prior sample — the first read after boot, or after any gap) and a non-positive elapsed time
|
|
1646
|
+
* (a clock that hasn't advanced, or moved backward) both report 0 rather than a division artifact.
|
|
1647
|
+
* Extracted as a standalone pure function so it's unit-testable without booting a fleet node.
|
|
1648
|
+
*/
|
|
1649
|
+
declare function deriveFlushesPerSec(prevReadMs: number | null, prevFlushCount: number, nowMs: number, nowFlushCount: number): number;
|
|
1650
|
+
/**
|
|
1651
|
+
* Wire the running fleet node. A writer node is already fully live (promoted in `prepareFleetNode`,
|
|
1652
|
+
* drivers started at `create()`) — it just gets handles. A sync node starts the replica tailer (its
|
|
1653
|
+
* `start()` catch-up is the node's ready gate) and the lease acquire loop, and promotes on acquire
|
|
1654
|
+
* via `promoteFleetNode`.
|
|
1655
|
+
*/
|
|
1656
|
+
declare function startFleetNode(deps: StartFleetNodeDeps): Promise<FleetHandles>;
|
|
1657
|
+
|
|
1658
|
+
/**
|
|
1659
|
+
* `StablePrefixTs` (Fenced Frontier B1, D6) — a compile-time brand over `bigint` marking a
|
|
1660
|
+
* timestamp as a member of the FENCED FRONTIER: the stable prefix of the primary's MVCC log that
|
|
1661
|
+
* `ReplicaTailer` is allowed to pull from, per D5. A raw `bigint` (e.g. `primary.maxTimestamp()`,
|
|
1662
|
+
* which reports the log's live high-water mark INCLUDING any commit that raced past the last fence
|
|
1663
|
+
* check) is NOT a `StablePrefixTs` — only a value read straight off `shard_leases.frontier_ts`, or
|
|
1664
|
+
* a watermark previously derived from one, is.
|
|
1665
|
+
*
|
|
1666
|
+
* This is deliberately narrow: it brands ONLY `ReplicaTailer`'s internal pull target (`F`) and its
|
|
1667
|
+
* own watermark (`wm`), which must never advance past `F` (D5's "wm never exceeds F" invariant).
|
|
1668
|
+
* It does NOT brand `waitFor`'s public `ts` parameter — that's a caller-supplied `commitTs` (a raw
|
|
1669
|
+
* log position by nature, e.g. forwarded from `WriteForwarder`'s `/_fleet/run` response), and RYOW
|
|
1670
|
+
* correctness only needs `wm >= ts`, which is a plain bigint comparison regardless of which side is
|
|
1671
|
+
* branded. Branding that parameter too would just force every call site to launder a raw commitTs
|
|
1672
|
+
* through the constructor for no safety benefit.
|
|
1673
|
+
*
|
|
1674
|
+
* The sole constructor, `stablePrefixFromFrontier`, is intentionally the ONLY way to produce one —
|
|
1675
|
+
* assigning a raw `bigint` to a `StablePrefixTs`-typed binding without going through it is a
|
|
1676
|
+
* compile error (see `replica-tailer.test.ts`'s `@ts-expect-error` cases). At runtime this is a
|
|
1677
|
+
* plain `bigint`; the brand (`__stablePrefix`) exists purely in the type system and costs nothing.
|
|
1678
|
+
*/
|
|
1679
|
+
type StablePrefixTs = bigint & {
|
|
1680
|
+
readonly __stablePrefix: unique symbol;
|
|
1681
|
+
};
|
|
1682
|
+
/**
|
|
1683
|
+
* The sole constructor. Call this ONLY where a value is actually a frontier — a fresh read of
|
|
1684
|
+
* `shard_leases.frontier_ts` (via the tailer's `CommitChannelClient`), or a watermark already
|
|
1685
|
+
* derived from a prior `StablePrefixTs` (e.g. the replica's own persisted `maxTimestamp()` on
|
|
1686
|
+
* restart, which IS this node's last-applied frontier). Do not use this to launder an arbitrary
|
|
1687
|
+
* `bigint` (e.g. `primary.maxTimestamp()`, or a document's raw `ts`) into the brand — that value is
|
|
1688
|
+
* NOT guaranteed to be a durably-fenced, dense prefix and defeats the whole point of the type.
|
|
1689
|
+
*/
|
|
1690
|
+
declare function stablePrefixFromFrontier(raw: bigint): StablePrefixTs;
|
|
1691
|
+
|
|
1692
|
+
/**
|
|
1693
|
+
* The offline fleet reshard tool (B5 Part 1, Task 9.1): change a STOPPED Postgres fleet's shard
|
|
1694
|
+
* count N→M by updating the persist-once `fleet:numShards` global + creating/deleting
|
|
1695
|
+
* `shard_leases` lane rows — atomically, in one transaction — and NOTHING else. Because a shard is
|
|
1696
|
+
* a logical commit-serialization lane over one shared store (`documents.shard_id` is decorative;
|
|
1697
|
+
* routing always recomputes `shardIdForKeyValue(key, numShards)` fresh), resharding moves no rows:
|
|
1698
|
+
* only which lane FUTURE commits for a key serialize through changes. See
|
|
1699
|
+
* `docs/dev/research/write-sharding/b5-reshard-and-object-storage.md` (Part 1) and
|
|
1700
|
+
* `docs/superpowers/plans/2026-02-20-fleet-reshard-b5.md` for the full design.
|
|
1701
|
+
*
|
|
1702
|
+
* Safety rests on the STOPPED-FLEET precondition (no `--force`): a live node makes the old-N and
|
|
1703
|
+
* new-M epochs potentially overlap in time, which is out of scope (needs an epoch-fence vocabulary
|
|
1704
|
+
* this tool doesn't have — see the plan's "online reshard" scope boundary). A quiesced fleet makes
|
|
1705
|
+
* the two epochs strictly non-overlapping, so changing a key's lane between them is as safe as any
|
|
1706
|
+
* config change to a stopped system.
|
|
1707
|
+
*
|
|
1708
|
+
* `F = min(frontier_ts)` is monotone across a reshard: new lanes are seeded at `MAX(ts)` (never
|
|
1709
|
+
* below the true high-water mark — the same F1 seed `LeaseManager.setup()`/`tryAcquire()` use, see
|
|
1710
|
+
* `frontierSeedExpr` in `./lease.ts`), retained lanes are GREATEST-bumped up to that same floor
|
|
1711
|
+
* (never lowered — a documented, safe deviation from "retained rows untouched": raise-only can
|
|
1712
|
+
* never regress F, and it proactively heals a lane whose frontier legitimately lags `MAX(ts)` after
|
|
1713
|
+
* a crash stop — no `selfFence` ran, so nothing bumped it since the last idle-closer beat — the
|
|
1714
|
+
* same healing a node boot's `seedFrontierAll` would eventually do), and deleting a lane can only
|
|
1715
|
+
* raise or hold `min(frontier_ts)`, never lower it. Net effect: EVERY post-reshard lane is
|
|
1716
|
+
* `>= MAX(ts)`, making the post-verify's F1 check a true post-condition rather than a best-effort one.
|
|
1717
|
+
*/
|
|
1718
|
+
|
|
1719
|
+
/** The `persistence_globals` key the resolved shard count is stamped under — byte-identical to
|
|
1720
|
+
* `packages/cli/src/boot.ts`'s `NUM_SHARDS_GLOBAL_KEY` (kept as an independent literal here: the
|
|
1721
|
+
* ee `@helipod/fleet` package has no dependency on core `packages/cli`, mirroring the same
|
|
1722
|
+
* independent-literal choice `boot.ts`'s own doc comment makes for `DEFAULT_NUM_SHARDS`). */
|
|
1723
|
+
declare const NUM_SHARDS_GLOBAL_KEY = "fleet:numShards";
|
|
1724
|
+
/** Thrown when `reshardFleet` is asked to run against a fleet with any live `fleet_nodes` or
|
|
1725
|
+
* `shard_leases.writer_url` row — the hard stopped-fleet precondition, no `--force` escape hatch.
|
|
1726
|
+
* `liveUrls` names every live node/writer URL found, so the operator error is actionable. */
|
|
1727
|
+
declare class ReshardFleetLiveError extends Error {
|
|
1728
|
+
readonly liveUrls: string[];
|
|
1729
|
+
constructor(liveUrls: string[]);
|
|
1730
|
+
}
|
|
1731
|
+
/** Thrown when the post-commit verification pass finds the reshard did not land as expected —
|
|
1732
|
+
* should be impossible (a guard against a logic bug, not an expected operational outcome). */
|
|
1733
|
+
declare class ReshardVerificationError extends Error {
|
|
1734
|
+
constructor(message: string);
|
|
1735
|
+
}
|
|
1736
|
+
/** Thrown when `reshardFleet` is pointed at a Postgres database that has never run a fleet on it —
|
|
1737
|
+
* no `shard_leases`/`fleet_nodes` tables. Without this pre-gate, the live-fleet gate query below
|
|
1738
|
+
* would throw a raw `relation "fleet_nodes" does not exist`, which reads like an internal SQL bug
|
|
1739
|
+
* rather than the operator-actionable "you haven't booted a fleet here yet" that it actually is. */
|
|
1740
|
+
declare class ReshardNotAFleetError extends Error {
|
|
1741
|
+
constructor();
|
|
1742
|
+
}
|
|
1743
|
+
interface ReshardResult {
|
|
1744
|
+
previousShards: number;
|
|
1745
|
+
newShards: number;
|
|
1746
|
+
/** Shard ids newly created (present in the target set, absent from the prior lane set). */
|
|
1747
|
+
created: string[];
|
|
1748
|
+
/** Shard ids removed (present in the prior lane set, absent from the target set). Never includes
|
|
1749
|
+
* `"default"` — it is a member of `shardIdList(M)` for every `M >= 1` and so can never be a
|
|
1750
|
+
* target-set omission. */
|
|
1751
|
+
deleted: string[];
|
|
1752
|
+
/** `min(frontier_ts)` across every post-reshard lane, as a decimal string (frontier_ts is a
|
|
1753
|
+
* Postgres BIGINT — carried as a string rather than risk a JS-number precision loss). */
|
|
1754
|
+
frontierFloor: string;
|
|
1755
|
+
}
|
|
1756
|
+
/**
|
|
1757
|
+
* Change a STOPPED Postgres fleet's shard count from whatever it currently is to `opts.targetShards`.
|
|
1758
|
+
* See the module doc comment for the full safety argument. Steps (verbatim to the plan, plus the
|
|
1759
|
+
* not-a-fleet pre-gate):
|
|
1760
|
+
* 1. validate `targetShards >= 1`
|
|
1761
|
+
* 2. probe that this is actually a fleet store (`shard_leases`/`fleet_nodes` exist) —
|
|
1762
|
+
* `ReshardNotAFleetError` otherwise, before the next step's raw SQL can throw an opaque
|
|
1763
|
+
* "relation does not exist"
|
|
1764
|
+
* 3. refuse if any `fleet_nodes`/`shard_leases` row is live (`ReshardFleetLiveError`)
|
|
1765
|
+
* 4. read the current lane set + the current `fleet:numShards` global
|
|
1766
|
+
* 5. one transaction: upsert the global, INSERT new lanes (seeded at `MAX(ts)`), DELETE surplus
|
|
1767
|
+
* lanes, then GREATEST-bump every remaining lane's frontier up to that same `MAX(ts)` floor
|
|
1768
|
+
* (heals a retained lane that crash-stopped below it — see the module doc comment)
|
|
1769
|
+
* 6. post-verify (count/set/frontier-floor/global) — `ReshardVerificationError` on any mismatch
|
|
1770
|
+
*/
|
|
1771
|
+
declare function reshardFleet(client: PgClient, opts: {
|
|
1772
|
+
targetShards: number;
|
|
1773
|
+
}): Promise<ReshardResult>;
|
|
1774
|
+
|
|
1775
|
+
declare const FLEET_VERSION = "0.0.0";
|
|
1776
|
+
|
|
1777
|
+
export { type AppliedInvalidation, type BalancerLease, type CommitChannelClient, DEFAULT_NUM_SHARDS, DensityViolationError, FLEET_VERSION, FLEET_WRITER_SESSION_TIMEOUTS, FencedError, type FleetHandles, type FleetPrep, type FleetRuntimeOptions, FrontierMonitor, type FrontierStats, IDEMPOTENCY_VALUE_CAP_BYTES, type IdempotencyReplay, LeaseManager, type LeaseManagerOptions, LeaseMonitor, type LeaseMonitorDeps, type LeaseRow, type LeaseState, NUM_SHARDS_GLOBAL_KEY, NotifyingFanoutAdapter, type PromotionRunDeps, REPLICA_DB_FILENAME, type RelinquishDeps, ReplicaTailer, type ReplicaTailerOptions, type ReplicaWaiter, ReshardFleetLiveError, ReshardNotAFleetError, type ReshardResult, ReshardVerificationError, ShardLeaseBalancer, type ShardLeaseBalancerDeps, type StablePrefixTs, type StartFleetNodeDeps, SwitchableDocStore, WriteForwarder, type WriteForwarderOptions, acquireShardAsWriter, createAsyncChain, deriveFlushesPerSec, fleetApplicationName, fleetMultiWriterEnabled, groupCommitEnabled, installCommitGuard, prepareFleetNode, relinquish, rendezvousOwner, rendezvousWeight, reshardFleet, runPromotion, stablePrefixFromFrontier, startFleetNode };
|