@rindle/room 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.sh +53 -0
- package/dist/authority.d.ts +57 -0
- package/dist/authority.d.ts.map +1 -0
- package/dist/authority.js +73 -0
- package/dist/authority.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +22 -0
- package/dist/index.js.map +1 -0
- package/dist/journal.d.ts +64 -0
- package/dist/journal.d.ts.map +1 -0
- package/dist/journal.js +53 -0
- package/dist/journal.js.map +1 -0
- package/dist/mutation-tx.d.ts +79 -0
- package/dist/mutation-tx.d.ts.map +1 -0
- package/dist/mutation-tx.js +207 -0
- package/dist/mutation-tx.js.map +1 -0
- package/dist/shell.d.ts +134 -0
- package/dist/shell.d.ts.map +1 -0
- package/dist/shell.js +1589 -0
- package/dist/shell.js.map +1 -0
- package/dist/token.d.ts +89 -0
- package/dist/token.d.ts.map +1 -0
- package/dist/token.js +158 -0
- package/dist/token.js.map +1 -0
- package/dist/wasm.d.ts +7 -0
- package/dist/wasm.d.ts.map +1 -0
- package/dist/wasm.js +29 -0
- package/dist/wasm.js.map +1 -0
- package/package.json +71 -0
- package/pkg/rindle_room.d.ts +345 -0
- package/pkg/rindle_room.js +924 -0
- package/pkg/rindle_room_bg.wasm +0 -0
- package/pkg/rindle_room_bg.wasm.d.ts +45 -0
- package/src/authority.ts +127 -0
- package/src/index.ts +49 -0
- package/src/journal.ts +110 -0
- package/src/mutation-tx.ts +275 -0
- package/src/shell.ts +1859 -0
- package/src/token.ts +219 -0
- package/src/wasm.ts +32 -0
package/src/shell.ts
ADDED
|
@@ -0,0 +1,1859 @@
|
|
|
1
|
+
// The Node test shell (RINDLE-REALTIME-DESIGN.md §11, P0/P1): a plain process running
|
|
2
|
+
// the SAME wasm build the Durable Object shell will run, wired to real sockets.
|
|
3
|
+
//
|
|
4
|
+
// rindled ──ws (normalized protocol)──▶ WasmRoom (base store) ──serving──▶ subscribers
|
|
5
|
+
// upstream leg (§3) downstream leg (§4)
|
|
6
|
+
//
|
|
7
|
+
// Both legs speak the one protocol. Upstream, the shell is a client of rindled's public
|
|
8
|
+
// ws plane: `init` → `subscribe {queryId, leaseToken}` → `nhello` → seq-0 `nbatch` →
|
|
9
|
+
// live tail; the lease is minted on rindled's private control plane (`POST /materialize`)
|
|
10
|
+
// — the room fetches its own footprint lease at boot, the same shape as the DO shell's
|
|
11
|
+
// boot callback (§10.1). Downstream, the shell serves the `@rindle/remote` wire verbatim,
|
|
12
|
+
// gated by **self-authorizing signed lease tokens** (§10.1): `subscribe {queryId,
|
|
13
|
+
// leaseToken}` verifies the token (signature, doc, expiry, revocation) and materializes
|
|
14
|
+
// the approved AST it carries on presentation — no `/materialize` control call, no named
|
|
15
|
+
// registry, ASTs never composed by clients. Identical queries share one pipeline
|
|
16
|
+
// (QueryKey dedup in the wasm room); each subscriber gets its own epoch/seq envelope.
|
|
17
|
+
//
|
|
18
|
+
// Lease lifecycle (§4.1): tokens are short-lived; the shell drops any subscription whose
|
|
19
|
+
// lease passes `exp` unrenewed (a `queryError`, then detach — renewal is
|
|
20
|
+
// re-authorization through the API server, which is a fresh subscribe with a fresh
|
|
21
|
+
// token). The private control plane's `POST /revoke {userId}` terminates a user's
|
|
22
|
+
// subscriptions and sockets immediately and refuses pre-revocation tokens (`iat` <
|
|
23
|
+
// revocation) — the synchronous layer; the TTL is the backstop.
|
|
24
|
+
//
|
|
25
|
+
// The write path (§5.1, P2 — no write-behind yet): a connection `init`s its clientID,
|
|
26
|
+
// then pushes `{t:"pushMutation", envelope:{clientID, mid, name, args}}` frames. Per
|
|
27
|
+
// mutation: dedup/gap-check → run the registered mutator against a staged wasm tx →
|
|
28
|
+
// commit to the shared head → fan the data `nbatch`es out IMMEDIATELY (step 2: fanout
|
|
29
|
+
// is never gated on durability) → append the envelope to the journal under a ≤~5ms
|
|
30
|
+
// group commit → on append, ACK: advance the author's `_rindle_client_mutations` row
|
|
31
|
+
// and fan the lmid frame (step 4: the ack a client observes survives a process crash
|
|
32
|
+
// by construction — §8.1). A failed/unknown mutator is a silent reject that still
|
|
33
|
+
// consumes the mid (the ledger advances with no effects; the author's prediction snaps
|
|
34
|
+
// back — `rindle-replica`'s contract). On boot, the journal replays by RE-INVOKING the
|
|
35
|
+
// mutators against the fresh base (§3.3), then acks everything replayed.
|
|
36
|
+
//
|
|
37
|
+
// Release protocol: every downstream `nbatch` is stamped with the room's `head_cv`,
|
|
38
|
+
// and after each drain the shell sends the connection-level `{t:"progress",
|
|
39
|
+
// frame:{cvMin}}` the optimistic client releases on. One §1.3.1-coherence rule: the
|
|
40
|
+
// AUTHOR's cvMin holds below its oldest un-acked mutation's apply-cv until the ack
|
|
41
|
+
// lands (data released before its lmid advance would make the client re-invoke a
|
|
42
|
+
// confirmed mutation on top of its own effect); everyone else releases at head_cv.
|
|
43
|
+
//
|
|
44
|
+
// Backpressure (§9, T8): each downstream socket has a bounded send budget. A stalled
|
|
45
|
+
// client that overflows it is TERMINATED with a gap — the normalized protocol's
|
|
46
|
+
// re-subscribe repair — so one slow client never holds frames in room memory.
|
|
47
|
+
//
|
|
48
|
+
// Failure posture (§3.4): the room is an INCARNATION. Any upstream violation — a frame
|
|
49
|
+
// that fails to decode, a seq gap, a poisoned apply, an upstream `queryError`, the
|
|
50
|
+
// socket dropping — kills the incarnation: the wasm store is freed, every downstream
|
|
51
|
+
// socket is closed (subscriptions die with the incarnation; reconnecting clients
|
|
52
|
+
// re-subscribe against the next one), and the shell re-subscribes upstream. On the same
|
|
53
|
+
// socket that is a `subscribe` re-send (rindled bumps the epoch and re-snapshots); on a
|
|
54
|
+
// fresh socket it is a new subscription. There is no patch-up path, by design.
|
|
55
|
+
|
|
56
|
+
import { createServer, type Server } from "node:http";
|
|
57
|
+
import { randomUUID } from "node:crypto";
|
|
58
|
+
|
|
59
|
+
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
|
60
|
+
|
|
61
|
+
import { initRoomWasm, WasmRoom } from "./wasm.ts";
|
|
62
|
+
import { verifyRoomToken, RoomTokenError, scopeSpecsHash } from "./token.ts";
|
|
63
|
+
import {
|
|
64
|
+
assertSyncMutatorReturn,
|
|
65
|
+
isEnvironmentShortfall,
|
|
66
|
+
mutationTx,
|
|
67
|
+
type RoomMutator,
|
|
68
|
+
type TableShape,
|
|
69
|
+
} from "./mutation-tx.ts";
|
|
70
|
+
import {
|
|
71
|
+
journalEntryOutcome,
|
|
72
|
+
memoryJournal,
|
|
73
|
+
type RoomJournal,
|
|
74
|
+
type RoomJournalEntry,
|
|
75
|
+
} from "./journal.ts";
|
|
76
|
+
import type { RoomAuthority } from "./authority.ts";
|
|
77
|
+
|
|
78
|
+
/** The upstream half of the shell's config: where rindled lives and what to follow. */
|
|
79
|
+
export interface UpstreamOptions {
|
|
80
|
+
/** rindled's public subscription plane, e.g. `ws://127.0.0.1:7601`. */
|
|
81
|
+
wsUrl: string;
|
|
82
|
+
/** rindled's private control plane, e.g. `http://127.0.0.1:7600` (lease minting). */
|
|
83
|
+
controlUrl: string;
|
|
84
|
+
/** Bearer token for the control plane (required unless rindled runs unauthenticated). */
|
|
85
|
+
authToken?: string;
|
|
86
|
+
/** The document footprint — a wire `Ast` (§3.1). What this room follows and serves from. */
|
|
87
|
+
footprintAst: unknown;
|
|
88
|
+
/** Lease TTL passed to `/materialize` (rindled's default when omitted). */
|
|
89
|
+
leaseTtlMs?: number;
|
|
90
|
+
/** The `init` clientID on the upstream socket (diagnostic identity). */
|
|
91
|
+
clientId?: string;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** The downstream half: how clients are authorized and served (§4/§10.1). */
|
|
95
|
+
export interface DownstreamOptions {
|
|
96
|
+
/** This room's document id — lease tokens for any other doc are refused. */
|
|
97
|
+
docId: string;
|
|
98
|
+
/** Token key ring: `kid` → shared secret (the API server signs with the same ring). */
|
|
99
|
+
tokenKeys: Record<string, string>;
|
|
100
|
+
/** Idle grace before an unsubscribed query's pipeline is reclaimed (default 30s). */
|
|
101
|
+
idleTtlMs?: number;
|
|
102
|
+
/** Lease-expiry / idle-sweep cadence (default 1s). */
|
|
103
|
+
sweepIntervalMs?: number;
|
|
104
|
+
/** How long a revocation keeps refusing pre-revocation tokens (default 30min — set
|
|
105
|
+
* it ≥ the longest token TTL the API server mints). */
|
|
106
|
+
revocationWindowMs?: number;
|
|
107
|
+
/** The room's private control plane (`POST /revoke`, `GET /stats`). Omit to run
|
|
108
|
+
* without one (no revocation surface). */
|
|
109
|
+
control?: { authToken: string; port?: number };
|
|
110
|
+
/** The write plane (§5.1). Omit to run read-only (writes are refused). */
|
|
111
|
+
writes?: WritesOptions;
|
|
112
|
+
/** Per-socket downstream send budget in bytes (§9; default 4 MiB). A socket whose
|
|
113
|
+
* queued bytes would exceed it is terminated with a gap — re-subscribing is the
|
|
114
|
+
* repair. */
|
|
115
|
+
sendBudgetBytes?: number;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** One table's §3.3 scope spec (H-iv-b): an element of the api-server's
|
|
119
|
+
* `RoomBootResponse.scopes`, passed VERBATIM to the wasm room's `enableWritesV2`.
|
|
120
|
+
* Structurally identical to `@rindle/api-server`'s `RoomScopeSpec` (declared locally —
|
|
121
|
+
* the shell must not depend on the api-server package; TS structural typing keeps the
|
|
122
|
+
* two in lockstep at the host's threading site). */
|
|
123
|
+
export interface RoomScopeSpec {
|
|
124
|
+
table: string;
|
|
125
|
+
/** The footprint's row-local predicate for this table (a wire `Condition`) — drives
|
|
126
|
+
* the commit gate's absent-read proof. Absent ⇒ absent reads on this table always
|
|
127
|
+
* deopt (fail closed). */
|
|
128
|
+
footprintWhere?: unknown;
|
|
129
|
+
writable:
|
|
130
|
+
| { kind: "none" }
|
|
131
|
+
| { kind: "predicate"; where?: unknown; joinKeyCols: string[] };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** The §5.1 write plane: the room's own mutator registry over its owned tables. */
|
|
135
|
+
export interface WritesOptions {
|
|
136
|
+
/** The server registry (§4.2): named mutators run against the shared head. PLAIN
|
|
137
|
+
* synchronous `(tx, args, ctx)` functions only — a `shared(...)` GENERATOR registry
|
|
138
|
+
* does NOT register verbatim (nothing drives it here; the shell rejects a mutator
|
|
139
|
+
* that returns a generator/promise — see `assertSyncMutatorReturn`). */
|
|
140
|
+
mutators: Record<string, RoomMutator>;
|
|
141
|
+
/** §3.2's owned set — the only tables mutators may write. Must all be in the
|
|
142
|
+
* upstream footprint; followed tables and the ledger are never writable. */
|
|
143
|
+
ownedTables: string[];
|
|
144
|
+
/** The §3.3 per-table scope specs from the boot wire (`RoomBootResponse.scopes` —
|
|
145
|
+
* H-iv-b). Present ⇒ the write plane enables in v2 GATED mode (`enableWritesV2`):
|
|
146
|
+
* staged writes validate against the writable predicates, join-key edits refuse,
|
|
147
|
+
* absent reads must prove against `footprintWhere`, context (`kind:"none"`) tables
|
|
148
|
+
* become txGet-READABLE, and a violating commit returns a structured DEOPT instead
|
|
149
|
+
* of applying. The scopes' writable tables must be a SUBSET of {@link ownedTables}
|
|
150
|
+
* (the host's own declaration) — a wider server scope throws at construction, so a
|
|
151
|
+
* self-hoster's owned set can never be extended from the wire. Absent ⇒ the v1
|
|
152
|
+
* table-granular write plane, byte-identical to before. */
|
|
153
|
+
scopes?: RoomScopeSpec[];
|
|
154
|
+
/** The durable sidecar an ack means (§8.1). Defaults to `memoryJournal()` — the
|
|
155
|
+
* "survives nothing beyond the process" class; hosts bring their own. */
|
|
156
|
+
journal?: RoomJournal;
|
|
157
|
+
/** The journal group-commit window (default 5ms): mutations arriving within it
|
|
158
|
+
* share one append, and their acks ride one ledger commit. */
|
|
159
|
+
groupCommitMs?: number;
|
|
160
|
+
/** The write authority (§5.3.1) — the API server's `/apply-row-change-txn` host
|
|
161
|
+
* (or the P3 gate's mock). Omit to run journal-only (P2 semantics: nothing is
|
|
162
|
+
* ever durable upstream). With an authority, the shell claims a placement epoch
|
|
163
|
+
* at boot, probes durable lmids before replay, and write-behinds on the flush
|
|
164
|
+
* cadence (§5.3). */
|
|
165
|
+
authority?: RoomAuthority;
|
|
166
|
+
/** The flush debounce (§5.3; default 250ms, within the design's ≤1s budget). */
|
|
167
|
+
flushDebounceMs?: number;
|
|
168
|
+
/** Flush immediately once this many keys are dirty (default 512). */
|
|
169
|
+
flushDirtyMax?: number;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export interface RoomShellOptions {
|
|
173
|
+
upstream: UpstreamOptions;
|
|
174
|
+
downstream: DownstreamOptions;
|
|
175
|
+
/** Downstream ws port (default 0 = ephemeral; bound on 127.0.0.1). */
|
|
176
|
+
port?: number;
|
|
177
|
+
/** Diagnostic sink (default: silent). */
|
|
178
|
+
log?: (line: string) => void;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export interface RoomShell {
|
|
182
|
+
/** The bound downstream port. */
|
|
183
|
+
readonly port: number;
|
|
184
|
+
/** The bound control-plane port (0 when no control plane was configured). */
|
|
185
|
+
readonly controlPort: number;
|
|
186
|
+
/** Resolves when the CURRENT incarnation is live (seq-0 snapshot applied);
|
|
187
|
+
* immediately if it already is. */
|
|
188
|
+
awaitLive(): Promise<void>;
|
|
189
|
+
/** The last-applied upstream commit version, if live. */
|
|
190
|
+
cv(): number | undefined;
|
|
191
|
+
/** The upstream subscription epoch of the current incarnation, if any. */
|
|
192
|
+
upstreamEpoch(): number | undefined;
|
|
193
|
+
/** This incarnation's downstream bootId (rotates on every re-subscribe). */
|
|
194
|
+
bootId(): string;
|
|
195
|
+
/** Fire the write-behind flush now (instead of the debounce) and await its
|
|
196
|
+
* settlement — deterministic flushing for tests and drain-before-close. No-op
|
|
197
|
+
* without an authority. */
|
|
198
|
+
flushNow(): Promise<void>;
|
|
199
|
+
close(): Promise<void>;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const UPSTREAM_QID = 1;
|
|
203
|
+
/** Delay before re-subscribing after a violation — keeps a persistent violation from
|
|
204
|
+
* becoming a hot loop while staying far below human-perceptible recovery time. */
|
|
205
|
+
const RESUBSCRIBE_DELAY_MS = 250;
|
|
206
|
+
const RECONNECT_MAX_MS = 5_000;
|
|
207
|
+
const DEFAULT_IDLE_TTL_MS = 30_000;
|
|
208
|
+
const DEFAULT_SWEEP_INTERVAL_MS = 1_000;
|
|
209
|
+
const DEFAULT_REVOCATION_WINDOW_MS = 30 * 60_000;
|
|
210
|
+
const DEFAULT_GROUP_COMMIT_MS = 5;
|
|
211
|
+
const DEFAULT_SEND_BUDGET_BYTES = 4 * 1024 * 1024;
|
|
212
|
+
const DEFAULT_FLUSH_DEBOUNCE_MS = 250;
|
|
213
|
+
const DEFAULT_FLUSH_DIRTY_MAX = 512;
|
|
214
|
+
/** §4.2 drain-before-downgrade iteration cap: a healthy room quiesces in a handful of flushes; a
|
|
215
|
+
* runaway (a straggler push per flush, or a wedged CAS loop) fails LOUD rather than spinning. */
|
|
216
|
+
const DRAIN_MAX_ITERATIONS = 100;
|
|
217
|
+
/** Flush-retry backoff bounds (network-class failures only; same journaled bytes). */
|
|
218
|
+
const FLUSH_RETRY_MIN_MS = 100;
|
|
219
|
+
const FLUSH_RETRY_MAX_MS = 2_000;
|
|
220
|
+
/** The reserved lmid system query — subscribed BY NAME even in lease mode; identity
|
|
221
|
+
* comes from the connection's `init`, never from args. */
|
|
222
|
+
const LMID_QUERY_NAME = "_rindle/clientLmid";
|
|
223
|
+
|
|
224
|
+
/** Mint the room's upstream footprint lease on rindled's control plane (§4: the room
|
|
225
|
+
* fetches its own lease — on the DO shell this is the boot callback's job). */
|
|
226
|
+
async function mintLease(up: UpstreamOptions): Promise<string> {
|
|
227
|
+
const res = await fetch(new URL("/materialize", up.controlUrl), {
|
|
228
|
+
method: "POST",
|
|
229
|
+
headers: {
|
|
230
|
+
"content-type": "application/json",
|
|
231
|
+
...(up.authToken ? { authorization: `Bearer ${up.authToken}` } : {}),
|
|
232
|
+
},
|
|
233
|
+
body: JSON.stringify({
|
|
234
|
+
ast: up.footprintAst,
|
|
235
|
+
...(up.leaseTtlMs !== undefined ? { leaseTtlMs: up.leaseTtlMs } : {}),
|
|
236
|
+
}),
|
|
237
|
+
});
|
|
238
|
+
if (!res.ok) {
|
|
239
|
+
throw new Error(`upstream /materialize failed: ${res.status} ${await res.text()}`);
|
|
240
|
+
}
|
|
241
|
+
const out = (await res.json()) as { leaseToken?: string };
|
|
242
|
+
if (typeof out.leaseToken !== "string") {
|
|
243
|
+
throw new Error("upstream /materialize returned no leaseToken");
|
|
244
|
+
}
|
|
245
|
+
return out.leaseToken;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function send(ws: WebSocket, frame: unknown): void {
|
|
249
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
250
|
+
ws.send(JSON.stringify(frame));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** One downstream subscription's routing + lease state (keyed by its wasm subKey). */
|
|
255
|
+
interface SubMeta {
|
|
256
|
+
ws: WebSocket;
|
|
257
|
+
conn: ConnState;
|
|
258
|
+
clientQid: number;
|
|
259
|
+
/** The token's subject — the §4.1 revocation key. */
|
|
260
|
+
user: string;
|
|
261
|
+
/** The token's expiry — enforced by the sweep (drop at `exp` unrenewed). */
|
|
262
|
+
exp: number;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
interface ConnQuery {
|
|
266
|
+
subKey: string;
|
|
267
|
+
epoch: number;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
interface ConnState {
|
|
271
|
+
id: number;
|
|
272
|
+
ws: WebSocket;
|
|
273
|
+
queries: Map<number, ConnQuery>;
|
|
274
|
+
/** Serializes message handling per connection (token verify is async; a client's
|
|
275
|
+
* subscribe/unsubscribe order must hold). */
|
|
276
|
+
busy: Promise<void>;
|
|
277
|
+
/** The `init` identity — what the lmid subscribe and pushMutation key on. An
|
|
278
|
+
* idempotency key, NOT an identity (it is client-supplied). */
|
|
279
|
+
clientID: string | null;
|
|
280
|
+
/** The connection's AUTHENTICATED subject (managed-writes §3.1): the `sub` of the
|
|
281
|
+
* first verified lease token presented on this connection. Shell-stamped, one
|
|
282
|
+
* principal per connection, and the gate `pushMutation` requires. */
|
|
283
|
+
sub: string | null;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** One mutation applied to the head but not yet acked: the author's release point
|
|
287
|
+
* holds below `applyCv` until the journal append lands (§1.3.1 coherence — see the
|
|
288
|
+
* module docs' release-protocol note). */
|
|
289
|
+
interface UnackedMutation {
|
|
290
|
+
mid: number;
|
|
291
|
+
applyCv: number;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** A recorded NON-APPLIED verdict (H-iv-b): what the `mutationOutcome` frame carries,
|
|
295
|
+
* and what a re-sent mid whose `beginMutation` dedups is re-answered with. `name`/
|
|
296
|
+
* `args` are kept for DEOPT verdicts only — the frame must be self-contained (a
|
|
297
|
+
* client that already retired the entry re-invokes from the frame). */
|
|
298
|
+
interface RecordedOutcome {
|
|
299
|
+
kind: "deopt" | "rejected";
|
|
300
|
+
reason?: string;
|
|
301
|
+
name?: string;
|
|
302
|
+
args?: unknown;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Per-client retention cap for {@link RecordedOutcome}s. The map only holds
|
|
306
|
+
* NON-APPLIED mids, and a client only re-sends a mid while its ledger lmid trails it
|
|
307
|
+
* — a contiguously-advancing window bounded by the client's in-flight backlog, far
|
|
308
|
+
* below this cap. Past it the oldest records evict (insertion order): a re-send of an
|
|
309
|
+
* evicted mid degrades to today's silence — the mid still dedups and the ledger still
|
|
310
|
+
* covers it, only the outcome re-answer is lost. */
|
|
311
|
+
const MAX_RECORDED_OUTCOMES_PER_CLIENT = 512;
|
|
312
|
+
|
|
313
|
+
class Shell implements RoomShell {
|
|
314
|
+
private readonly opts: RoomShellOptions;
|
|
315
|
+
private readonly log: (line: string) => void;
|
|
316
|
+
private readonly wss: WebSocketServer;
|
|
317
|
+
private control: Server | null = null;
|
|
318
|
+
|
|
319
|
+
private room: WasmRoom | null = null;
|
|
320
|
+
private live = false;
|
|
321
|
+
private incarnationBootId = randomUUID();
|
|
322
|
+
private liveWaiters: Array<() => void> = [];
|
|
323
|
+
|
|
324
|
+
private upstream: WebSocket | null = null;
|
|
325
|
+
private leaseToken = "";
|
|
326
|
+
private subscribeInFlight = false;
|
|
327
|
+
private reconnectDelayMs = RESUBSCRIBE_DELAY_MS;
|
|
328
|
+
private closed = false;
|
|
329
|
+
private nextConnId = 1;
|
|
330
|
+
private sweepTimer: ReturnType<typeof setInterval> | null = null;
|
|
331
|
+
|
|
332
|
+
/** subKey (what the wasm room routes by) → where its frames go + lease state. */
|
|
333
|
+
private readonly subs = new Map<string, SubMeta>();
|
|
334
|
+
/** Every open downstream connection — revocation must reach a BOUND conn even when
|
|
335
|
+
* it holds no live subscription (it is still write-capable). */
|
|
336
|
+
private readonly conns = new Set<ConnState>();
|
|
337
|
+
/** userId → when they were revoked (refuses tokens with `iat` ≤ this; pruned after
|
|
338
|
+
* the revocation window). */
|
|
339
|
+
private readonly revoked = new Map<string, number>();
|
|
340
|
+
/** Downstream subscribes queued while no incarnation is live. */
|
|
341
|
+
private pendingSubs: Array<{ ws: WebSocket; conn: ConnState; msg: SubscribeMsg }> = [];
|
|
342
|
+
|
|
343
|
+
// ------------------------------ write-plane state ------------------------------
|
|
344
|
+
/** The {@link scopeSpecsHash} of the boot-wire scopes THIS shell armed its §3.3 gate
|
|
345
|
+
* with — `undefined` in v1 (ungated) mode. Compared against each lease token's
|
|
346
|
+
* `scopesHash` to flag scope skew (a profile edited under this live room), which
|
|
347
|
+
* otherwise manifests only as an undiagnosable deopt loop. */
|
|
348
|
+
private readonly armedScopesHash: string | undefined;
|
|
349
|
+
/** Scope-skew hash pairs already logged (`lease→armed`), so the diagnostic fires ONCE
|
|
350
|
+
* per distinct skew, not once per subscribe. */
|
|
351
|
+
private readonly loggedScopeSkew = new Set<string>();
|
|
352
|
+
/** The journal (write plane only). One per shell — it outlives incarnations; that
|
|
353
|
+
* is the point (§3.3: pending is replayed from it on every re-subscribe). */
|
|
354
|
+
private journal: RoomJournal | null = null;
|
|
355
|
+
/** Positional table shapes from the upstream hello (keyed layer of MutationTx). */
|
|
356
|
+
private tableShapes = new Map<string, TableShape>();
|
|
357
|
+
/** Mutations queued while no incarnation is live (drained after pendingSubs). */
|
|
358
|
+
private pendingMutes: Array<{ ws: WebSocket; conn: ConnState; envelope: PushEnvelope }> = [];
|
|
359
|
+
/** clientID → its applied-but-unacked mutations, oldest first (release holdback). */
|
|
360
|
+
private readonly unacked = new Map<string, UnackedMutation[]>();
|
|
361
|
+
/** clientID → its recorded NON-APPLIED outcomes (mid → verdict), insertion-ordered
|
|
362
|
+
* and capped per client ({@link MAX_RECORDED_OUTCOMES_PER_CLIENT}). Seeded from the
|
|
363
|
+
* journal replay each incarnation (`finishBoot` — so it reflects what THIS
|
|
364
|
+
* incarnation's replay produced), appended live, and cleared with the incarnation. */
|
|
365
|
+
private readonly outcomes = new Map<string, Map<number, RecordedOutcome>>();
|
|
366
|
+
/** The group-commit window: entries awaiting the next journal append. */
|
|
367
|
+
private ackQueue: RoomJournalEntry[] = [];
|
|
368
|
+
private ackTimer: ReturnType<typeof setTimeout> | null = null;
|
|
369
|
+
/** Serializes journal appends (one in flight; acks apply in append order). */
|
|
370
|
+
private ackChain: Promise<void> = Promise.resolve();
|
|
371
|
+
|
|
372
|
+
// ------------------------------ flush state (§5.3) ------------------------------
|
|
373
|
+
/** The placement epoch (§2.5), claimed once per shell process at start. 0 = no
|
|
374
|
+
* authority configured. */
|
|
375
|
+
private placementEpoch = 0;
|
|
376
|
+
/** The next flush-stream seq — also the (zero-padded) wire offset. Seeded from the
|
|
377
|
+
* journal so offset strings stay monotone across restarts sharing one journal. */
|
|
378
|
+
private flushSeq = 1;
|
|
379
|
+
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
380
|
+
/** One flush settlement in flight at a time (the wasm room guards too). */
|
|
381
|
+
private flushBusy = false;
|
|
382
|
+
/** The settlement chain `flushNow()` awaits. */
|
|
383
|
+
private flushChain: Promise<void> = Promise.resolve();
|
|
384
|
+
private flushesConfirmed = 0;
|
|
385
|
+
/** The last flush seq that COMMITTED at the authority (§4.2/§5.4 `flush_ok`): the value the
|
|
386
|
+
* `/drain` control reports as `finalFlushSeq`, the fence a downgraded client's ghost waits on
|
|
387
|
+
* (`_rindle_room_watermark(doc) ≥ finalFlushSeq`, and the watermark row's `flush_seq` IS this
|
|
388
|
+
* offset — consumer.rs). Seeded from the journal at boot (a re-booted room that already flushed
|
|
389
|
+
* reports its journaled max, not 0); 0 for a never-flushed room. */
|
|
390
|
+
private lastCommittedFlushSeq = 0;
|
|
391
|
+
/** Fenced at the authority (§2.5): this room is superseded — terminal. */
|
|
392
|
+
private moved = false;
|
|
393
|
+
|
|
394
|
+
// --------------------- §301 upstream-absorption advert (301-ECHO-FENCE §1.2) ---------------------
|
|
395
|
+
/** The daemon boot id the upstream connection serves (each upstream `nhello` stamps it) —
|
|
396
|
+
* rides the downstream progress frames as `upstreamBoot`, the §2.4 boot-rule input. */
|
|
397
|
+
private upstreamBoot: string | undefined;
|
|
398
|
+
/** The daemon's released `cvMin` this room has provably absorbed through — recorded from the
|
|
399
|
+
* upstream `progress` frames (same-socket ordering: every nbatch at-or-below it was already
|
|
400
|
+
* applied when the frame is read), NOT from the applied head cv (a sparse footprint would
|
|
401
|
+
* starve the advert while the daemon advances). Rides downstream as `upstreamCv`;
|
|
402
|
+
* `undefined` until the first upstream progress of the current daemon boot. */
|
|
403
|
+
private upstreamCvMin: number | undefined;
|
|
404
|
+
|
|
405
|
+
constructor(opts: RoomShellOptions) {
|
|
406
|
+
this.opts = opts;
|
|
407
|
+
this.log = opts.log ?? (() => {});
|
|
408
|
+
if (opts.downstream.writes) {
|
|
409
|
+
if (opts.downstream.writes.ownedTables.length === 0) {
|
|
410
|
+
throw new Error("writes.ownedTables must name at least one table");
|
|
411
|
+
}
|
|
412
|
+
// H-iv-b: the boot-wire scopes may only ever NARROW the host's owned set, never
|
|
413
|
+
// extend it — a wider server scope would let mutators write tables this
|
|
414
|
+
// deployment never declared writable (self-hoster semantics preserved). Loud at
|
|
415
|
+
// construction, exactly like the empty-owned check above.
|
|
416
|
+
const scopes = opts.downstream.writes.scopes;
|
|
417
|
+
if (scopes !== undefined) {
|
|
418
|
+
const owned = new Set(opts.downstream.writes.ownedTables);
|
|
419
|
+
const rogue = scopes
|
|
420
|
+
.filter((s) => s.writable.kind !== "none" && !owned.has(s.table))
|
|
421
|
+
.map((s) => s.table);
|
|
422
|
+
if (rogue.length > 0) {
|
|
423
|
+
throw new Error(
|
|
424
|
+
`writes.scopes marks ${rogue.map((t) => `\`${t}\``).join(", ")} writable, but the ` +
|
|
425
|
+
`host's writes.ownedTables does not include ${rogue.length === 1 ? "it" : "them"} — ` +
|
|
426
|
+
`the boot-wire scopes may only ever narrow the host's owned set, never extend it`,
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
// The armed-scope fingerprint: what the gate enforces, hashed once, for the
|
|
431
|
+
// skew check on the token path. `undefined` scopes = v1 ungated → no check.
|
|
432
|
+
if (scopes !== undefined) this.armedScopesHash = scopeSpecsHash(scopes);
|
|
433
|
+
this.journal = opts.downstream.writes.journal ?? memoryJournal();
|
|
434
|
+
}
|
|
435
|
+
this.wss = new WebSocketServer({ port: opts.port ?? 0, host: "127.0.0.1" });
|
|
436
|
+
this.wss.on("connection", (ws) => this.serveDownstream(ws));
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
async start(): Promise<void> {
|
|
440
|
+
await initRoomWasm();
|
|
441
|
+
await new Promise<void>((resolve) => {
|
|
442
|
+
if (this.wss.address() !== null) return resolve();
|
|
443
|
+
this.wss.on("listening", () => resolve());
|
|
444
|
+
});
|
|
445
|
+
if (this.opts.downstream.control) {
|
|
446
|
+
await this.startControl(this.opts.downstream.control);
|
|
447
|
+
}
|
|
448
|
+
const interval = this.opts.downstream.sweepIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
449
|
+
this.sweepTimer = setInterval(() => this.sweep(), interval);
|
|
450
|
+
this.sweepTimer.unref?.();
|
|
451
|
+
const authority = this.opts.downstream.writes?.authority;
|
|
452
|
+
if (authority && this.journal) {
|
|
453
|
+
// Seed the flush stream past everything this journal ever numbered, then claim
|
|
454
|
+
// the placement epoch (§2.5) — once per process, BEFORE anything else touches
|
|
455
|
+
// the authority: from this moment every stale body a dead predecessor left
|
|
456
|
+
// mid-network is fenced, which is what closes the probe race for good.
|
|
457
|
+
// Unconfirmed batches settle per incarnation in finishBoot — a prior process's
|
|
458
|
+
// records are fenced there (routine: their mutations recover by envelope
|
|
459
|
+
// replay under OUR epoch), our own land or dedup.
|
|
460
|
+
const { maxSeq } = await this.journal.replayFlushes();
|
|
461
|
+
this.flushSeq = maxSeq + 1;
|
|
462
|
+
this.lastCommittedFlushSeq = maxSeq; // journal-seeded (self-corrects upward on each flush_ok)
|
|
463
|
+
this.placementEpoch = await authority.claimEpoch(this.opts.downstream.docId);
|
|
464
|
+
this.log(`placement epoch ${this.placementEpoch}`);
|
|
465
|
+
}
|
|
466
|
+
this.leaseToken = await mintLease(this.opts.upstream);
|
|
467
|
+
this.connectUpstream();
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
get port(): number {
|
|
471
|
+
const addr = this.wss.address();
|
|
472
|
+
return typeof addr === "object" && addr !== null ? addr.port : 0;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
get controlPort(): number {
|
|
476
|
+
const addr = this.control?.address();
|
|
477
|
+
return typeof addr === "object" && addr !== null ? addr.port : 0;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
awaitLive(): Promise<void> {
|
|
481
|
+
if (this.live) return Promise.resolve();
|
|
482
|
+
return new Promise((resolve) => this.liveWaiters.push(resolve));
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
cv(): number | undefined {
|
|
486
|
+
return this.room?.cv();
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
upstreamEpoch(): number | undefined {
|
|
490
|
+
return this.room?.epoch();
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
bootId(): string {
|
|
494
|
+
return this.incarnationBootId;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
async close(): Promise<void> {
|
|
498
|
+
this.closed = true;
|
|
499
|
+
if (this.sweepTimer) clearInterval(this.sweepTimer);
|
|
500
|
+
if (this.ackTimer !== null) {
|
|
501
|
+
clearTimeout(this.ackTimer);
|
|
502
|
+
this.ackTimer = null;
|
|
503
|
+
}
|
|
504
|
+
if (this.flushTimer !== null) {
|
|
505
|
+
clearTimeout(this.flushTimer);
|
|
506
|
+
this.flushTimer = null;
|
|
507
|
+
}
|
|
508
|
+
this.upstream?.close();
|
|
509
|
+
for (const client of this.wss.clients) client.close();
|
|
510
|
+
await new Promise<void>((resolve) => this.wss.close(() => resolve()));
|
|
511
|
+
if (this.control) {
|
|
512
|
+
await new Promise<void>((resolve) => this.control?.close(() => resolve()));
|
|
513
|
+
}
|
|
514
|
+
if (this.room) {
|
|
515
|
+
this.room.free();
|
|
516
|
+
this.room = null;
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// ------------------------------ upstream leg ------------------------------
|
|
521
|
+
|
|
522
|
+
private connectUpstream(): void {
|
|
523
|
+
if (this.closed) return;
|
|
524
|
+
const ws = new WebSocket(this.opts.upstream.wsUrl);
|
|
525
|
+
this.upstream = ws;
|
|
526
|
+
ws.on("open", () => {
|
|
527
|
+
this.reconnectDelayMs = RESUBSCRIBE_DELAY_MS;
|
|
528
|
+
send(ws, {
|
|
529
|
+
t: "init",
|
|
530
|
+
clientID: this.opts.upstream.clientId ?? `room-shell-${this.incarnationBootId}`,
|
|
531
|
+
});
|
|
532
|
+
this.sendSubscribe();
|
|
533
|
+
});
|
|
534
|
+
ws.on("message", (data) => this.onUpstreamFrame(data));
|
|
535
|
+
ws.on("error", (err) => this.log(`upstream socket error: ${String(err)}`));
|
|
536
|
+
ws.on("close", () => {
|
|
537
|
+
if (this.closed) return;
|
|
538
|
+
// A subscribe may have died with the socket — the fresh connection must be free
|
|
539
|
+
// to send its own.
|
|
540
|
+
this.subscribeInFlight = false;
|
|
541
|
+
this.incarnationDead("upstream socket closed");
|
|
542
|
+
const delay = this.reconnectDelayMs;
|
|
543
|
+
this.reconnectDelayMs = Math.min(delay * 2, RECONNECT_MAX_MS);
|
|
544
|
+
this.log(`upstream reconnect in ${delay}ms`);
|
|
545
|
+
setTimeout(() => this.connectUpstream(), delay).unref?.();
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/** (Re-)send the upstream subscribe. On an already-open socket rindled treats a
|
|
550
|
+
* re-send for the same queryId as gap recovery: tears down the old attachment, bumps
|
|
551
|
+
* the epoch, re-snapshots. */
|
|
552
|
+
private sendSubscribe(): void {
|
|
553
|
+
if (this.closed || this.subscribeInFlight) return;
|
|
554
|
+
if (this.upstream?.readyState !== WebSocket.OPEN) return; // reconnect path re-enters
|
|
555
|
+
this.subscribeInFlight = true;
|
|
556
|
+
send(this.upstream, { t: "subscribe", queryId: UPSTREAM_QID, leaseToken: this.leaseToken });
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
private onUpstreamFrame(data: RawData): void {
|
|
560
|
+
let frame: Record<string, unknown>;
|
|
561
|
+
try {
|
|
562
|
+
frame = JSON.parse(String(data)) as Record<string, unknown>;
|
|
563
|
+
} catch {
|
|
564
|
+
// Not even JSON: the transport is garbage — violation posture.
|
|
565
|
+
this.violation("upstream frame is not JSON");
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
if (frame.queryId !== undefined && frame.queryId !== UPSTREAM_QID) return;
|
|
569
|
+
switch (frame.t) {
|
|
570
|
+
case "nhello": {
|
|
571
|
+
this.subscribeInFlight = false;
|
|
572
|
+
// §301: the daemon boot this connection now serves. A CHANGED boot resets the absorbed
|
|
573
|
+
// cv watermark — the new boot's cv space starts over, and the advert must never pair an
|
|
574
|
+
// old cv with a new boot (the client's §2.4 boot rule keys on the pair).
|
|
575
|
+
const boot = typeof frame.bootId === "string" ? frame.bootId : undefined;
|
|
576
|
+
if (boot !== this.upstreamBoot) {
|
|
577
|
+
this.upstreamBoot = boot;
|
|
578
|
+
this.upstreamCvMin = undefined;
|
|
579
|
+
}
|
|
580
|
+
// A hello supersedes any prior incarnation (e.g. rindled restarted and
|
|
581
|
+
// re-served us): the old store is gone either way.
|
|
582
|
+
if (this.room) this.incarnationDead("superseded by a new upstream hello");
|
|
583
|
+
try {
|
|
584
|
+
this.room = WasmRoom.open(
|
|
585
|
+
JSON.stringify(frame.hello),
|
|
586
|
+
this.opts.downstream.idleTtlMs ?? DEFAULT_IDLE_TTL_MS,
|
|
587
|
+
);
|
|
588
|
+
this.tableShapes = shapesOf(frame.hello);
|
|
589
|
+
this.log(`upstream hello: epoch ${this.room.epoch()}`);
|
|
590
|
+
} catch (e) {
|
|
591
|
+
this.violation(`upstream hello rejected: ${String(e)}`);
|
|
592
|
+
}
|
|
593
|
+
break;
|
|
594
|
+
}
|
|
595
|
+
case "nbatch": {
|
|
596
|
+
if (!this.room) return; // pre-hello or post-death stragglers: drop
|
|
597
|
+
let status: { applied: string; rows?: number; ops?: number };
|
|
598
|
+
try {
|
|
599
|
+
status = JSON.parse(this.room.apply(JSON.stringify(frame.batch))) as typeof status;
|
|
600
|
+
} catch (e) {
|
|
601
|
+
this.violation(`upstream apply failed: ${String(e)}`);
|
|
602
|
+
return;
|
|
603
|
+
}
|
|
604
|
+
if (status.applied === "snapshot") {
|
|
605
|
+
this.log(`upstream snapshot: ${status.rows} rows @ cv ${this.room.cv()}`);
|
|
606
|
+
void this.finishBoot(this.room);
|
|
607
|
+
} else if (status.applied === "live") {
|
|
608
|
+
this.drainDownstream();
|
|
609
|
+
} else if (status.applied === "staleEpoch") {
|
|
610
|
+
this.log("dropped stale-epoch upstream frame");
|
|
611
|
+
}
|
|
612
|
+
break;
|
|
613
|
+
}
|
|
614
|
+
case "progress": {
|
|
615
|
+
// §301 (301-ECHO-FENCE-DESIGN.md §1.2): the daemon's release point is THE provable
|
|
616
|
+
// absorption statement — "everything ≤ cvMin this room will ever be sent has been
|
|
617
|
+
// sent", and same-socket ordering means it is already applied. Record it and
|
|
618
|
+
// RE-ADVERTISE to every subscribed connection even with no data flowing (an
|
|
619
|
+
// out-of-footprint echo's fence must clear without waiting for unrelated churn); the
|
|
620
|
+
// room's own cvMin in that frame is unchanged, so clients release nothing new.
|
|
621
|
+
const cvMin = (frame.frame as { cvMin?: unknown } | undefined)?.cvMin;
|
|
622
|
+
if (typeof cvMin === "number" && cvMin > (this.upstreamCvMin ?? -1)) {
|
|
623
|
+
this.upstreamCvMin = cvMin;
|
|
624
|
+
const touched = new Set<ConnState>();
|
|
625
|
+
for (const meta of this.subs.values()) touched.add(meta.conn);
|
|
626
|
+
for (const conn of touched) this.sendProgress(conn);
|
|
627
|
+
}
|
|
628
|
+
break;
|
|
629
|
+
}
|
|
630
|
+
case "queryError": {
|
|
631
|
+
// The lease died (expiry, dematerialize, daemon restart): re-mint, re-subscribe.
|
|
632
|
+
this.subscribeInFlight = false;
|
|
633
|
+
this.incarnationDead(`upstream queryError: ${String(frame.message)}`);
|
|
634
|
+
void mintLease(this.opts.upstream)
|
|
635
|
+
.then((token) => {
|
|
636
|
+
this.leaseToken = token;
|
|
637
|
+
setTimeout(() => this.sendSubscribe(), RESUBSCRIBE_DELAY_MS).unref?.();
|
|
638
|
+
})
|
|
639
|
+
.catch((e) => {
|
|
640
|
+
this.log(`lease re-mint failed: ${String(e)}`);
|
|
641
|
+
// The socket is still up; retry the whole path after a beat.
|
|
642
|
+
setTimeout(() => {
|
|
643
|
+
if (!this.closed) this.onUpstreamFrame(data);
|
|
644
|
+
}, RECONNECT_MAX_MS).unref?.();
|
|
645
|
+
});
|
|
646
|
+
break;
|
|
647
|
+
}
|
|
648
|
+
default:
|
|
649
|
+
break;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/** A §3.4 violation: kill the incarnation and re-subscribe (never patch up). */
|
|
654
|
+
private violation(reason: string): void {
|
|
655
|
+
this.incarnationDead(reason);
|
|
656
|
+
setTimeout(() => this.sendSubscribe(), RESUBSCRIBE_DELAY_MS).unref?.();
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
private incarnationDead(reason: string): void {
|
|
660
|
+
this.log(`incarnation dead: ${reason}`);
|
|
661
|
+
if (this.room) {
|
|
662
|
+
this.room.free();
|
|
663
|
+
this.room = null;
|
|
664
|
+
}
|
|
665
|
+
this.live = false;
|
|
666
|
+
this.incarnationBootId = randomUUID();
|
|
667
|
+
// Downstream subscriptions die with the incarnation (§3.4): close the sockets;
|
|
668
|
+
// reconnecting clients re-subscribe against the next incarnation.
|
|
669
|
+
for (const meta of this.subs.values()) {
|
|
670
|
+
meta.ws.close(1012, "room re-subscribing");
|
|
671
|
+
}
|
|
672
|
+
this.subs.clear();
|
|
673
|
+
this.pendingSubs = [];
|
|
674
|
+
// The write plane's optimism dies with the incarnation too: whatever was not yet
|
|
675
|
+
// journaled was never acked, and never will be (§8.1's `applied` class). The
|
|
676
|
+
// journal itself outlives us — the next incarnation replays it.
|
|
677
|
+
this.pendingMutes = [];
|
|
678
|
+
this.unacked.clear();
|
|
679
|
+
// Recorded outcomes die with the incarnation too — the next incarnation reseeds
|
|
680
|
+
// them from its own journal replay (finishBoot), which is the source of truth for
|
|
681
|
+
// what THAT incarnation's base produced.
|
|
682
|
+
this.outcomes.clear();
|
|
683
|
+
this.ackQueue = [];
|
|
684
|
+
if (this.ackTimer !== null) {
|
|
685
|
+
clearTimeout(this.ackTimer);
|
|
686
|
+
this.ackTimer = null;
|
|
687
|
+
}
|
|
688
|
+
// The flush debounce dies with the incarnation; an in-flight settlement keeps
|
|
689
|
+
// running (its batch is journaled and epoch-bound — the bootId guard keeps its
|
|
690
|
+
// outcome from touching the next incarnation's state).
|
|
691
|
+
if (this.flushTimer !== null) {
|
|
692
|
+
clearTimeout(this.flushTimer);
|
|
693
|
+
this.flushTimer = null;
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
/** After the seq-0 snapshot: enable the write plane, settle any unconfirmed flush
|
|
698
|
+
* batches, probe the authority's durable lmids, and replay the journal onto the
|
|
699
|
+
* fresh base (§3.3 — pending re-invoked; a mid the probe covered replays as dedup,
|
|
700
|
+
* its effects already in the snapshot; everything replayed acked, since journal
|
|
701
|
+
* presence IS the ack), then start serving. The order is load-bearing: the
|
|
702
|
+
* resubmission SETTLES before the probe, so a batch a dead incarnation left
|
|
703
|
+
* mid-network can never land between the probe and our first flush. No subscriber
|
|
704
|
+
* exists yet (they died with the previous incarnation), so replay fans out to
|
|
705
|
+
* nobody — the snapshots the queued subscribes get below already include the
|
|
706
|
+
* replayed rows. */
|
|
707
|
+
private async finishBoot(room: WasmRoom): Promise<void> {
|
|
708
|
+
const writes = this.opts.downstream.writes;
|
|
709
|
+
if (writes && this.journal) {
|
|
710
|
+
let entries: RoomJournalEntry[];
|
|
711
|
+
try {
|
|
712
|
+
// H-iv-b: boot-wire scopes arm the §3.3 commit gate (v2); without them the v1
|
|
713
|
+
// table-granular plane enables byte-identically to before. Either enable
|
|
714
|
+
// throws loudly on a malformed input and leaves writes disabled — the
|
|
715
|
+
// violation below keeps that from becoming a half-enabled room.
|
|
716
|
+
if (writes.scopes !== undefined) {
|
|
717
|
+
room.enableWritesV2(JSON.stringify(writes.scopes));
|
|
718
|
+
} else {
|
|
719
|
+
room.enableWrites(JSON.stringify(writes.ownedTables));
|
|
720
|
+
}
|
|
721
|
+
entries = await this.journal.replay();
|
|
722
|
+
} catch (e) {
|
|
723
|
+
this.violation(`write-plane boot failed: ${String(e)}`);
|
|
724
|
+
return;
|
|
725
|
+
}
|
|
726
|
+
if (this.room !== room) return; // an incarnation death raced the replay
|
|
727
|
+
if (writes.authority) {
|
|
728
|
+
try {
|
|
729
|
+
await this.settleUnconfirmedFlushes(writes.authority);
|
|
730
|
+
} catch (e) {
|
|
731
|
+
if (this.moved || this.closed) return;
|
|
732
|
+
this.violation(`flush resubmission failed: ${String(e)}`);
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
if (this.room !== room) return;
|
|
736
|
+
const clients = [...new Set(entries.map((e) => e.clientID))];
|
|
737
|
+
if (clients.length > 0) {
|
|
738
|
+
let lmids: Record<string, number>;
|
|
739
|
+
try {
|
|
740
|
+
lmids = await withNetRetry(
|
|
741
|
+
() => writes.authority!.lmids(this.opts.downstream.docId, clients),
|
|
742
|
+
() => this.closed || this.room !== room,
|
|
743
|
+
this.log,
|
|
744
|
+
);
|
|
745
|
+
} catch (e) {
|
|
746
|
+
if (this.room !== room || this.closed) return;
|
|
747
|
+
this.violation(`durable-lmid probe failed: ${String(e)}`);
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
if (this.room !== room) return;
|
|
751
|
+
const seeds = Object.entries(lmids)
|
|
752
|
+
.filter(([, lmid]) => lmid > 0)
|
|
753
|
+
.map(([clientID, lmid]) => ({ clientID, lmid }));
|
|
754
|
+
if (seeds.length > 0) {
|
|
755
|
+
try {
|
|
756
|
+
room.seedDurable(JSON.stringify(seeds));
|
|
757
|
+
room.commitAll(); // drop the (subscriber-less) ledger fanout
|
|
758
|
+
} catch (e) {
|
|
759
|
+
this.violation(`durable seed failed: ${String(e)}`);
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
this.log(`durable seed: ${seeds.map((s) => `${s.clientID}→${s.lmid}`).join(", ")}`);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
try {
|
|
767
|
+
for (const entry of entries) {
|
|
768
|
+
const res = this.runMutation(room, entry);
|
|
769
|
+
// Seed the recorded-outcome map with what THIS incarnation's replay
|
|
770
|
+
// produced. That includes the H-iv-a replay gotcha: an entry journaled
|
|
771
|
+
// APPLIED can legitimately DEOPT (or reject) re-invoked against the moved
|
|
772
|
+
// base — the §3.3 rebase class. The journal record is never rewritten
|
|
773
|
+
// (history stays what the acking incarnation observed); the map is this
|
|
774
|
+
// incarnation's answer sheet for re-sent mids.
|
|
775
|
+
if (res.outcome === "rejected" || res.outcome === "deopt") {
|
|
776
|
+
this.recordOutcome(entry.clientID, entry.mid, {
|
|
777
|
+
kind: res.outcome,
|
|
778
|
+
...(res.reason !== undefined ? { reason: res.reason } : {}),
|
|
779
|
+
...(res.outcome === "deopt" ? { name: entry.name, args: entry.args } : {}),
|
|
780
|
+
});
|
|
781
|
+
if (journalEntryOutcome(entry) === "applied") {
|
|
782
|
+
this.log(
|
|
783
|
+
`replayed APPLIED mutation ${entry.clientID}:${entry.mid} (\`${entry.name}\`) ` +
|
|
784
|
+
`now ${res.outcome}s against the moved base — mid stays burnt, no effects`,
|
|
785
|
+
);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
if (entries.length > 0) {
|
|
790
|
+
room.ack(
|
|
791
|
+
JSON.stringify(entries.map(({ clientID, mid }) => ({ clientID, mid }))),
|
|
792
|
+
);
|
|
793
|
+
room.commitAll(); // drop the (subscriber-less) fanout of the replay
|
|
794
|
+
}
|
|
795
|
+
} catch (e) {
|
|
796
|
+
this.violation(`journal replay failed: ${String(e)}`);
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
if (entries.length > 0) {
|
|
800
|
+
this.log(`journal replay: ${entries.length} mutation(s) re-invoked and acked`);
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
this.becomeLive();
|
|
804
|
+
this.scheduleFlush(); // replay may have left dirty entries / unflushed lmids
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
/** Resubmit every unconfirmed journaled flush batch, byte-identically, in seq order
|
|
808
|
+
* (§3.3/§5.3 step 4). Outcomes: committed or deduped → confirm; fenced under an
|
|
809
|
+
* OLD epoch → routine cleanup (a prior process's batch — its mutations recover by
|
|
810
|
+
* replay under OUR epoch); fenced under OUR epoch → another room claimed the doc:
|
|
811
|
+
* terminal (`room_moved`); conflict → drop (the replay + next flush re-derive the
|
|
812
|
+
* net effect against the authority's current rows); identity mismatch → fatal. */
|
|
813
|
+
private async settleUnconfirmedFlushes(authority: RoomAuthority): Promise<void> {
|
|
814
|
+
if (!this.journal) return;
|
|
815
|
+
const { records } = await this.journal.replayFlushes();
|
|
816
|
+
for (const record of records) {
|
|
817
|
+
if (this.closed || this.moved) return;
|
|
818
|
+
let res: Awaited<ReturnType<RoomAuthority["applyRowChangeTxn"]>>;
|
|
819
|
+
try {
|
|
820
|
+
res = await withNetRetry(
|
|
821
|
+
() => authority.applyRowChangeTxn(record.body),
|
|
822
|
+
() => this.closed || this.moved,
|
|
823
|
+
this.log,
|
|
824
|
+
);
|
|
825
|
+
} catch (e) {
|
|
826
|
+
if ((e as { fatal?: boolean })?.fatal !== true) throw e;
|
|
827
|
+
// The §8.3 identity check tripped on a journaled record — a same-id/
|
|
828
|
+
// different-body bug. LOUD, dropped, never a silent dedup and never a
|
|
829
|
+
// violation loop: the batch is dead; the envelope replay re-derives its
|
|
830
|
+
// effects under a fresh flush id.
|
|
831
|
+
this.log(`flush ${record.seq} DROPPED: batch identity mismatch: ${String(e)}`);
|
|
832
|
+
await this.journal.confirmFlush(record.seq);
|
|
833
|
+
continue;
|
|
834
|
+
}
|
|
835
|
+
if (res.kind === "ok") {
|
|
836
|
+
await this.journal.confirmFlush(record.seq);
|
|
837
|
+
this.log(
|
|
838
|
+
`flush ${record.seq} settled at boot: ${res.applied ? "applied" : "already applied"}`,
|
|
839
|
+
);
|
|
840
|
+
} else if (res.kind === "conflict") {
|
|
841
|
+
await this.journal.confirmFlush(record.seq);
|
|
842
|
+
this.log(`flush ${record.seq} dropped at boot: conflict (replay supersedes)`);
|
|
843
|
+
} else {
|
|
844
|
+
await this.journal.confirmFlush(record.seq);
|
|
845
|
+
if (record.epoch === this.placementEpoch) {
|
|
846
|
+
this.roomMoved(`flush ${record.seq} fenced under our epoch`);
|
|
847
|
+
return;
|
|
848
|
+
}
|
|
849
|
+
this.log(`flush ${record.seq} (old epoch ${record.epoch}) fenced: dropped`);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
private becomeLive(): void {
|
|
855
|
+
this.live = true;
|
|
856
|
+
const waiters = this.liveWaiters;
|
|
857
|
+
this.liveWaiters = [];
|
|
858
|
+
for (const w of waiters) w();
|
|
859
|
+
const queued = this.pendingSubs;
|
|
860
|
+
this.pendingSubs = [];
|
|
861
|
+
for (const { ws, conn, msg } of queued) {
|
|
862
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
863
|
+
conn.busy = conn.busy.then(() => this.handleSubscribe(ws, conn, msg));
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
const mutes = this.pendingMutes;
|
|
867
|
+
this.pendingMutes = [];
|
|
868
|
+
for (const { ws, conn, envelope } of mutes) {
|
|
869
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
870
|
+
conn.busy = conn.busy.then(() => this.handlePushMutation(ws, conn, envelope));
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
/** After any commit (an upstream apply, a local mutation, an ack): one commitAll
|
|
876
|
+
* drains every query's net delta, fanned out per subscriber envelope (empty → no
|
|
877
|
+
* frame, no seq — every stream stays gap-free over emitted frames), then each
|
|
878
|
+
* touched connection gets its `progress` release point. */
|
|
879
|
+
private drainDownstream(): void {
|
|
880
|
+
if (!this.room) return;
|
|
881
|
+
const frames = JSON.parse(this.room.commitAll()) as Array<{ sub: string; batch: unknown }>;
|
|
882
|
+
const touched = new Set<ConnState>();
|
|
883
|
+
for (const { sub, batch } of frames) {
|
|
884
|
+
const meta = this.subs.get(sub);
|
|
885
|
+
if (meta) {
|
|
886
|
+
this.sendBudgeted(meta.conn, { t: "nbatch", queryId: meta.clientQid, batch });
|
|
887
|
+
touched.add(meta.conn);
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
for (const conn of touched) this.sendProgress(conn);
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
/** The connection-level release point (`{t:"progress", frame:{cvMin}}`): `head_cv`,
|
|
894
|
+
* except an author with un-acked mutations holds below its oldest one's apply-cv —
|
|
895
|
+
* its own data must never release ahead of the lmid advance that confirms it
|
|
896
|
+
* (§1.3.1: the client would re-invoke the mutator on top of its own effect). */
|
|
897
|
+
private sendProgress(conn: ConnState): void {
|
|
898
|
+
if (!this.room) return;
|
|
899
|
+
let cvMin = this.room.headCv();
|
|
900
|
+
if (conn.clientID) {
|
|
901
|
+
const held = this.unacked.get(conn.clientID);
|
|
902
|
+
if (held && held.length > 0) {
|
|
903
|
+
cvMin = Math.min(cvMin, held[0].applyCv - 1);
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
// §301: stamp the upstream-absorption advert (once the current daemon boot has released
|
|
907
|
+
// anything to us) — the client's direction-B pin fence input. Optional fields: an old
|
|
908
|
+
// client reads `cvMin` only.
|
|
909
|
+
const frame: { cvMin: number; upstreamCv?: number; upstreamBoot?: string } = { cvMin };
|
|
910
|
+
if (this.upstreamCvMin !== undefined) {
|
|
911
|
+
frame.upstreamCv = this.upstreamCvMin;
|
|
912
|
+
if (this.upstreamBoot !== undefined) frame.upstreamBoot = this.upstreamBoot;
|
|
913
|
+
}
|
|
914
|
+
this.sendBudgeted(conn, { t: "progress", frame });
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
/** §9's per-socket send budget (T8): a socket whose queued bytes would exceed it is
|
|
918
|
+
* TERMINATED — the close handler reclaims its subscriptions, so room memory stays
|
|
919
|
+
* bounded and the client repairs by re-subscribing (a gap, §8.5). Never buffer
|
|
920
|
+
* unboundedly on behalf of one slow reader. */
|
|
921
|
+
private sendBudgeted(conn: ConnState, frame: unknown): void {
|
|
922
|
+
const ws = conn.ws;
|
|
923
|
+
if (ws.readyState !== WebSocket.OPEN) return;
|
|
924
|
+
const text = JSON.stringify(frame);
|
|
925
|
+
const budget = this.opts.downstream.sendBudgetBytes ?? DEFAULT_SEND_BUDGET_BYTES;
|
|
926
|
+
if (ws.bufferedAmount + text.length > budget) {
|
|
927
|
+
this.log(`send budget exceeded (conn ${conn.id}): closed with a gap`);
|
|
928
|
+
ws.terminate();
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
ws.send(text);
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
// ------------------------------ the write path ------------------------------
|
|
935
|
+
|
|
936
|
+
/** Apply one envelope to the room: dedup → run the mutator against a staged tx →
|
|
937
|
+
* commit (or reject/deopt, consuming the mid). Shared verbatim by the live path and
|
|
938
|
+
* the boot-time journal replay (§3.3 — recovery is re-invocation; a journaled
|
|
939
|
+
* NON-APPLIED entry replays its recorded outcome without running anything). Verdict
|
|
940
|
+
* classification (H-iv-b): the §3.3 commit gate's structured verdict and an
|
|
941
|
+
* environment shortfall (a capability the room lacks — `tx.query`) are DEOPTS (the
|
|
942
|
+
* client re-routes the mutation to the daemon stream); an unknown mutator or an
|
|
943
|
+
* authz/validation throw is a FINAL rejection. Throws only when the room is torn
|
|
944
|
+
* (a mid gap, a poisoned commit) — the caller decides the blast radius. */
|
|
945
|
+
private runMutation(
|
|
946
|
+
room: WasmRoom,
|
|
947
|
+
entry: RoomJournalEntry,
|
|
948
|
+
):
|
|
949
|
+
| { outcome: "applied"; applyCv: number }
|
|
950
|
+
| { outcome: "rejected"; reason?: string }
|
|
951
|
+
| { outcome: "deopt"; reason?: string }
|
|
952
|
+
| { outcome: "dedup" } {
|
|
953
|
+
// The envelope rides the wasm tx (Slice I-ii): should this mid end non-applied,
|
|
954
|
+
// its durable outcome row echoes `name`/`args` on a DEOPT — the same
|
|
955
|
+
// self-contained-re-invoke rule as the frame (H-iv-b), but readable through the
|
|
956
|
+
// daemon after downgrade when no room socket exists to deliver one.
|
|
957
|
+
const begin = JSON.parse(
|
|
958
|
+
room.beginMutation(
|
|
959
|
+
entry.clientID,
|
|
960
|
+
entry.mid,
|
|
961
|
+
entry.name,
|
|
962
|
+
entry.args === undefined ? undefined : JSON.stringify(entry.args),
|
|
963
|
+
),
|
|
964
|
+
) as {
|
|
965
|
+
begin: "tx" | "dedup";
|
|
966
|
+
};
|
|
967
|
+
if (begin.begin === "dedup") return { outcome: "dedup" };
|
|
968
|
+
// A journaled non-applied entry replays its RECORDED verdict without running: the
|
|
969
|
+
// live run already consumed the mid with no effects, and re-judging (a deopt that
|
|
970
|
+
// would now PASS against the moved base) would invent effects for a mutation the
|
|
971
|
+
// client was told to re-route elsewhere — a double apply. Passing the verdict
|
|
972
|
+
// through the reject IS the replay re-seed (I-ii): a recorded-but-never-flushed
|
|
973
|
+
// outcome re-enters the core buffer here, so the next flush carries its row —
|
|
974
|
+
// while a mid the boot probe covered dedups above and never re-seeds (its row
|
|
975
|
+
// co-committed with the flush that made it durable). The journal keeps no
|
|
976
|
+
// `reason`, so a replayed row carries kind + envelope only, like the replayed
|
|
977
|
+
// frame.
|
|
978
|
+
const journaled = journalEntryOutcome(entry);
|
|
979
|
+
if (journaled !== "applied") {
|
|
980
|
+
room.rejectMutation(journaled, undefined);
|
|
981
|
+
return { outcome: journaled };
|
|
982
|
+
}
|
|
983
|
+
const mutator = this.opts.downstream.writes?.mutators[entry.name];
|
|
984
|
+
if (!mutator) {
|
|
985
|
+
const reason = `unknown mutator \`${entry.name}\``;
|
|
986
|
+
room.rejectMutation("rejected", reason);
|
|
987
|
+
this.log(`mutation ${entry.clientID}:${entry.mid} rejected: ${reason}`);
|
|
988
|
+
return { outcome: "rejected", reason };
|
|
989
|
+
}
|
|
990
|
+
try {
|
|
991
|
+
// The ambient auth context (managed-writes §3.2): the journaled subject, so a
|
|
992
|
+
// replayed invocation sees exactly the identity the live one did (`""` for
|
|
993
|
+
// entries journaled before the identity plane — unauthenticated).
|
|
994
|
+
const returned = (mutator as (...a: unknown[]) => unknown)(
|
|
995
|
+
mutationTx(room, this.tableShapes),
|
|
996
|
+
entry.args as never,
|
|
997
|
+
{ user: entry.sub ?? "" },
|
|
998
|
+
);
|
|
999
|
+
assertSyncMutatorReturn(returned, entry.name);
|
|
1000
|
+
} catch (e) {
|
|
1001
|
+
// The shell owns the H-iv-b classification, so it rides the reject into the
|
|
1002
|
+
// core's outcome record (the flush carries the durable row) exactly as it rides
|
|
1003
|
+
// the frame below.
|
|
1004
|
+
const message = String((e as Error)?.message ?? e);
|
|
1005
|
+
if (isEnvironmentShortfall(e)) {
|
|
1006
|
+
room.rejectMutation("deopt", "environment");
|
|
1007
|
+
this.log(
|
|
1008
|
+
`mutation \`${entry.name}\` (${entry.clientID}:${entry.mid}) deopted: ` +
|
|
1009
|
+
`environment shortfall: ${message}`,
|
|
1010
|
+
);
|
|
1011
|
+
return { outcome: "deopt", reason: "environment" };
|
|
1012
|
+
}
|
|
1013
|
+
room.rejectMutation("rejected", message);
|
|
1014
|
+
this.log(`mutation \`${entry.name}\` (${entry.clientID}:${entry.mid}) rejected: ${message}`);
|
|
1015
|
+
return { outcome: "rejected", reason: message };
|
|
1016
|
+
}
|
|
1017
|
+
const out = JSON.parse(room.commitMutation()) as {
|
|
1018
|
+
headCv?: number;
|
|
1019
|
+
deopt?: { reason: string; table: string; pk: unknown[] };
|
|
1020
|
+
};
|
|
1021
|
+
if (out.deopt !== undefined) {
|
|
1022
|
+
// The gate refused at commit and consumed the tx INTERNALLY (mid burnt,
|
|
1023
|
+
// watermark advanced, no head commit) — do NOT call rejectMutation here.
|
|
1024
|
+
this.log(
|
|
1025
|
+
`mutation \`${entry.name}\` (${entry.clientID}:${entry.mid}) deopted: ` +
|
|
1026
|
+
`${out.deopt.reason} on \`${out.deopt.table}\` pk ${JSON.stringify(out.deopt.pk)}`,
|
|
1027
|
+
);
|
|
1028
|
+
return { outcome: "deopt", reason: out.deopt.reason };
|
|
1029
|
+
}
|
|
1030
|
+
return { outcome: "applied", applyCv: out.headCv as number };
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
/** The live `pushMutation` path (§5.1): apply → fan the data out NOW → journal under
|
|
1034
|
+
* the group commit → ack (the ledger advance) once the append resolves. */
|
|
1035
|
+
private handlePushMutation(ws: WebSocket, conn: ConnState, envelope: PushEnvelope): void {
|
|
1036
|
+
const fail = (message: string) => send(ws, { t: "error", message });
|
|
1037
|
+
if (!this.opts.downstream.writes || !this.journal) {
|
|
1038
|
+
fail("this room is read-only (no write plane configured)");
|
|
1039
|
+
return;
|
|
1040
|
+
}
|
|
1041
|
+
if (!conn.clientID) {
|
|
1042
|
+
fail("init with a clientID before pushMutation");
|
|
1043
|
+
return;
|
|
1044
|
+
}
|
|
1045
|
+
if (envelope.clientID !== conn.clientID) {
|
|
1046
|
+
// The envelope's clientID is bound to the connection identity — a session may
|
|
1047
|
+
// not write another client's mid stream.
|
|
1048
|
+
fail("envelope clientID does not match the connection identity");
|
|
1049
|
+
return;
|
|
1050
|
+
}
|
|
1051
|
+
if (!this.live || !this.room) {
|
|
1052
|
+
this.pendingMutes.push({ ws, conn, envelope });
|
|
1053
|
+
return;
|
|
1054
|
+
}
|
|
1055
|
+
// Checked at execution time (after the live gate), so a mutation queued behind a
|
|
1056
|
+
// still-pending subscribe is judged AFTER that subscribe bound the subject.
|
|
1057
|
+
if (conn.sub === null) {
|
|
1058
|
+
fail("pushMutation requires an authenticated subject — subscribe with a lease token first");
|
|
1059
|
+
return;
|
|
1060
|
+
}
|
|
1061
|
+
const room = this.room;
|
|
1062
|
+
let result: ReturnType<Shell["runMutation"]>;
|
|
1063
|
+
try {
|
|
1064
|
+
result = this.runMutation(room, { ...envelope, sub: conn.sub });
|
|
1065
|
+
} catch (e) {
|
|
1066
|
+
const message = String((e as Error)?.message ?? e);
|
|
1067
|
+
if (room.isPoisoned()) {
|
|
1068
|
+
this.violation(`mutation commit tore the head: ${message}`);
|
|
1069
|
+
} else {
|
|
1070
|
+
// The mid-gap contract: the exact "mutation gap …" text the client's
|
|
1071
|
+
// recovery keys on rides an error frame.
|
|
1072
|
+
fail(message);
|
|
1073
|
+
}
|
|
1074
|
+
return;
|
|
1075
|
+
}
|
|
1076
|
+
if (result.outcome === "dedup") {
|
|
1077
|
+
// Absorbed; the ledger already covers it (or its in-flight ack will). But a
|
|
1078
|
+
// re-sent NON-APPLIED mid is ANSWERED with its recorded outcome (H-iv-b): the
|
|
1079
|
+
// client may have missed the original frame (reconnect), and silence would
|
|
1080
|
+
// leave its deopted mutation parked forever.
|
|
1081
|
+
const recorded = this.outcomes.get(conn.clientID)?.get(envelope.mid);
|
|
1082
|
+
if (recorded !== undefined) this.sendOutcome(conn, envelope.mid, recorded);
|
|
1083
|
+
return;
|
|
1084
|
+
}
|
|
1085
|
+
if (result.outcome === "applied") {
|
|
1086
|
+
const held = this.unacked.get(conn.clientID) ?? [];
|
|
1087
|
+
held.push({ mid: envelope.mid, applyCv: result.applyCv });
|
|
1088
|
+
this.unacked.set(conn.clientID, held);
|
|
1089
|
+
this.drainDownstream(); // §5.1 step 2: never gated on durability
|
|
1090
|
+
} else {
|
|
1091
|
+
// Non-applied (deopt/rejected): record + answer NOW — synchronously, before the
|
|
1092
|
+
// journal enqueue below, so on this ordered socket the `mutationOutcome` frame
|
|
1093
|
+
// always precedes the lmid ack that burns the mid. Applied mutations send
|
|
1094
|
+
// NOTHING (additive frame: old clients drop unknown `t`).
|
|
1095
|
+
const recorded: RecordedOutcome = {
|
|
1096
|
+
kind: result.outcome,
|
|
1097
|
+
...(result.reason !== undefined ? { reason: result.reason } : {}),
|
|
1098
|
+
...(result.outcome === "deopt" ? { name: envelope.name, args: envelope.args } : {}),
|
|
1099
|
+
};
|
|
1100
|
+
this.recordOutcome(conn.clientID, envelope.mid, recorded);
|
|
1101
|
+
this.sendOutcome(conn, envelope.mid, recorded);
|
|
1102
|
+
}
|
|
1103
|
+
this.enqueueJournal({
|
|
1104
|
+
clientID: envelope.clientID,
|
|
1105
|
+
mid: envelope.mid,
|
|
1106
|
+
name: envelope.name,
|
|
1107
|
+
args: envelope.args,
|
|
1108
|
+
sub: conn.sub,
|
|
1109
|
+
outcome: result.outcome,
|
|
1110
|
+
// The legacy flag rides alongside BOTH non-applied kinds: a pre-H-iv-b reader
|
|
1111
|
+
// replays either as a consumed-mid-no-effect — exactly right (journal.ts).
|
|
1112
|
+
...(result.outcome !== "applied" ? { rejected: true } : {}),
|
|
1113
|
+
});
|
|
1114
|
+
// §5.1 step 5: the write-behind rides its own debounce — applied mutations dirty
|
|
1115
|
+
// rows, rejected/deopted ones still advance an lmid the authority must eventually
|
|
1116
|
+
// hold.
|
|
1117
|
+
this.scheduleFlush();
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/** Record a non-applied verdict for `(clientID, mid)` — the re-send answer sheet.
|
|
1121
|
+
* Insertion-ordered per client; past {@link MAX_RECORDED_OUTCOMES_PER_CLIENT} the
|
|
1122
|
+
* oldest evicts (see the constant's retention rationale). */
|
|
1123
|
+
private recordOutcome(clientID: string, mid: number, outcome: RecordedOutcome): void {
|
|
1124
|
+
let byMid = this.outcomes.get(clientID);
|
|
1125
|
+
if (byMid === undefined) {
|
|
1126
|
+
byMid = new Map();
|
|
1127
|
+
this.outcomes.set(clientID, byMid);
|
|
1128
|
+
}
|
|
1129
|
+
byMid.delete(mid); // re-recording refreshes recency
|
|
1130
|
+
byMid.set(mid, outcome);
|
|
1131
|
+
while (byMid.size > MAX_RECORDED_OUTCOMES_PER_CLIENT) {
|
|
1132
|
+
byMid.delete(byMid.keys().next().value as number);
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
/** The `mutationOutcome` frame (H-iv-b): `{t, mid, kind, reason?, name?, args?}` on
|
|
1137
|
+
* the mutating client's socket. DEOPT frames echo `name`/`args` so they are
|
|
1138
|
+
* self-contained — a client that already retired the entry can re-invoke from the
|
|
1139
|
+
* frame alone. Sent before the mid's journal append is even enqueued, so it always
|
|
1140
|
+
* precedes the lmid ack on this ordered socket. */
|
|
1141
|
+
private sendOutcome(conn: ConnState, mid: number, o: RecordedOutcome): void {
|
|
1142
|
+
this.sendBudgeted(conn, {
|
|
1143
|
+
t: "mutationOutcome",
|
|
1144
|
+
mid,
|
|
1145
|
+
kind: o.kind,
|
|
1146
|
+
...(o.reason !== undefined ? { reason: o.reason } : {}),
|
|
1147
|
+
...(o.kind === "deopt" ? { name: o.name, args: o.args } : {}),
|
|
1148
|
+
});
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
/** Collect entries for the next group commit (§5.1 step 3: one append per window). */
|
|
1152
|
+
private enqueueJournal(entry: RoomJournalEntry): void {
|
|
1153
|
+
this.ackQueue.push(entry);
|
|
1154
|
+
if (this.ackTimer === null) {
|
|
1155
|
+
const window = this.opts.downstream.writes?.groupCommitMs ?? DEFAULT_GROUP_COMMIT_MS;
|
|
1156
|
+
this.ackTimer = setTimeout(() => this.flushJournal(), window);
|
|
1157
|
+
this.ackTimer.unref?.();
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
private flushJournal(): void {
|
|
1162
|
+
this.ackTimer = null;
|
|
1163
|
+
const batch = this.ackQueue;
|
|
1164
|
+
this.ackQueue = [];
|
|
1165
|
+
if (batch.length === 0 || !this.journal) return;
|
|
1166
|
+
const journal = this.journal;
|
|
1167
|
+
const bootAtFlush = this.incarnationBootId;
|
|
1168
|
+
// One append in flight at a time; acks apply in append order.
|
|
1169
|
+
this.ackChain = this.ackChain.then(async () => {
|
|
1170
|
+
try {
|
|
1171
|
+
await journal.append(batch);
|
|
1172
|
+
} catch (e) {
|
|
1173
|
+
// An ack that might not survive must never be sent (§8.1): the incarnation
|
|
1174
|
+
// dies loudly instead.
|
|
1175
|
+
if (this.incarnationBootId === bootAtFlush && !this.closed) {
|
|
1176
|
+
this.violation(`journal append failed: ${String(e)}`);
|
|
1177
|
+
}
|
|
1178
|
+
return;
|
|
1179
|
+
}
|
|
1180
|
+
if (this.incarnationBootId !== bootAtFlush || !this.room) {
|
|
1181
|
+
return; // the incarnation died mid-append: its optimism died with it
|
|
1182
|
+
}
|
|
1183
|
+
this.applyAck(this.room, batch);
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
/** §5.1 step 4 — the ack: advance the ledger rows, ship the lmid frames, and raise
|
|
1188
|
+
* every author's release point past its confirmed data. */
|
|
1189
|
+
private applyAck(room: WasmRoom, batch: RoomJournalEntry[]): void {
|
|
1190
|
+
let headCv: number | null;
|
|
1191
|
+
try {
|
|
1192
|
+
const entries = batch.map(({ clientID, mid }) => ({ clientID, mid }));
|
|
1193
|
+
headCv = (JSON.parse(room.ack(JSON.stringify(entries))) as { headCv: number | null })
|
|
1194
|
+
.headCv;
|
|
1195
|
+
} catch (e) {
|
|
1196
|
+
this.violation(`ack failed: ${String(e)}`);
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
const authors = new Set<string>();
|
|
1200
|
+
for (const { clientID, mid } of batch) {
|
|
1201
|
+
authors.add(clientID);
|
|
1202
|
+
const held = this.unacked.get(clientID);
|
|
1203
|
+
if (held) {
|
|
1204
|
+
const rest = held.filter((u) => u.mid > mid);
|
|
1205
|
+
if (rest.length > 0) this.unacked.set(clientID, rest);
|
|
1206
|
+
else this.unacked.delete(clientID);
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
if (headCv !== null) {
|
|
1210
|
+
this.drainDownstream(); // the lmid frames (+ progress to their receivers)
|
|
1211
|
+
}
|
|
1212
|
+
// Raise the release point for every author connection even if no frame reached
|
|
1213
|
+
// it in this drain (e.g. its lmid slice was already current).
|
|
1214
|
+
for (const meta of this.subs.values()) {
|
|
1215
|
+
if (meta.conn.clientID && authors.has(meta.conn.clientID)) {
|
|
1216
|
+
this.sendProgress(meta.conn);
|
|
1217
|
+
authors.delete(meta.conn.clientID);
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
// --------------------------- the write-behind flush ---------------------------
|
|
1223
|
+
|
|
1224
|
+
/** Arm the flush debounce (or fire now past the dirty-size threshold). Cheap and
|
|
1225
|
+
* idempotent — called after every applied/rejected mutation and every settlement. */
|
|
1226
|
+
private scheduleFlush(): void {
|
|
1227
|
+
const writes = this.opts.downstream.writes;
|
|
1228
|
+
if (!writes?.authority || this.flushBusy || this.flushTimer !== null) return;
|
|
1229
|
+
if (this.closed || this.moved || !this.live || !this.room) return;
|
|
1230
|
+
const dirtyMax = writes.flushDirtyMax ?? DEFAULT_FLUSH_DIRTY_MAX;
|
|
1231
|
+
const delay =
|
|
1232
|
+
this.room.dirtyLen() >= dirtyMax ? 0 : writes.flushDebounceMs ?? DEFAULT_FLUSH_DEBOUNCE_MS;
|
|
1233
|
+
this.flushTimer = setTimeout(() => {
|
|
1234
|
+
this.flushTimer = null;
|
|
1235
|
+
this.fireFlush();
|
|
1236
|
+
}, delay);
|
|
1237
|
+
this.flushTimer.unref?.();
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
/** Fire the write-behind flush now and await its settlement (tests, drain paths). */
|
|
1241
|
+
flushNow(): Promise<void> {
|
|
1242
|
+
if (this.flushTimer !== null) {
|
|
1243
|
+
clearTimeout(this.flushTimer);
|
|
1244
|
+
this.flushTimer = null;
|
|
1245
|
+
}
|
|
1246
|
+
this.fireFlush();
|
|
1247
|
+
return this.flushChain;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
/** §4.2/§7.4 drain-before-downgrade: fire the write-behind repeatedly until pending AND dirty
|
|
1251
|
+
* are both empty, awaiting each settlement, then report the last COMMITTED flush seq — the
|
|
1252
|
+
* value a downgraded client's frozen ghost fences against
|
|
1253
|
+
* (`_rindle_room_watermark(doc) ≥ finalFlushSeq`). A never-flushed room reports 0. Idempotent:
|
|
1254
|
+
* re-draining an already-quiescent room re-reports the same seq without flushing. It does NOT
|
|
1255
|
+
* refuse later pushes (§4.2 R2: a straggler push after drain flushes at seq+1; the client
|
|
1256
|
+
* ghost's second conjunct — no sent room-domain pending — carries correctness). A room that
|
|
1257
|
+
* fails to quiesce within {@link DRAIN_MAX_ITERATIONS} throws LOUD (never silently). */
|
|
1258
|
+
async drainForDowngrade(): Promise<number> {
|
|
1259
|
+
const writes = this.opts.downstream.writes;
|
|
1260
|
+
let iterations = 0;
|
|
1261
|
+
while (
|
|
1262
|
+
writes?.authority &&
|
|
1263
|
+
this.live &&
|
|
1264
|
+
!this.moved &&
|
|
1265
|
+
!this.closed &&
|
|
1266
|
+
this.room !== null &&
|
|
1267
|
+
(this.room.dirtyLen() > 0 || this.room.pendingLen() > 0)
|
|
1268
|
+
) {
|
|
1269
|
+
if (++iterations > DRAIN_MAX_ITERATIONS) {
|
|
1270
|
+
throw new Error(
|
|
1271
|
+
`drain: room ${this.opts.downstream.docId} failed to quiesce after ${DRAIN_MAX_ITERATIONS} flushes ` +
|
|
1272
|
+
`(dirty=${this.room.dirtyLen()}, pending=${this.room.pendingLen()})`,
|
|
1273
|
+
);
|
|
1274
|
+
}
|
|
1275
|
+
await this.flushNow();
|
|
1276
|
+
}
|
|
1277
|
+
return this.lastCommittedFlushSeq;
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
/** §5.3: build the batch (synchronously — the build IS the snapshot), journal its
|
|
1281
|
+
* exact body bytes, then settle it against the authority. One in flight; the
|
|
1282
|
+
* settlement outcome drives the state machine (§5.4). */
|
|
1283
|
+
private fireFlush(): void {
|
|
1284
|
+
const writes = this.opts.downstream.writes;
|
|
1285
|
+
const authority = writes?.authority;
|
|
1286
|
+
if (!authority || !this.journal || this.flushBusy) return;
|
|
1287
|
+
if (this.closed || this.moved || !this.live || !this.room) return;
|
|
1288
|
+
const room = this.room;
|
|
1289
|
+
let out: string | undefined;
|
|
1290
|
+
try {
|
|
1291
|
+
out = room.beginFlush();
|
|
1292
|
+
} catch (e) {
|
|
1293
|
+
this.violation(`beginFlush failed: ${String(e)}`);
|
|
1294
|
+
return;
|
|
1295
|
+
}
|
|
1296
|
+
if (!out) return; // nothing dirty, nothing unflushed
|
|
1297
|
+
const { changes, batchHash } = JSON.parse(out) as {
|
|
1298
|
+
changes: unknown[];
|
|
1299
|
+
batchHash: string;
|
|
1300
|
+
};
|
|
1301
|
+
const seq = this.flushSeq++;
|
|
1302
|
+
// Composed ONCE; journaled and sent as this exact string forever (§5.3 step 4).
|
|
1303
|
+
const body = JSON.stringify({
|
|
1304
|
+
source: `room:${this.opts.downstream.docId}:${this.placementEpoch}`,
|
|
1305
|
+
offset: padOffset(seq),
|
|
1306
|
+
doc: this.opts.downstream.docId,
|
|
1307
|
+
epoch: this.placementEpoch,
|
|
1308
|
+
batchHash,
|
|
1309
|
+
cas: true,
|
|
1310
|
+
changes,
|
|
1311
|
+
});
|
|
1312
|
+
const journal = this.journal;
|
|
1313
|
+
const bootAtFlush = this.incarnationBootId;
|
|
1314
|
+
this.flushBusy = true;
|
|
1315
|
+
this.flushChain = (async () => {
|
|
1316
|
+
try {
|
|
1317
|
+
await journal.appendFlush({ seq, epoch: this.placementEpoch, body });
|
|
1318
|
+
} catch (e) {
|
|
1319
|
+
// An unjournaled batch must never reach the wire: a retry could rebuild
|
|
1320
|
+
// different bytes under the same id (§8.3). Loud incarnation death.
|
|
1321
|
+
this.flushBusy = false;
|
|
1322
|
+
if (this.incarnationBootId === bootAtFlush && !this.closed) {
|
|
1323
|
+
this.violation(`flush journal append failed: ${String(e)}`);
|
|
1324
|
+
}
|
|
1325
|
+
return;
|
|
1326
|
+
}
|
|
1327
|
+
let res: Awaited<ReturnType<RoomAuthority["applyRowChangeTxn"]>>;
|
|
1328
|
+
try {
|
|
1329
|
+
res = await withNetRetry(
|
|
1330
|
+
() => authority.applyRowChangeTxn(body),
|
|
1331
|
+
() => this.closed || this.moved,
|
|
1332
|
+
this.log,
|
|
1333
|
+
);
|
|
1334
|
+
} catch (e) {
|
|
1335
|
+
this.flushBusy = false;
|
|
1336
|
+
if (this.closed || this.moved) return;
|
|
1337
|
+
// Fatal-class apply error (identity mismatch): our bug, never retried.
|
|
1338
|
+
if (this.incarnationBootId === bootAtFlush) {
|
|
1339
|
+
this.violation(`flush apply failed fatally: ${String(e)}`);
|
|
1340
|
+
}
|
|
1341
|
+
return;
|
|
1342
|
+
}
|
|
1343
|
+
this.flushBusy = false;
|
|
1344
|
+
const sameIncarnation = this.incarnationBootId === bootAtFlush && this.room !== null;
|
|
1345
|
+
if (res.kind === "ok") {
|
|
1346
|
+
this.flushesConfirmed += 1;
|
|
1347
|
+
this.lastCommittedFlushSeq = Math.max(this.lastCommittedFlushSeq, seq); // §4.2 fence input
|
|
1348
|
+
await journal.confirmFlush(seq);
|
|
1349
|
+
if (!sameIncarnation) {
|
|
1350
|
+
// Settled; the new incarnation replayed its own state — let it flush.
|
|
1351
|
+
this.scheduleFlush();
|
|
1352
|
+
return;
|
|
1353
|
+
}
|
|
1354
|
+
try {
|
|
1355
|
+
this.room!.flushOk();
|
|
1356
|
+
} catch (e) {
|
|
1357
|
+
this.violation(`flushOk failed: ${String(e)}`);
|
|
1358
|
+
return;
|
|
1359
|
+
}
|
|
1360
|
+
this.scheduleFlush(); // in-flight re-dirties / new lmids
|
|
1361
|
+
} else if (res.kind === "conflict") {
|
|
1362
|
+
await journal.confirmFlush(seq); // nothing applied; the retry re-derives
|
|
1363
|
+
if (!sameIncarnation) {
|
|
1364
|
+
this.scheduleFlush();
|
|
1365
|
+
return;
|
|
1366
|
+
}
|
|
1367
|
+
try {
|
|
1368
|
+
const out = JSON.parse(this.room!.flushConflict(JSON.stringify(res.conflicts))) as {
|
|
1369
|
+
headCv: number | null;
|
|
1370
|
+
};
|
|
1371
|
+
if (out.headCv !== null) {
|
|
1372
|
+
this.drainDownstream(); // the corrective frames (§5.4: ordinary edits)
|
|
1373
|
+
}
|
|
1374
|
+
} catch (e) {
|
|
1375
|
+
this.violation(`flushConflict failed: ${String(e)}`);
|
|
1376
|
+
return;
|
|
1377
|
+
}
|
|
1378
|
+
this.log(`flush ${seq} CAS-conflicted: converged to the authority, retrying the rest`);
|
|
1379
|
+
this.scheduleFlush();
|
|
1380
|
+
} else {
|
|
1381
|
+
// Fenced under OUR epoch: this room is superseded (§2.5) — terminal.
|
|
1382
|
+
await journal.confirmFlush(seq);
|
|
1383
|
+
this.roomMoved(`flush ${seq} fenced (authority epoch ${res.currentEpoch ?? "?"})`);
|
|
1384
|
+
}
|
|
1385
|
+
})();
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
/** §2.5 stale-room behavior: the authority fenced us — another placement owns the
|
|
1389
|
+
* doc. Final state discarded, downstream closed with `room_moved`, nothing
|
|
1390
|
+
* reconnects. Clients re-open through the API server onto the current epoch. */
|
|
1391
|
+
private roomMoved(reason: string): void {
|
|
1392
|
+
if (this.moved) return;
|
|
1393
|
+
this.moved = true;
|
|
1394
|
+
this.log(`room moved: ${reason}`);
|
|
1395
|
+
for (const meta of this.subs.values()) {
|
|
1396
|
+
meta.ws.close(4009, "room_moved");
|
|
1397
|
+
}
|
|
1398
|
+
void this.close();
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
// ----------------------------- downstream leg -----------------------------
|
|
1402
|
+
|
|
1403
|
+
private serveDownstream(ws: WebSocket): void {
|
|
1404
|
+
const conn: ConnState = {
|
|
1405
|
+
id: this.nextConnId++,
|
|
1406
|
+
ws,
|
|
1407
|
+
queries: new Map(),
|
|
1408
|
+
busy: Promise.resolve(),
|
|
1409
|
+
clientID: null,
|
|
1410
|
+
sub: null,
|
|
1411
|
+
};
|
|
1412
|
+
this.conns.add(conn);
|
|
1413
|
+
ws.on("message", (data) => {
|
|
1414
|
+
let msg: Record<string, unknown>;
|
|
1415
|
+
try {
|
|
1416
|
+
msg = JSON.parse(String(data)) as Record<string, unknown>;
|
|
1417
|
+
} catch {
|
|
1418
|
+
return;
|
|
1419
|
+
}
|
|
1420
|
+
// Serialize per connection: subscribe verification is async, and a client's
|
|
1421
|
+
// subscribe → unsubscribe order must hold. A per-message throw is isolated to
|
|
1422
|
+
// THIS connection (the reference server's #12).
|
|
1423
|
+
conn.busy = conn.busy.then(async () => {
|
|
1424
|
+
try {
|
|
1425
|
+
await this.handleDownstreamMsg(ws, conn, msg);
|
|
1426
|
+
} catch (err) {
|
|
1427
|
+
send(ws, {
|
|
1428
|
+
t: "error",
|
|
1429
|
+
queryId: msg.queryId,
|
|
1430
|
+
message: String((err as Error)?.message ?? err),
|
|
1431
|
+
});
|
|
1432
|
+
}
|
|
1433
|
+
});
|
|
1434
|
+
});
|
|
1435
|
+
ws.on("close", () => {
|
|
1436
|
+
const now = Date.now();
|
|
1437
|
+
for (const q of conn.queries.values()) {
|
|
1438
|
+
if (this.subs.get(q.subKey)?.ws === ws) {
|
|
1439
|
+
this.subs.delete(q.subKey);
|
|
1440
|
+
this.room?.unsubscribe(q.subKey, now);
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
conn.queries.clear();
|
|
1444
|
+
this.conns.delete(conn);
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
private async handleDownstreamMsg(
|
|
1449
|
+
ws: WebSocket,
|
|
1450
|
+
conn: ConnState,
|
|
1451
|
+
msg: Record<string, unknown>,
|
|
1452
|
+
): Promise<void> {
|
|
1453
|
+
switch (msg.t) {
|
|
1454
|
+
case "init": {
|
|
1455
|
+
// The connection identity: what the lmid subscribe and pushMutation key on.
|
|
1456
|
+
// No reply frame — exactly rindled's wire.
|
|
1457
|
+
if (typeof msg.clientID === "string" && msg.clientID.length > 0) {
|
|
1458
|
+
conn.clientID = msg.clientID;
|
|
1459
|
+
}
|
|
1460
|
+
break;
|
|
1461
|
+
}
|
|
1462
|
+
case "subscribe": {
|
|
1463
|
+
if (typeof msg.queryId !== "number") return;
|
|
1464
|
+
if (!this.live || !this.room) {
|
|
1465
|
+
// Not live yet: hold the subscribe until the seq-0 snapshot lands, so a
|
|
1466
|
+
// booting room doesn't refuse its first clients.
|
|
1467
|
+
this.pendingSubs.push({ ws, conn, msg: msg as unknown as SubscribeMsg });
|
|
1468
|
+
return;
|
|
1469
|
+
}
|
|
1470
|
+
await this.handleSubscribe(ws, conn, msg as unknown as SubscribeMsg);
|
|
1471
|
+
break;
|
|
1472
|
+
}
|
|
1473
|
+
case "unsubscribe": {
|
|
1474
|
+
if (typeof msg.queryId !== "number") return;
|
|
1475
|
+
const prev = conn.queries.get(msg.queryId);
|
|
1476
|
+
if (prev) {
|
|
1477
|
+
conn.queries.delete(msg.queryId);
|
|
1478
|
+
this.subs.delete(prev.subKey);
|
|
1479
|
+
this.room?.unsubscribe(prev.subKey, Date.now());
|
|
1480
|
+
}
|
|
1481
|
+
break;
|
|
1482
|
+
}
|
|
1483
|
+
case "pushMutation": {
|
|
1484
|
+
const e = msg.envelope as Partial<PushEnvelope> | undefined;
|
|
1485
|
+
if (
|
|
1486
|
+
e === null ||
|
|
1487
|
+
typeof e !== "object" ||
|
|
1488
|
+
typeof e.clientID !== "string" ||
|
|
1489
|
+
typeof e.mid !== "number" ||
|
|
1490
|
+
!Number.isFinite(e.mid) ||
|
|
1491
|
+
typeof e.name !== "string"
|
|
1492
|
+
) {
|
|
1493
|
+
send(ws, { t: "error", message: "malformed pushMutation envelope" });
|
|
1494
|
+
break;
|
|
1495
|
+
}
|
|
1496
|
+
this.handlePushMutation(ws, conn, {
|
|
1497
|
+
clientID: e.clientID,
|
|
1498
|
+
mid: e.mid,
|
|
1499
|
+
name: e.name,
|
|
1500
|
+
args: e.args,
|
|
1501
|
+
});
|
|
1502
|
+
break;
|
|
1503
|
+
}
|
|
1504
|
+
case "mutate":
|
|
1505
|
+
// Raw CRUD never crosses this wire: room writes are named mutators (§4.2),
|
|
1506
|
+
// validated against the owned set at the transaction boundary.
|
|
1507
|
+
send(ws, {
|
|
1508
|
+
t: "error",
|
|
1509
|
+
queryId: msg.queryId,
|
|
1510
|
+
message: "this room accepts named mutators only (pushMutation)",
|
|
1511
|
+
});
|
|
1512
|
+
break;
|
|
1513
|
+
default:
|
|
1514
|
+
break;
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
private async handleSubscribe(
|
|
1519
|
+
ws: WebSocket,
|
|
1520
|
+
conn: ConnState,
|
|
1521
|
+
msg: SubscribeMsg,
|
|
1522
|
+
): Promise<void> {
|
|
1523
|
+
if (!this.room) return; // raced an incarnation death; the socket is being closed
|
|
1524
|
+
const queryError = (message: string) =>
|
|
1525
|
+
send(ws, { t: "queryError", queryId: msg.queryId, message });
|
|
1526
|
+
|
|
1527
|
+
// The reserved lmid system query arrives BY NAME even in lease mode — the write
|
|
1528
|
+
// plane's confirmation stream. The room composes the AST itself from the
|
|
1529
|
+
// connection's `init` identity (client args are ignored, exactly as on rindled).
|
|
1530
|
+
if (typeof msg.name === "string") {
|
|
1531
|
+
if (msg.name !== LMID_QUERY_NAME) {
|
|
1532
|
+
queryError("subscribe requires a lease token");
|
|
1533
|
+
return;
|
|
1534
|
+
}
|
|
1535
|
+
if (!this.opts.downstream.writes) {
|
|
1536
|
+
queryError("this room is read-only (no write plane configured)");
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
if (!conn.clientID) {
|
|
1540
|
+
queryError("subscribe lmid query before init");
|
|
1541
|
+
return;
|
|
1542
|
+
}
|
|
1543
|
+
const prev = conn.queries.get(msg.queryId);
|
|
1544
|
+
const epoch = prev ? prev.epoch + 1 : 1;
|
|
1545
|
+
const subKey = `${conn.id}:${msg.queryId}`;
|
|
1546
|
+
if (prev) this.subs.delete(prev.subKey);
|
|
1547
|
+
let res: { hello: unknown; snapshot: unknown };
|
|
1548
|
+
try {
|
|
1549
|
+
res = JSON.parse(
|
|
1550
|
+
this.room.lmidSubscribe(subKey, epoch, conn.clientID, Date.now()),
|
|
1551
|
+
) as typeof res;
|
|
1552
|
+
} catch (e) {
|
|
1553
|
+
queryError(`materialize failed: ${String((e as Error)?.message ?? e)}`);
|
|
1554
|
+
return;
|
|
1555
|
+
}
|
|
1556
|
+
conn.queries.set(msg.queryId, { subKey, epoch });
|
|
1557
|
+
this.subs.set(subKey, {
|
|
1558
|
+
ws,
|
|
1559
|
+
conn,
|
|
1560
|
+
clientQid: msg.queryId,
|
|
1561
|
+
// Not lease-gated: it lives exactly as long as its connection — never
|
|
1562
|
+
// swept by exp, never a revocation key (revoking a user closes the socket).
|
|
1563
|
+
user: "",
|
|
1564
|
+
exp: Number.POSITIVE_INFINITY,
|
|
1565
|
+
});
|
|
1566
|
+
send(ws, {
|
|
1567
|
+
t: "nhello",
|
|
1568
|
+
queryId: msg.queryId,
|
|
1569
|
+
hello: res.hello,
|
|
1570
|
+
bootId: this.incarnationBootId,
|
|
1571
|
+
});
|
|
1572
|
+
send(ws, { t: "nbatch", queryId: msg.queryId, batch: res.snapshot });
|
|
1573
|
+
this.sendProgress(conn);
|
|
1574
|
+
return;
|
|
1575
|
+
}
|
|
1576
|
+
if (typeof msg.leaseToken !== "string") {
|
|
1577
|
+
queryError("subscribe requires a lease token");
|
|
1578
|
+
return;
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
// The §10.1 gate: signature, doc, expiry — then the §4.1 revocation check.
|
|
1582
|
+
const now = Date.now();
|
|
1583
|
+
let payload;
|
|
1584
|
+
try {
|
|
1585
|
+
payload = await verifyRoomToken(msg.leaseToken, {
|
|
1586
|
+
doc: this.opts.downstream.docId,
|
|
1587
|
+
keys: this.opts.downstream.tokenKeys,
|
|
1588
|
+
now,
|
|
1589
|
+
});
|
|
1590
|
+
} catch (e) {
|
|
1591
|
+
queryError(e instanceof RoomTokenError ? e.message : "lease token refused");
|
|
1592
|
+
return;
|
|
1593
|
+
}
|
|
1594
|
+
const revokedAt = this.revoked.get(payload.sub);
|
|
1595
|
+
if (revokedAt !== undefined && payload.iat <= revokedAt) {
|
|
1596
|
+
queryError("lease token refused: revoked");
|
|
1597
|
+
return;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
// Scope-skew tripwire: a lease proving against scopes this room's gate did NOT arm
|
|
1601
|
+
// with means a profile was edited after this room booted (the gate arms once). The
|
|
1602
|
+
// subscribe is NOT refused — the gate is still sound, it will just deopt routed
|
|
1603
|
+
// writes until the room re-boots. Log once per distinct skew so the otherwise-silent
|
|
1604
|
+
// deopt loop is diagnosable. Both hashes present required (a pre-stamp token or v1
|
|
1605
|
+
// gate skips).
|
|
1606
|
+
if (
|
|
1607
|
+
this.armedScopesHash !== undefined &&
|
|
1608
|
+
payload.scopesHash !== undefined &&
|
|
1609
|
+
payload.scopesHash !== this.armedScopesHash
|
|
1610
|
+
) {
|
|
1611
|
+
const key = `${payload.scopesHash}${this.armedScopesHash}`;
|
|
1612
|
+
if (!this.loggedScopeSkew.has(key)) {
|
|
1613
|
+
this.loggedScopeSkew.add(key);
|
|
1614
|
+
this.log(
|
|
1615
|
+
`scope skew: this room's gate armed with scopes ${this.armedScopesHash} but a lease ` +
|
|
1616
|
+
`proves against ${payload.scopesHash} — a room profile was edited under this live ` +
|
|
1617
|
+
`room. Routed writes will deopt (safe, but degraded) until the room re-boots.`,
|
|
1618
|
+
);
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
// Subject binding (managed-writes §3.1): the FIRST verified lease binds the
|
|
1623
|
+
// connection's authenticated subject; a later lease whose `sub` differs is refused
|
|
1624
|
+
// — one principal per connection, so "the connection's user" is well-defined and
|
|
1625
|
+
// privilege-mixing is closed.
|
|
1626
|
+
if (conn.sub === null) {
|
|
1627
|
+
conn.sub = payload.sub;
|
|
1628
|
+
} else if (conn.sub !== payload.sub) {
|
|
1629
|
+
queryError("lease token refused: connection is bound to another subject");
|
|
1630
|
+
return;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
// A re-subscribe (gap recovery) replaces the prior envelope and bumps the
|
|
1634
|
+
// DOWNSTREAM epoch — the same rule rindled applies per (conn, queryId).
|
|
1635
|
+
const prev = conn.queries.get(msg.queryId);
|
|
1636
|
+
const epoch = prev ? prev.epoch + 1 : 1;
|
|
1637
|
+
const subKey = `${conn.id}:${msg.queryId}`;
|
|
1638
|
+
if (prev) this.subs.delete(prev.subKey);
|
|
1639
|
+
|
|
1640
|
+
let res: { queryKey: string; reused: boolean; hello: unknown; snapshot: unknown };
|
|
1641
|
+
try {
|
|
1642
|
+
res = JSON.parse(
|
|
1643
|
+
this.room.subscribe(subKey, epoch, JSON.stringify(payload.ast), now),
|
|
1644
|
+
) as typeof res;
|
|
1645
|
+
} catch (e) {
|
|
1646
|
+
queryError(`materialize failed: ${String((e as Error)?.message ?? e)}`);
|
|
1647
|
+
return;
|
|
1648
|
+
}
|
|
1649
|
+
conn.queries.set(msg.queryId, { subKey, epoch });
|
|
1650
|
+
this.subs.set(subKey, {
|
|
1651
|
+
ws,
|
|
1652
|
+
conn,
|
|
1653
|
+
clientQid: msg.queryId,
|
|
1654
|
+
user: payload.sub,
|
|
1655
|
+
exp: payload.exp,
|
|
1656
|
+
});
|
|
1657
|
+
send(ws, {
|
|
1658
|
+
t: "nhello",
|
|
1659
|
+
queryId: msg.queryId,
|
|
1660
|
+
hello: res.hello,
|
|
1661
|
+
bootId: this.incarnationBootId,
|
|
1662
|
+
});
|
|
1663
|
+
send(ws, { t: "nbatch", queryId: msg.queryId, batch: res.snapshot });
|
|
1664
|
+
// The release point for the snapshot: an optimistic client buffers every nbatch
|
|
1665
|
+
// until a progress frame's cvMin covers its cv.
|
|
1666
|
+
this.sendProgress(conn);
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
/** The periodic enforcement tick: drop subscriptions whose lease passed `exp`
|
|
1670
|
+
* unrenewed (§4.1's TTL backstop), reclaim idle pipelines, prune old revocations. */
|
|
1671
|
+
private sweep(): void {
|
|
1672
|
+
const now = Date.now();
|
|
1673
|
+
for (const [subKey, meta] of [...this.subs]) {
|
|
1674
|
+
if (meta.exp <= now) {
|
|
1675
|
+
send(meta.ws, {
|
|
1676
|
+
t: "queryError",
|
|
1677
|
+
queryId: meta.clientQid,
|
|
1678
|
+
message: "expired lease — renew through the API server",
|
|
1679
|
+
});
|
|
1680
|
+
this.dropSub(subKey, meta, now);
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
if (this.room && this.live) {
|
|
1684
|
+
this.room.sweep(now);
|
|
1685
|
+
}
|
|
1686
|
+
const window = this.opts.downstream.revocationWindowMs ?? DEFAULT_REVOCATION_WINDOW_MS;
|
|
1687
|
+
for (const [user, at] of [...this.revoked]) {
|
|
1688
|
+
if (now - at > window) this.revoked.delete(user);
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
private dropSub(subKey: string, meta: SubMeta, now: number): void {
|
|
1693
|
+
this.subs.delete(subKey);
|
|
1694
|
+
meta.conn.queries.delete(meta.clientQid);
|
|
1695
|
+
this.room?.unsubscribe(subKey, now);
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
/** §4.1 layer 2: synchronous revocation. Terminates every subscription and socket of
|
|
1699
|
+
* `user` and refuses their pre-revocation tokens from here on. */
|
|
1700
|
+
private revokeUser(user: string): number {
|
|
1701
|
+
const now = Date.now();
|
|
1702
|
+
this.revoked.set(user, now);
|
|
1703
|
+
const sockets = new Set<WebSocket>();
|
|
1704
|
+
let dropped = 0;
|
|
1705
|
+
for (const [subKey, meta] of [...this.subs]) {
|
|
1706
|
+
if (meta.user !== user) continue;
|
|
1707
|
+
send(meta.ws, {
|
|
1708
|
+
t: "queryError",
|
|
1709
|
+
queryId: meta.clientQid,
|
|
1710
|
+
message: "lease revoked",
|
|
1711
|
+
});
|
|
1712
|
+
this.dropSub(subKey, meta, now);
|
|
1713
|
+
sockets.add(meta.ws);
|
|
1714
|
+
dropped++;
|
|
1715
|
+
}
|
|
1716
|
+
// A conn BOUND to the subject is write-capable even with no live subscription
|
|
1717
|
+
// (managed-writes §3.1/§3.4) — its socket goes too.
|
|
1718
|
+
for (const conn of this.conns) {
|
|
1719
|
+
if (conn.sub === user) sockets.add(conn.ws);
|
|
1720
|
+
}
|
|
1721
|
+
for (const ws of sockets) {
|
|
1722
|
+
ws.close(1008, "revoked");
|
|
1723
|
+
}
|
|
1724
|
+
this.log(`revoked ${user}: ${dropped} subscription(s)`);
|
|
1725
|
+
return dropped;
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
// ----------------------------- control plane ------------------------------
|
|
1729
|
+
|
|
1730
|
+
private startControl(control: { authToken: string; port?: number }): Promise<void> {
|
|
1731
|
+
const server = createServer((req, res) => {
|
|
1732
|
+
const reply = (status: number, body: unknown) => {
|
|
1733
|
+
res.writeHead(status, { "content-type": "application/json" });
|
|
1734
|
+
res.end(JSON.stringify(body));
|
|
1735
|
+
};
|
|
1736
|
+
if (req.headers.authorization !== `Bearer ${control.authToken}`) {
|
|
1737
|
+
return reply(401, { error: "unauthorized" });
|
|
1738
|
+
}
|
|
1739
|
+
if (req.method === "GET" && req.url === "/stats") {
|
|
1740
|
+
return reply(200, {
|
|
1741
|
+
live: this.live,
|
|
1742
|
+
connections: this.wss.clients.size,
|
|
1743
|
+
subscriptions: this.subs.size,
|
|
1744
|
+
materializations: this.room?.materializationCount() ?? 0,
|
|
1745
|
+
pendingMutations: this.room?.pendingLen() ?? 0,
|
|
1746
|
+
dirtyKeys: this.room?.dirtyLen() ?? 0,
|
|
1747
|
+
placementEpoch: this.placementEpoch,
|
|
1748
|
+
flushesConfirmed: this.flushesConfirmed,
|
|
1749
|
+
flushInFlight: this.flushBusy,
|
|
1750
|
+
});
|
|
1751
|
+
}
|
|
1752
|
+
if (req.method === "POST" && req.url === "/revoke") {
|
|
1753
|
+
let body = "";
|
|
1754
|
+
req.on("data", (c) => (body += c));
|
|
1755
|
+
req.on("end", () => {
|
|
1756
|
+
try {
|
|
1757
|
+
const { userId } = JSON.parse(body) as { userId?: string };
|
|
1758
|
+
if (typeof userId !== "string" || userId.length === 0) {
|
|
1759
|
+
return reply(400, { error: "userId required" });
|
|
1760
|
+
}
|
|
1761
|
+
reply(200, { revoked: this.revokeUser(userId) });
|
|
1762
|
+
} catch {
|
|
1763
|
+
reply(400, { error: "invalid JSON" });
|
|
1764
|
+
}
|
|
1765
|
+
});
|
|
1766
|
+
return;
|
|
1767
|
+
}
|
|
1768
|
+
if (req.method === "POST" && req.url === "/drain") {
|
|
1769
|
+
// §4.2 drain-before-downgrade: the api-server's `drainRoom` hook lands here. No body.
|
|
1770
|
+
this.drainForDowngrade().then(
|
|
1771
|
+
(finalFlushSeq) => reply(200, { finalFlushSeq, drained: true }),
|
|
1772
|
+
(e) => reply(500, { error: String((e as Error)?.message ?? e) }),
|
|
1773
|
+
);
|
|
1774
|
+
return;
|
|
1775
|
+
}
|
|
1776
|
+
reply(404, { error: "unknown endpoint" });
|
|
1777
|
+
});
|
|
1778
|
+
this.control = server;
|
|
1779
|
+
return new Promise((resolve) => {
|
|
1780
|
+
server.listen(control.port ?? 0, "127.0.0.1", () => resolve());
|
|
1781
|
+
});
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
interface SubscribeMsg {
|
|
1786
|
+
queryId: number;
|
|
1787
|
+
name?: unknown;
|
|
1788
|
+
leaseToken?: unknown;
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
/** The wire mutation envelope (`{t:"pushMutation", envelope}`) — `clientID` capital
|
|
1792
|
+
* ID, exactly as `@rindle/remote` sends it. */
|
|
1793
|
+
interface PushEnvelope {
|
|
1794
|
+
clientID: string;
|
|
1795
|
+
mid: number;
|
|
1796
|
+
name: string;
|
|
1797
|
+
args: unknown;
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
/** A flush seq as its wire offset: zero-padded so `rindled`'s lexicographic keyset
|
|
1801
|
+
* compare (`run_already_applied`) orders it like the number it is. */
|
|
1802
|
+
function padOffset(seq: number): string {
|
|
1803
|
+
return String(seq).padStart(20, "0");
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
/** Retry `op` on network-class failures (backoff, same inputs — for the flush that
|
|
1807
|
+
* means the same journaled bytes) until it settles, `stop()` says quit, or the error
|
|
1808
|
+
* is fatal (`{fatal: true}` — retrying cannot help). */
|
|
1809
|
+
async function withNetRetry<T>(
|
|
1810
|
+
op: () => Promise<T>,
|
|
1811
|
+
stop: () => boolean,
|
|
1812
|
+
log: (line: string) => void,
|
|
1813
|
+
): Promise<T> {
|
|
1814
|
+
let delay = FLUSH_RETRY_MIN_MS;
|
|
1815
|
+
for (;;) {
|
|
1816
|
+
try {
|
|
1817
|
+
return await op();
|
|
1818
|
+
} catch (e) {
|
|
1819
|
+
if ((e as { fatal?: boolean })?.fatal === true || stop()) throw e;
|
|
1820
|
+
log(`authority call failed (retrying in ${delay}ms): ${String(e)}`);
|
|
1821
|
+
await new Promise((r) => {
|
|
1822
|
+
const t = setTimeout(r, delay);
|
|
1823
|
+
(t as { unref?: () => void }).unref?.();
|
|
1824
|
+
});
|
|
1825
|
+
if (stop()) throw e;
|
|
1826
|
+
delay = Math.min(delay * 2, FLUSH_RETRY_MAX_MS);
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
/** Positional table shapes from the upstream hello — the keyed MutationTx layer's
|
|
1832
|
+
* schema. (The hello was already validated by `WasmRoom.open` before this runs.) */
|
|
1833
|
+
function shapesOf(hello: unknown): Map<string, TableShape> {
|
|
1834
|
+
const shapes = new Map<string, TableShape>();
|
|
1835
|
+
const tables = (
|
|
1836
|
+
hello as { tables?: Array<{ name: string; columns: string[]; primaryKey: number[] }> }
|
|
1837
|
+
)?.tables;
|
|
1838
|
+
if (Array.isArray(tables)) {
|
|
1839
|
+
for (const t of tables) {
|
|
1840
|
+
shapes.set(t.name, { columns: t.columns, primaryKey: t.primaryKey });
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
return shapes;
|
|
1844
|
+
}
|
|
1845
|
+
|
|
1846
|
+
/** Boot a room shell: init the wasm, mint the upstream lease, connect the upstream leg,
|
|
1847
|
+
* and serve the downstream ws (+ the private control plane, if configured). Returns
|
|
1848
|
+
* once the ports are bound and the upstream connection is underway — await
|
|
1849
|
+
* `shell.awaitLive()` for the seed. */
|
|
1850
|
+
export async function createRoomShell(opts: RoomShellOptions): Promise<RoomShell> {
|
|
1851
|
+
const shell = new Shell(opts);
|
|
1852
|
+
try {
|
|
1853
|
+
await shell.start();
|
|
1854
|
+
} catch (e) {
|
|
1855
|
+
await shell.close();
|
|
1856
|
+
throw e;
|
|
1857
|
+
}
|
|
1858
|
+
return shell;
|
|
1859
|
+
}
|