@rindle/room 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.sh +53 -0
- package/dist/authority.d.ts +57 -0
- package/dist/authority.d.ts.map +1 -0
- package/dist/authority.js +73 -0
- package/dist/authority.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +22 -0
- package/dist/index.js.map +1 -0
- package/dist/journal.d.ts +64 -0
- package/dist/journal.d.ts.map +1 -0
- package/dist/journal.js +53 -0
- package/dist/journal.js.map +1 -0
- package/dist/mutation-tx.d.ts +79 -0
- package/dist/mutation-tx.d.ts.map +1 -0
- package/dist/mutation-tx.js +207 -0
- package/dist/mutation-tx.js.map +1 -0
- package/dist/shell.d.ts +134 -0
- package/dist/shell.d.ts.map +1 -0
- package/dist/shell.js +1589 -0
- package/dist/shell.js.map +1 -0
- package/dist/token.d.ts +89 -0
- package/dist/token.d.ts.map +1 -0
- package/dist/token.js +158 -0
- package/dist/token.js.map +1 -0
- package/dist/wasm.d.ts +7 -0
- package/dist/wasm.d.ts.map +1 -0
- package/dist/wasm.js +29 -0
- package/dist/wasm.js.map +1 -0
- package/package.json +71 -0
- package/pkg/rindle_room.d.ts +345 -0
- package/pkg/rindle_room.js +924 -0
- package/pkg/rindle_room_bg.wasm +0 -0
- package/pkg/rindle_room_bg.wasm.d.ts +45 -0
- package/src/authority.ts +127 -0
- package/src/index.ts +49 -0
- package/src/journal.ts +110 -0
- package/src/mutation-tx.ts +275 -0
- package/src/shell.ts +1859 -0
- package/src/token.ts +219 -0
- package/src/wasm.ts +32 -0
package/dist/shell.js
ADDED
|
@@ -0,0 +1,1589 @@
|
|
|
1
|
+
// The Node test shell (RINDLE-REALTIME-DESIGN.md §11, P0/P1): a plain process running
|
|
2
|
+
// the SAME wasm build the Durable Object shell will run, wired to real sockets.
|
|
3
|
+
//
|
|
4
|
+
// rindled ──ws (normalized protocol)──▶ WasmRoom (base store) ──serving──▶ subscribers
|
|
5
|
+
// upstream leg (§3) downstream leg (§4)
|
|
6
|
+
//
|
|
7
|
+
// Both legs speak the one protocol. Upstream, the shell is a client of rindled's public
|
|
8
|
+
// ws plane: `init` → `subscribe {queryId, leaseToken}` → `nhello` → seq-0 `nbatch` →
|
|
9
|
+
// live tail; the lease is minted on rindled's private control plane (`POST /materialize`)
|
|
10
|
+
// — the room fetches its own footprint lease at boot, the same shape as the DO shell's
|
|
11
|
+
// boot callback (§10.1). Downstream, the shell serves the `@rindle/remote` wire verbatim,
|
|
12
|
+
// gated by **self-authorizing signed lease tokens** (§10.1): `subscribe {queryId,
|
|
13
|
+
// leaseToken}` verifies the token (signature, doc, expiry, revocation) and materializes
|
|
14
|
+
// the approved AST it carries on presentation — no `/materialize` control call, no named
|
|
15
|
+
// registry, ASTs never composed by clients. Identical queries share one pipeline
|
|
16
|
+
// (QueryKey dedup in the wasm room); each subscriber gets its own epoch/seq envelope.
|
|
17
|
+
//
|
|
18
|
+
// Lease lifecycle (§4.1): tokens are short-lived; the shell drops any subscription whose
|
|
19
|
+
// lease passes `exp` unrenewed (a `queryError`, then detach — renewal is
|
|
20
|
+
// re-authorization through the API server, which is a fresh subscribe with a fresh
|
|
21
|
+
// token). The private control plane's `POST /revoke {userId}` terminates a user's
|
|
22
|
+
// subscriptions and sockets immediately and refuses pre-revocation tokens (`iat` <
|
|
23
|
+
// revocation) — the synchronous layer; the TTL is the backstop.
|
|
24
|
+
//
|
|
25
|
+
// The write path (§5.1, P2 — no write-behind yet): a connection `init`s its clientID,
|
|
26
|
+
// then pushes `{t:"pushMutation", envelope:{clientID, mid, name, args}}` frames. Per
|
|
27
|
+
// mutation: dedup/gap-check → run the registered mutator against a staged wasm tx →
|
|
28
|
+
// commit to the shared head → fan the data `nbatch`es out IMMEDIATELY (step 2: fanout
|
|
29
|
+
// is never gated on durability) → append the envelope to the journal under a ≤~5ms
|
|
30
|
+
// group commit → on append, ACK: advance the author's `_rindle_client_mutations` row
|
|
31
|
+
// and fan the lmid frame (step 4: the ack a client observes survives a process crash
|
|
32
|
+
// by construction — §8.1). A failed/unknown mutator is a silent reject that still
|
|
33
|
+
// consumes the mid (the ledger advances with no effects; the author's prediction snaps
|
|
34
|
+
// back — `rindle-replica`'s contract). On boot, the journal replays by RE-INVOKING the
|
|
35
|
+
// mutators against the fresh base (§3.3), then acks everything replayed.
|
|
36
|
+
//
|
|
37
|
+
// Release protocol: every downstream `nbatch` is stamped with the room's `head_cv`,
|
|
38
|
+
// and after each drain the shell sends the connection-level `{t:"progress",
|
|
39
|
+
// frame:{cvMin}}` the optimistic client releases on. One §1.3.1-coherence rule: the
|
|
40
|
+
// AUTHOR's cvMin holds below its oldest un-acked mutation's apply-cv until the ack
|
|
41
|
+
// lands (data released before its lmid advance would make the client re-invoke a
|
|
42
|
+
// confirmed mutation on top of its own effect); everyone else releases at head_cv.
|
|
43
|
+
//
|
|
44
|
+
// Backpressure (§9, T8): each downstream socket has a bounded send budget. A stalled
|
|
45
|
+
// client that overflows it is TERMINATED with a gap — the normalized protocol's
|
|
46
|
+
// re-subscribe repair — so one slow client never holds frames in room memory.
|
|
47
|
+
//
|
|
48
|
+
// Failure posture (§3.4): the room is an INCARNATION. Any upstream violation — a frame
|
|
49
|
+
// that fails to decode, a seq gap, a poisoned apply, an upstream `queryError`, the
|
|
50
|
+
// socket dropping — kills the incarnation: the wasm store is freed, every downstream
|
|
51
|
+
// socket is closed (subscriptions die with the incarnation; reconnecting clients
|
|
52
|
+
// re-subscribe against the next one), and the shell re-subscribes upstream. On the same
|
|
53
|
+
// socket that is a `subscribe` re-send (rindled bumps the epoch and re-snapshots); on a
|
|
54
|
+
// fresh socket it is a new subscription. There is no patch-up path, by design.
|
|
55
|
+
import { createServer } from "node:http";
|
|
56
|
+
import { randomUUID } from "node:crypto";
|
|
57
|
+
import { WebSocket, WebSocketServer } from "ws";
|
|
58
|
+
import { initRoomWasm, WasmRoom } from "./wasm.js";
|
|
59
|
+
import { verifyRoomToken, RoomTokenError, scopeSpecsHash } from "./token.js";
|
|
60
|
+
import { assertSyncMutatorReturn, isEnvironmentShortfall, mutationTx, } from "./mutation-tx.js";
|
|
61
|
+
import { journalEntryOutcome, memoryJournal, } from "./journal.js";
|
|
62
|
+
const UPSTREAM_QID = 1;
|
|
63
|
+
/** Delay before re-subscribing after a violation — keeps a persistent violation from
|
|
64
|
+
* becoming a hot loop while staying far below human-perceptible recovery time. */
|
|
65
|
+
const RESUBSCRIBE_DELAY_MS = 250;
|
|
66
|
+
const RECONNECT_MAX_MS = 5_000;
|
|
67
|
+
const DEFAULT_IDLE_TTL_MS = 30_000;
|
|
68
|
+
const DEFAULT_SWEEP_INTERVAL_MS = 1_000;
|
|
69
|
+
const DEFAULT_REVOCATION_WINDOW_MS = 30 * 60_000;
|
|
70
|
+
const DEFAULT_GROUP_COMMIT_MS = 5;
|
|
71
|
+
const DEFAULT_SEND_BUDGET_BYTES = 4 * 1024 * 1024;
|
|
72
|
+
const DEFAULT_FLUSH_DEBOUNCE_MS = 250;
|
|
73
|
+
const DEFAULT_FLUSH_DIRTY_MAX = 512;
|
|
74
|
+
/** §4.2 drain-before-downgrade iteration cap: a healthy room quiesces in a handful of flushes; a
|
|
75
|
+
* runaway (a straggler push per flush, or a wedged CAS loop) fails LOUD rather than spinning. */
|
|
76
|
+
const DRAIN_MAX_ITERATIONS = 100;
|
|
77
|
+
/** Flush-retry backoff bounds (network-class failures only; same journaled bytes). */
|
|
78
|
+
const FLUSH_RETRY_MIN_MS = 100;
|
|
79
|
+
const FLUSH_RETRY_MAX_MS = 2_000;
|
|
80
|
+
/** The reserved lmid system query — subscribed BY NAME even in lease mode; identity
|
|
81
|
+
* comes from the connection's `init`, never from args. */
|
|
82
|
+
const LMID_QUERY_NAME = "_rindle/clientLmid";
|
|
83
|
+
/** Mint the room's upstream footprint lease on rindled's control plane (§4: the room
|
|
84
|
+
* fetches its own lease — on the DO shell this is the boot callback's job). */
|
|
85
|
+
async function mintLease(up) {
|
|
86
|
+
const res = await fetch(new URL("/materialize", up.controlUrl), {
|
|
87
|
+
method: "POST",
|
|
88
|
+
headers: {
|
|
89
|
+
"content-type": "application/json",
|
|
90
|
+
...(up.authToken ? { authorization: `Bearer ${up.authToken}` } : {}),
|
|
91
|
+
},
|
|
92
|
+
body: JSON.stringify({
|
|
93
|
+
ast: up.footprintAst,
|
|
94
|
+
...(up.leaseTtlMs !== undefined ? { leaseTtlMs: up.leaseTtlMs } : {}),
|
|
95
|
+
}),
|
|
96
|
+
});
|
|
97
|
+
if (!res.ok) {
|
|
98
|
+
throw new Error(`upstream /materialize failed: ${res.status} ${await res.text()}`);
|
|
99
|
+
}
|
|
100
|
+
const out = (await res.json());
|
|
101
|
+
if (typeof out.leaseToken !== "string") {
|
|
102
|
+
throw new Error("upstream /materialize returned no leaseToken");
|
|
103
|
+
}
|
|
104
|
+
return out.leaseToken;
|
|
105
|
+
}
|
|
106
|
+
function send(ws, frame) {
|
|
107
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
108
|
+
ws.send(JSON.stringify(frame));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/** Per-client retention cap for {@link RecordedOutcome}s. The map only holds
|
|
112
|
+
* NON-APPLIED mids, and a client only re-sends a mid while its ledger lmid trails it
|
|
113
|
+
* — a contiguously-advancing window bounded by the client's in-flight backlog, far
|
|
114
|
+
* below this cap. Past it the oldest records evict (insertion order): a re-send of an
|
|
115
|
+
* evicted mid degrades to today's silence — the mid still dedups and the ledger still
|
|
116
|
+
* covers it, only the outcome re-answer is lost. */
|
|
117
|
+
const MAX_RECORDED_OUTCOMES_PER_CLIENT = 512;
|
|
118
|
+
class Shell {
|
|
119
|
+
opts;
|
|
120
|
+
log;
|
|
121
|
+
wss;
|
|
122
|
+
control = null;
|
|
123
|
+
room = null;
|
|
124
|
+
live = false;
|
|
125
|
+
incarnationBootId = randomUUID();
|
|
126
|
+
liveWaiters = [];
|
|
127
|
+
upstream = null;
|
|
128
|
+
leaseToken = "";
|
|
129
|
+
subscribeInFlight = false;
|
|
130
|
+
reconnectDelayMs = RESUBSCRIBE_DELAY_MS;
|
|
131
|
+
closed = false;
|
|
132
|
+
nextConnId = 1;
|
|
133
|
+
sweepTimer = null;
|
|
134
|
+
/** subKey (what the wasm room routes by) → where its frames go + lease state. */
|
|
135
|
+
subs = new Map();
|
|
136
|
+
/** Every open downstream connection — revocation must reach a BOUND conn even when
|
|
137
|
+
* it holds no live subscription (it is still write-capable). */
|
|
138
|
+
conns = new Set();
|
|
139
|
+
/** userId → when they were revoked (refuses tokens with `iat` ≤ this; pruned after
|
|
140
|
+
* the revocation window). */
|
|
141
|
+
revoked = new Map();
|
|
142
|
+
/** Downstream subscribes queued while no incarnation is live. */
|
|
143
|
+
pendingSubs = [];
|
|
144
|
+
// ------------------------------ write-plane state ------------------------------
|
|
145
|
+
/** The {@link scopeSpecsHash} of the boot-wire scopes THIS shell armed its §3.3 gate
|
|
146
|
+
* with — `undefined` in v1 (ungated) mode. Compared against each lease token's
|
|
147
|
+
* `scopesHash` to flag scope skew (a profile edited under this live room), which
|
|
148
|
+
* otherwise manifests only as an undiagnosable deopt loop. */
|
|
149
|
+
armedScopesHash;
|
|
150
|
+
/** Scope-skew hash pairs already logged (`lease→armed`), so the diagnostic fires ONCE
|
|
151
|
+
* per distinct skew, not once per subscribe. */
|
|
152
|
+
loggedScopeSkew = new Set();
|
|
153
|
+
/** The journal (write plane only). One per shell — it outlives incarnations; that
|
|
154
|
+
* is the point (§3.3: pending is replayed from it on every re-subscribe). */
|
|
155
|
+
journal = null;
|
|
156
|
+
/** Positional table shapes from the upstream hello (keyed layer of MutationTx). */
|
|
157
|
+
tableShapes = new Map();
|
|
158
|
+
/** Mutations queued while no incarnation is live (drained after pendingSubs). */
|
|
159
|
+
pendingMutes = [];
|
|
160
|
+
/** clientID → its applied-but-unacked mutations, oldest first (release holdback). */
|
|
161
|
+
unacked = new Map();
|
|
162
|
+
/** clientID → its recorded NON-APPLIED outcomes (mid → verdict), insertion-ordered
|
|
163
|
+
* and capped per client ({@link MAX_RECORDED_OUTCOMES_PER_CLIENT}). Seeded from the
|
|
164
|
+
* journal replay each incarnation (`finishBoot` — so it reflects what THIS
|
|
165
|
+
* incarnation's replay produced), appended live, and cleared with the incarnation. */
|
|
166
|
+
outcomes = new Map();
|
|
167
|
+
/** The group-commit window: entries awaiting the next journal append. */
|
|
168
|
+
ackQueue = [];
|
|
169
|
+
ackTimer = null;
|
|
170
|
+
/** Serializes journal appends (one in flight; acks apply in append order). */
|
|
171
|
+
ackChain = Promise.resolve();
|
|
172
|
+
// ------------------------------ flush state (§5.3) ------------------------------
|
|
173
|
+
/** The placement epoch (§2.5), claimed once per shell process at start. 0 = no
|
|
174
|
+
* authority configured. */
|
|
175
|
+
placementEpoch = 0;
|
|
176
|
+
/** The next flush-stream seq — also the (zero-padded) wire offset. Seeded from the
|
|
177
|
+
* journal so offset strings stay monotone across restarts sharing one journal. */
|
|
178
|
+
flushSeq = 1;
|
|
179
|
+
flushTimer = null;
|
|
180
|
+
/** One flush settlement in flight at a time (the wasm room guards too). */
|
|
181
|
+
flushBusy = false;
|
|
182
|
+
/** The settlement chain `flushNow()` awaits. */
|
|
183
|
+
flushChain = Promise.resolve();
|
|
184
|
+
flushesConfirmed = 0;
|
|
185
|
+
/** The last flush seq that COMMITTED at the authority (§4.2/§5.4 `flush_ok`): the value the
|
|
186
|
+
* `/drain` control reports as `finalFlushSeq`, the fence a downgraded client's ghost waits on
|
|
187
|
+
* (`_rindle_room_watermark(doc) ≥ finalFlushSeq`, and the watermark row's `flush_seq` IS this
|
|
188
|
+
* offset — consumer.rs). Seeded from the journal at boot (a re-booted room that already flushed
|
|
189
|
+
* reports its journaled max, not 0); 0 for a never-flushed room. */
|
|
190
|
+
lastCommittedFlushSeq = 0;
|
|
191
|
+
/** Fenced at the authority (§2.5): this room is superseded — terminal. */
|
|
192
|
+
moved = false;
|
|
193
|
+
// --------------------- §301 upstream-absorption advert (301-ECHO-FENCE §1.2) ---------------------
|
|
194
|
+
/** The daemon boot id the upstream connection serves (each upstream `nhello` stamps it) —
|
|
195
|
+
* rides the downstream progress frames as `upstreamBoot`, the §2.4 boot-rule input. */
|
|
196
|
+
upstreamBoot;
|
|
197
|
+
/** The daemon's released `cvMin` this room has provably absorbed through — recorded from the
|
|
198
|
+
* upstream `progress` frames (same-socket ordering: every nbatch at-or-below it was already
|
|
199
|
+
* applied when the frame is read), NOT from the applied head cv (a sparse footprint would
|
|
200
|
+
* starve the advert while the daemon advances). Rides downstream as `upstreamCv`;
|
|
201
|
+
* `undefined` until the first upstream progress of the current daemon boot. */
|
|
202
|
+
upstreamCvMin;
|
|
203
|
+
constructor(opts) {
|
|
204
|
+
this.opts = opts;
|
|
205
|
+
this.log = opts.log ?? (() => { });
|
|
206
|
+
if (opts.downstream.writes) {
|
|
207
|
+
if (opts.downstream.writes.ownedTables.length === 0) {
|
|
208
|
+
throw new Error("writes.ownedTables must name at least one table");
|
|
209
|
+
}
|
|
210
|
+
// H-iv-b: the boot-wire scopes may only ever NARROW the host's owned set, never
|
|
211
|
+
// extend it — a wider server scope would let mutators write tables this
|
|
212
|
+
// deployment never declared writable (self-hoster semantics preserved). Loud at
|
|
213
|
+
// construction, exactly like the empty-owned check above.
|
|
214
|
+
const scopes = opts.downstream.writes.scopes;
|
|
215
|
+
if (scopes !== undefined) {
|
|
216
|
+
const owned = new Set(opts.downstream.writes.ownedTables);
|
|
217
|
+
const rogue = scopes
|
|
218
|
+
.filter((s) => s.writable.kind !== "none" && !owned.has(s.table))
|
|
219
|
+
.map((s) => s.table);
|
|
220
|
+
if (rogue.length > 0) {
|
|
221
|
+
throw new Error(`writes.scopes marks ${rogue.map((t) => `\`${t}\``).join(", ")} writable, but the ` +
|
|
222
|
+
`host's writes.ownedTables does not include ${rogue.length === 1 ? "it" : "them"} — ` +
|
|
223
|
+
`the boot-wire scopes may only ever narrow the host's owned set, never extend it`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
// The armed-scope fingerprint: what the gate enforces, hashed once, for the
|
|
227
|
+
// skew check on the token path. `undefined` scopes = v1 ungated → no check.
|
|
228
|
+
if (scopes !== undefined)
|
|
229
|
+
this.armedScopesHash = scopeSpecsHash(scopes);
|
|
230
|
+
this.journal = opts.downstream.writes.journal ?? memoryJournal();
|
|
231
|
+
}
|
|
232
|
+
this.wss = new WebSocketServer({ port: opts.port ?? 0, host: "127.0.0.1" });
|
|
233
|
+
this.wss.on("connection", (ws) => this.serveDownstream(ws));
|
|
234
|
+
}
|
|
235
|
+
async start() {
|
|
236
|
+
await initRoomWasm();
|
|
237
|
+
await new Promise((resolve) => {
|
|
238
|
+
if (this.wss.address() !== null)
|
|
239
|
+
return resolve();
|
|
240
|
+
this.wss.on("listening", () => resolve());
|
|
241
|
+
});
|
|
242
|
+
if (this.opts.downstream.control) {
|
|
243
|
+
await this.startControl(this.opts.downstream.control);
|
|
244
|
+
}
|
|
245
|
+
const interval = this.opts.downstream.sweepIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
246
|
+
this.sweepTimer = setInterval(() => this.sweep(), interval);
|
|
247
|
+
this.sweepTimer.unref?.();
|
|
248
|
+
const authority = this.opts.downstream.writes?.authority;
|
|
249
|
+
if (authority && this.journal) {
|
|
250
|
+
// Seed the flush stream past everything this journal ever numbered, then claim
|
|
251
|
+
// the placement epoch (§2.5) — once per process, BEFORE anything else touches
|
|
252
|
+
// the authority: from this moment every stale body a dead predecessor left
|
|
253
|
+
// mid-network is fenced, which is what closes the probe race for good.
|
|
254
|
+
// Unconfirmed batches settle per incarnation in finishBoot — a prior process's
|
|
255
|
+
// records are fenced there (routine: their mutations recover by envelope
|
|
256
|
+
// replay under OUR epoch), our own land or dedup.
|
|
257
|
+
const { maxSeq } = await this.journal.replayFlushes();
|
|
258
|
+
this.flushSeq = maxSeq + 1;
|
|
259
|
+
this.lastCommittedFlushSeq = maxSeq; // journal-seeded (self-corrects upward on each flush_ok)
|
|
260
|
+
this.placementEpoch = await authority.claimEpoch(this.opts.downstream.docId);
|
|
261
|
+
this.log(`placement epoch ${this.placementEpoch}`);
|
|
262
|
+
}
|
|
263
|
+
this.leaseToken = await mintLease(this.opts.upstream);
|
|
264
|
+
this.connectUpstream();
|
|
265
|
+
}
|
|
266
|
+
get port() {
|
|
267
|
+
const addr = this.wss.address();
|
|
268
|
+
return typeof addr === "object" && addr !== null ? addr.port : 0;
|
|
269
|
+
}
|
|
270
|
+
get controlPort() {
|
|
271
|
+
const addr = this.control?.address();
|
|
272
|
+
return typeof addr === "object" && addr !== null ? addr.port : 0;
|
|
273
|
+
}
|
|
274
|
+
awaitLive() {
|
|
275
|
+
if (this.live)
|
|
276
|
+
return Promise.resolve();
|
|
277
|
+
return new Promise((resolve) => this.liveWaiters.push(resolve));
|
|
278
|
+
}
|
|
279
|
+
cv() {
|
|
280
|
+
return this.room?.cv();
|
|
281
|
+
}
|
|
282
|
+
upstreamEpoch() {
|
|
283
|
+
return this.room?.epoch();
|
|
284
|
+
}
|
|
285
|
+
bootId() {
|
|
286
|
+
return this.incarnationBootId;
|
|
287
|
+
}
|
|
288
|
+
async close() {
|
|
289
|
+
this.closed = true;
|
|
290
|
+
if (this.sweepTimer)
|
|
291
|
+
clearInterval(this.sweepTimer);
|
|
292
|
+
if (this.ackTimer !== null) {
|
|
293
|
+
clearTimeout(this.ackTimer);
|
|
294
|
+
this.ackTimer = null;
|
|
295
|
+
}
|
|
296
|
+
if (this.flushTimer !== null) {
|
|
297
|
+
clearTimeout(this.flushTimer);
|
|
298
|
+
this.flushTimer = null;
|
|
299
|
+
}
|
|
300
|
+
this.upstream?.close();
|
|
301
|
+
for (const client of this.wss.clients)
|
|
302
|
+
client.close();
|
|
303
|
+
await new Promise((resolve) => this.wss.close(() => resolve()));
|
|
304
|
+
if (this.control) {
|
|
305
|
+
await new Promise((resolve) => this.control?.close(() => resolve()));
|
|
306
|
+
}
|
|
307
|
+
if (this.room) {
|
|
308
|
+
this.room.free();
|
|
309
|
+
this.room = null;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
// ------------------------------ upstream leg ------------------------------
|
|
313
|
+
connectUpstream() {
|
|
314
|
+
if (this.closed)
|
|
315
|
+
return;
|
|
316
|
+
const ws = new WebSocket(this.opts.upstream.wsUrl);
|
|
317
|
+
this.upstream = ws;
|
|
318
|
+
ws.on("open", () => {
|
|
319
|
+
this.reconnectDelayMs = RESUBSCRIBE_DELAY_MS;
|
|
320
|
+
send(ws, {
|
|
321
|
+
t: "init",
|
|
322
|
+
clientID: this.opts.upstream.clientId ?? `room-shell-${this.incarnationBootId}`,
|
|
323
|
+
});
|
|
324
|
+
this.sendSubscribe();
|
|
325
|
+
});
|
|
326
|
+
ws.on("message", (data) => this.onUpstreamFrame(data));
|
|
327
|
+
ws.on("error", (err) => this.log(`upstream socket error: ${String(err)}`));
|
|
328
|
+
ws.on("close", () => {
|
|
329
|
+
if (this.closed)
|
|
330
|
+
return;
|
|
331
|
+
// A subscribe may have died with the socket — the fresh connection must be free
|
|
332
|
+
// to send its own.
|
|
333
|
+
this.subscribeInFlight = false;
|
|
334
|
+
this.incarnationDead("upstream socket closed");
|
|
335
|
+
const delay = this.reconnectDelayMs;
|
|
336
|
+
this.reconnectDelayMs = Math.min(delay * 2, RECONNECT_MAX_MS);
|
|
337
|
+
this.log(`upstream reconnect in ${delay}ms`);
|
|
338
|
+
setTimeout(() => this.connectUpstream(), delay).unref?.();
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
/** (Re-)send the upstream subscribe. On an already-open socket rindled treats a
|
|
342
|
+
* re-send for the same queryId as gap recovery: tears down the old attachment, bumps
|
|
343
|
+
* the epoch, re-snapshots. */
|
|
344
|
+
sendSubscribe() {
|
|
345
|
+
if (this.closed || this.subscribeInFlight)
|
|
346
|
+
return;
|
|
347
|
+
if (this.upstream?.readyState !== WebSocket.OPEN)
|
|
348
|
+
return; // reconnect path re-enters
|
|
349
|
+
this.subscribeInFlight = true;
|
|
350
|
+
send(this.upstream, { t: "subscribe", queryId: UPSTREAM_QID, leaseToken: this.leaseToken });
|
|
351
|
+
}
|
|
352
|
+
onUpstreamFrame(data) {
|
|
353
|
+
let frame;
|
|
354
|
+
try {
|
|
355
|
+
frame = JSON.parse(String(data));
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
// Not even JSON: the transport is garbage — violation posture.
|
|
359
|
+
this.violation("upstream frame is not JSON");
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
if (frame.queryId !== undefined && frame.queryId !== UPSTREAM_QID)
|
|
363
|
+
return;
|
|
364
|
+
switch (frame.t) {
|
|
365
|
+
case "nhello": {
|
|
366
|
+
this.subscribeInFlight = false;
|
|
367
|
+
// §301: the daemon boot this connection now serves. A CHANGED boot resets the absorbed
|
|
368
|
+
// cv watermark — the new boot's cv space starts over, and the advert must never pair an
|
|
369
|
+
// old cv with a new boot (the client's §2.4 boot rule keys on the pair).
|
|
370
|
+
const boot = typeof frame.bootId === "string" ? frame.bootId : undefined;
|
|
371
|
+
if (boot !== this.upstreamBoot) {
|
|
372
|
+
this.upstreamBoot = boot;
|
|
373
|
+
this.upstreamCvMin = undefined;
|
|
374
|
+
}
|
|
375
|
+
// A hello supersedes any prior incarnation (e.g. rindled restarted and
|
|
376
|
+
// re-served us): the old store is gone either way.
|
|
377
|
+
if (this.room)
|
|
378
|
+
this.incarnationDead("superseded by a new upstream hello");
|
|
379
|
+
try {
|
|
380
|
+
this.room = WasmRoom.open(JSON.stringify(frame.hello), this.opts.downstream.idleTtlMs ?? DEFAULT_IDLE_TTL_MS);
|
|
381
|
+
this.tableShapes = shapesOf(frame.hello);
|
|
382
|
+
this.log(`upstream hello: epoch ${this.room.epoch()}`);
|
|
383
|
+
}
|
|
384
|
+
catch (e) {
|
|
385
|
+
this.violation(`upstream hello rejected: ${String(e)}`);
|
|
386
|
+
}
|
|
387
|
+
break;
|
|
388
|
+
}
|
|
389
|
+
case "nbatch": {
|
|
390
|
+
if (!this.room)
|
|
391
|
+
return; // pre-hello or post-death stragglers: drop
|
|
392
|
+
let status;
|
|
393
|
+
try {
|
|
394
|
+
status = JSON.parse(this.room.apply(JSON.stringify(frame.batch)));
|
|
395
|
+
}
|
|
396
|
+
catch (e) {
|
|
397
|
+
this.violation(`upstream apply failed: ${String(e)}`);
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
if (status.applied === "snapshot") {
|
|
401
|
+
this.log(`upstream snapshot: ${status.rows} rows @ cv ${this.room.cv()}`);
|
|
402
|
+
void this.finishBoot(this.room);
|
|
403
|
+
}
|
|
404
|
+
else if (status.applied === "live") {
|
|
405
|
+
this.drainDownstream();
|
|
406
|
+
}
|
|
407
|
+
else if (status.applied === "staleEpoch") {
|
|
408
|
+
this.log("dropped stale-epoch upstream frame");
|
|
409
|
+
}
|
|
410
|
+
break;
|
|
411
|
+
}
|
|
412
|
+
case "progress": {
|
|
413
|
+
// §301 (301-ECHO-FENCE-DESIGN.md §1.2): the daemon's release point is THE provable
|
|
414
|
+
// absorption statement — "everything ≤ cvMin this room will ever be sent has been
|
|
415
|
+
// sent", and same-socket ordering means it is already applied. Record it and
|
|
416
|
+
// RE-ADVERTISE to every subscribed connection even with no data flowing (an
|
|
417
|
+
// out-of-footprint echo's fence must clear without waiting for unrelated churn); the
|
|
418
|
+
// room's own cvMin in that frame is unchanged, so clients release nothing new.
|
|
419
|
+
const cvMin = frame.frame?.cvMin;
|
|
420
|
+
if (typeof cvMin === "number" && cvMin > (this.upstreamCvMin ?? -1)) {
|
|
421
|
+
this.upstreamCvMin = cvMin;
|
|
422
|
+
const touched = new Set();
|
|
423
|
+
for (const meta of this.subs.values())
|
|
424
|
+
touched.add(meta.conn);
|
|
425
|
+
for (const conn of touched)
|
|
426
|
+
this.sendProgress(conn);
|
|
427
|
+
}
|
|
428
|
+
break;
|
|
429
|
+
}
|
|
430
|
+
case "queryError": {
|
|
431
|
+
// The lease died (expiry, dematerialize, daemon restart): re-mint, re-subscribe.
|
|
432
|
+
this.subscribeInFlight = false;
|
|
433
|
+
this.incarnationDead(`upstream queryError: ${String(frame.message)}`);
|
|
434
|
+
void mintLease(this.opts.upstream)
|
|
435
|
+
.then((token) => {
|
|
436
|
+
this.leaseToken = token;
|
|
437
|
+
setTimeout(() => this.sendSubscribe(), RESUBSCRIBE_DELAY_MS).unref?.();
|
|
438
|
+
})
|
|
439
|
+
.catch((e) => {
|
|
440
|
+
this.log(`lease re-mint failed: ${String(e)}`);
|
|
441
|
+
// The socket is still up; retry the whole path after a beat.
|
|
442
|
+
setTimeout(() => {
|
|
443
|
+
if (!this.closed)
|
|
444
|
+
this.onUpstreamFrame(data);
|
|
445
|
+
}, RECONNECT_MAX_MS).unref?.();
|
|
446
|
+
});
|
|
447
|
+
break;
|
|
448
|
+
}
|
|
449
|
+
default:
|
|
450
|
+
break;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
/** A §3.4 violation: kill the incarnation and re-subscribe (never patch up). */
|
|
454
|
+
violation(reason) {
|
|
455
|
+
this.incarnationDead(reason);
|
|
456
|
+
setTimeout(() => this.sendSubscribe(), RESUBSCRIBE_DELAY_MS).unref?.();
|
|
457
|
+
}
|
|
458
|
+
incarnationDead(reason) {
|
|
459
|
+
this.log(`incarnation dead: ${reason}`);
|
|
460
|
+
if (this.room) {
|
|
461
|
+
this.room.free();
|
|
462
|
+
this.room = null;
|
|
463
|
+
}
|
|
464
|
+
this.live = false;
|
|
465
|
+
this.incarnationBootId = randomUUID();
|
|
466
|
+
// Downstream subscriptions die with the incarnation (§3.4): close the sockets;
|
|
467
|
+
// reconnecting clients re-subscribe against the next incarnation.
|
|
468
|
+
for (const meta of this.subs.values()) {
|
|
469
|
+
meta.ws.close(1012, "room re-subscribing");
|
|
470
|
+
}
|
|
471
|
+
this.subs.clear();
|
|
472
|
+
this.pendingSubs = [];
|
|
473
|
+
// The write plane's optimism dies with the incarnation too: whatever was not yet
|
|
474
|
+
// journaled was never acked, and never will be (§8.1's `applied` class). The
|
|
475
|
+
// journal itself outlives us — the next incarnation replays it.
|
|
476
|
+
this.pendingMutes = [];
|
|
477
|
+
this.unacked.clear();
|
|
478
|
+
// Recorded outcomes die with the incarnation too — the next incarnation reseeds
|
|
479
|
+
// them from its own journal replay (finishBoot), which is the source of truth for
|
|
480
|
+
// what THAT incarnation's base produced.
|
|
481
|
+
this.outcomes.clear();
|
|
482
|
+
this.ackQueue = [];
|
|
483
|
+
if (this.ackTimer !== null) {
|
|
484
|
+
clearTimeout(this.ackTimer);
|
|
485
|
+
this.ackTimer = null;
|
|
486
|
+
}
|
|
487
|
+
// The flush debounce dies with the incarnation; an in-flight settlement keeps
|
|
488
|
+
// running (its batch is journaled and epoch-bound — the bootId guard keeps its
|
|
489
|
+
// outcome from touching the next incarnation's state).
|
|
490
|
+
if (this.flushTimer !== null) {
|
|
491
|
+
clearTimeout(this.flushTimer);
|
|
492
|
+
this.flushTimer = null;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
/** After the seq-0 snapshot: enable the write plane, settle any unconfirmed flush
|
|
496
|
+
* batches, probe the authority's durable lmids, and replay the journal onto the
|
|
497
|
+
* fresh base (§3.3 — pending re-invoked; a mid the probe covered replays as dedup,
|
|
498
|
+
* its effects already in the snapshot; everything replayed acked, since journal
|
|
499
|
+
* presence IS the ack), then start serving. The order is load-bearing: the
|
|
500
|
+
* resubmission SETTLES before the probe, so a batch a dead incarnation left
|
|
501
|
+
* mid-network can never land between the probe and our first flush. No subscriber
|
|
502
|
+
* exists yet (they died with the previous incarnation), so replay fans out to
|
|
503
|
+
* nobody — the snapshots the queued subscribes get below already include the
|
|
504
|
+
* replayed rows. */
|
|
505
|
+
async finishBoot(room) {
|
|
506
|
+
const writes = this.opts.downstream.writes;
|
|
507
|
+
if (writes && this.journal) {
|
|
508
|
+
let entries;
|
|
509
|
+
try {
|
|
510
|
+
// H-iv-b: boot-wire scopes arm the §3.3 commit gate (v2); without them the v1
|
|
511
|
+
// table-granular plane enables byte-identically to before. Either enable
|
|
512
|
+
// throws loudly on a malformed input and leaves writes disabled — the
|
|
513
|
+
// violation below keeps that from becoming a half-enabled room.
|
|
514
|
+
if (writes.scopes !== undefined) {
|
|
515
|
+
room.enableWritesV2(JSON.stringify(writes.scopes));
|
|
516
|
+
}
|
|
517
|
+
else {
|
|
518
|
+
room.enableWrites(JSON.stringify(writes.ownedTables));
|
|
519
|
+
}
|
|
520
|
+
entries = await this.journal.replay();
|
|
521
|
+
}
|
|
522
|
+
catch (e) {
|
|
523
|
+
this.violation(`write-plane boot failed: ${String(e)}`);
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
if (this.room !== room)
|
|
527
|
+
return; // an incarnation death raced the replay
|
|
528
|
+
if (writes.authority) {
|
|
529
|
+
try {
|
|
530
|
+
await this.settleUnconfirmedFlushes(writes.authority);
|
|
531
|
+
}
|
|
532
|
+
catch (e) {
|
|
533
|
+
if (this.moved || this.closed)
|
|
534
|
+
return;
|
|
535
|
+
this.violation(`flush resubmission failed: ${String(e)}`);
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
if (this.room !== room)
|
|
539
|
+
return;
|
|
540
|
+
const clients = [...new Set(entries.map((e) => e.clientID))];
|
|
541
|
+
if (clients.length > 0) {
|
|
542
|
+
let lmids;
|
|
543
|
+
try {
|
|
544
|
+
lmids = await withNetRetry(() => writes.authority.lmids(this.opts.downstream.docId, clients), () => this.closed || this.room !== room, this.log);
|
|
545
|
+
}
|
|
546
|
+
catch (e) {
|
|
547
|
+
if (this.room !== room || this.closed)
|
|
548
|
+
return;
|
|
549
|
+
this.violation(`durable-lmid probe failed: ${String(e)}`);
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
if (this.room !== room)
|
|
553
|
+
return;
|
|
554
|
+
const seeds = Object.entries(lmids)
|
|
555
|
+
.filter(([, lmid]) => lmid > 0)
|
|
556
|
+
.map(([clientID, lmid]) => ({ clientID, lmid }));
|
|
557
|
+
if (seeds.length > 0) {
|
|
558
|
+
try {
|
|
559
|
+
room.seedDurable(JSON.stringify(seeds));
|
|
560
|
+
room.commitAll(); // drop the (subscriber-less) ledger fanout
|
|
561
|
+
}
|
|
562
|
+
catch (e) {
|
|
563
|
+
this.violation(`durable seed failed: ${String(e)}`);
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
this.log(`durable seed: ${seeds.map((s) => `${s.clientID}→${s.lmid}`).join(", ")}`);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
try {
|
|
571
|
+
for (const entry of entries) {
|
|
572
|
+
const res = this.runMutation(room, entry);
|
|
573
|
+
// Seed the recorded-outcome map with what THIS incarnation's replay
|
|
574
|
+
// produced. That includes the H-iv-a replay gotcha: an entry journaled
|
|
575
|
+
// APPLIED can legitimately DEOPT (or reject) re-invoked against the moved
|
|
576
|
+
// base — the §3.3 rebase class. The journal record is never rewritten
|
|
577
|
+
// (history stays what the acking incarnation observed); the map is this
|
|
578
|
+
// incarnation's answer sheet for re-sent mids.
|
|
579
|
+
if (res.outcome === "rejected" || res.outcome === "deopt") {
|
|
580
|
+
this.recordOutcome(entry.clientID, entry.mid, {
|
|
581
|
+
kind: res.outcome,
|
|
582
|
+
...(res.reason !== undefined ? { reason: res.reason } : {}),
|
|
583
|
+
...(res.outcome === "deopt" ? { name: entry.name, args: entry.args } : {}),
|
|
584
|
+
});
|
|
585
|
+
if (journalEntryOutcome(entry) === "applied") {
|
|
586
|
+
this.log(`replayed APPLIED mutation ${entry.clientID}:${entry.mid} (\`${entry.name}\`) ` +
|
|
587
|
+
`now ${res.outcome}s against the moved base — mid stays burnt, no effects`);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
if (entries.length > 0) {
|
|
592
|
+
room.ack(JSON.stringify(entries.map(({ clientID, mid }) => ({ clientID, mid }))));
|
|
593
|
+
room.commitAll(); // drop the (subscriber-less) fanout of the replay
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
catch (e) {
|
|
597
|
+
this.violation(`journal replay failed: ${String(e)}`);
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
if (entries.length > 0) {
|
|
601
|
+
this.log(`journal replay: ${entries.length} mutation(s) re-invoked and acked`);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
this.becomeLive();
|
|
605
|
+
this.scheduleFlush(); // replay may have left dirty entries / unflushed lmids
|
|
606
|
+
}
|
|
607
|
+
/** Resubmit every unconfirmed journaled flush batch, byte-identically, in seq order
|
|
608
|
+
* (§3.3/§5.3 step 4). Outcomes: committed or deduped → confirm; fenced under an
|
|
609
|
+
* OLD epoch → routine cleanup (a prior process's batch — its mutations recover by
|
|
610
|
+
* replay under OUR epoch); fenced under OUR epoch → another room claimed the doc:
|
|
611
|
+
* terminal (`room_moved`); conflict → drop (the replay + next flush re-derive the
|
|
612
|
+
* net effect against the authority's current rows); identity mismatch → fatal. */
|
|
613
|
+
async settleUnconfirmedFlushes(authority) {
|
|
614
|
+
if (!this.journal)
|
|
615
|
+
return;
|
|
616
|
+
const { records } = await this.journal.replayFlushes();
|
|
617
|
+
for (const record of records) {
|
|
618
|
+
if (this.closed || this.moved)
|
|
619
|
+
return;
|
|
620
|
+
let res;
|
|
621
|
+
try {
|
|
622
|
+
res = await withNetRetry(() => authority.applyRowChangeTxn(record.body), () => this.closed || this.moved, this.log);
|
|
623
|
+
}
|
|
624
|
+
catch (e) {
|
|
625
|
+
if (e?.fatal !== true)
|
|
626
|
+
throw e;
|
|
627
|
+
// The §8.3 identity check tripped on a journaled record — a same-id/
|
|
628
|
+
// different-body bug. LOUD, dropped, never a silent dedup and never a
|
|
629
|
+
// violation loop: the batch is dead; the envelope replay re-derives its
|
|
630
|
+
// effects under a fresh flush id.
|
|
631
|
+
this.log(`flush ${record.seq} DROPPED: batch identity mismatch: ${String(e)}`);
|
|
632
|
+
await this.journal.confirmFlush(record.seq);
|
|
633
|
+
continue;
|
|
634
|
+
}
|
|
635
|
+
if (res.kind === "ok") {
|
|
636
|
+
await this.journal.confirmFlush(record.seq);
|
|
637
|
+
this.log(`flush ${record.seq} settled at boot: ${res.applied ? "applied" : "already applied"}`);
|
|
638
|
+
}
|
|
639
|
+
else if (res.kind === "conflict") {
|
|
640
|
+
await this.journal.confirmFlush(record.seq);
|
|
641
|
+
this.log(`flush ${record.seq} dropped at boot: conflict (replay supersedes)`);
|
|
642
|
+
}
|
|
643
|
+
else {
|
|
644
|
+
await this.journal.confirmFlush(record.seq);
|
|
645
|
+
if (record.epoch === this.placementEpoch) {
|
|
646
|
+
this.roomMoved(`flush ${record.seq} fenced under our epoch`);
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
this.log(`flush ${record.seq} (old epoch ${record.epoch}) fenced: dropped`);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
becomeLive() {
|
|
654
|
+
this.live = true;
|
|
655
|
+
const waiters = this.liveWaiters;
|
|
656
|
+
this.liveWaiters = [];
|
|
657
|
+
for (const w of waiters)
|
|
658
|
+
w();
|
|
659
|
+
const queued = this.pendingSubs;
|
|
660
|
+
this.pendingSubs = [];
|
|
661
|
+
for (const { ws, conn, msg } of queued) {
|
|
662
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
663
|
+
conn.busy = conn.busy.then(() => this.handleSubscribe(ws, conn, msg));
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
const mutes = this.pendingMutes;
|
|
667
|
+
this.pendingMutes = [];
|
|
668
|
+
for (const { ws, conn, envelope } of mutes) {
|
|
669
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
670
|
+
conn.busy = conn.busy.then(() => this.handlePushMutation(ws, conn, envelope));
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
/** After any commit (an upstream apply, a local mutation, an ack): one commitAll
|
|
675
|
+
* drains every query's net delta, fanned out per subscriber envelope (empty → no
|
|
676
|
+
* frame, no seq — every stream stays gap-free over emitted frames), then each
|
|
677
|
+
* touched connection gets its `progress` release point. */
|
|
678
|
+
drainDownstream() {
|
|
679
|
+
if (!this.room)
|
|
680
|
+
return;
|
|
681
|
+
const frames = JSON.parse(this.room.commitAll());
|
|
682
|
+
const touched = new Set();
|
|
683
|
+
for (const { sub, batch } of frames) {
|
|
684
|
+
const meta = this.subs.get(sub);
|
|
685
|
+
if (meta) {
|
|
686
|
+
this.sendBudgeted(meta.conn, { t: "nbatch", queryId: meta.clientQid, batch });
|
|
687
|
+
touched.add(meta.conn);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
for (const conn of touched)
|
|
691
|
+
this.sendProgress(conn);
|
|
692
|
+
}
|
|
693
|
+
/** The connection-level release point (`{t:"progress", frame:{cvMin}}`): `head_cv`,
|
|
694
|
+
* except an author with un-acked mutations holds below its oldest one's apply-cv —
|
|
695
|
+
* its own data must never release ahead of the lmid advance that confirms it
|
|
696
|
+
* (§1.3.1: the client would re-invoke the mutator on top of its own effect). */
|
|
697
|
+
sendProgress(conn) {
|
|
698
|
+
if (!this.room)
|
|
699
|
+
return;
|
|
700
|
+
let cvMin = this.room.headCv();
|
|
701
|
+
if (conn.clientID) {
|
|
702
|
+
const held = this.unacked.get(conn.clientID);
|
|
703
|
+
if (held && held.length > 0) {
|
|
704
|
+
cvMin = Math.min(cvMin, held[0].applyCv - 1);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
// §301: stamp the upstream-absorption advert (once the current daemon boot has released
|
|
708
|
+
// anything to us) — the client's direction-B pin fence input. Optional fields: an old
|
|
709
|
+
// client reads `cvMin` only.
|
|
710
|
+
const frame = { cvMin };
|
|
711
|
+
if (this.upstreamCvMin !== undefined) {
|
|
712
|
+
frame.upstreamCv = this.upstreamCvMin;
|
|
713
|
+
if (this.upstreamBoot !== undefined)
|
|
714
|
+
frame.upstreamBoot = this.upstreamBoot;
|
|
715
|
+
}
|
|
716
|
+
this.sendBudgeted(conn, { t: "progress", frame });
|
|
717
|
+
}
|
|
718
|
+
/** §9's per-socket send budget (T8): a socket whose queued bytes would exceed it is
|
|
719
|
+
* TERMINATED — the close handler reclaims its subscriptions, so room memory stays
|
|
720
|
+
* bounded and the client repairs by re-subscribing (a gap, §8.5). Never buffer
|
|
721
|
+
* unboundedly on behalf of one slow reader. */
|
|
722
|
+
sendBudgeted(conn, frame) {
|
|
723
|
+
const ws = conn.ws;
|
|
724
|
+
if (ws.readyState !== WebSocket.OPEN)
|
|
725
|
+
return;
|
|
726
|
+
const text = JSON.stringify(frame);
|
|
727
|
+
const budget = this.opts.downstream.sendBudgetBytes ?? DEFAULT_SEND_BUDGET_BYTES;
|
|
728
|
+
if (ws.bufferedAmount + text.length > budget) {
|
|
729
|
+
this.log(`send budget exceeded (conn ${conn.id}): closed with a gap`);
|
|
730
|
+
ws.terminate();
|
|
731
|
+
return;
|
|
732
|
+
}
|
|
733
|
+
ws.send(text);
|
|
734
|
+
}
|
|
735
|
+
// ------------------------------ the write path ------------------------------
|
|
736
|
+
/** Apply one envelope to the room: dedup → run the mutator against a staged tx →
|
|
737
|
+
* commit (or reject/deopt, consuming the mid). Shared verbatim by the live path and
|
|
738
|
+
* the boot-time journal replay (§3.3 — recovery is re-invocation; a journaled
|
|
739
|
+
* NON-APPLIED entry replays its recorded outcome without running anything). Verdict
|
|
740
|
+
* classification (H-iv-b): the §3.3 commit gate's structured verdict and an
|
|
741
|
+
* environment shortfall (a capability the room lacks — `tx.query`) are DEOPTS (the
|
|
742
|
+
* client re-routes the mutation to the daemon stream); an unknown mutator or an
|
|
743
|
+
* authz/validation throw is a FINAL rejection. Throws only when the room is torn
|
|
744
|
+
* (a mid gap, a poisoned commit) — the caller decides the blast radius. */
|
|
745
|
+
runMutation(room, entry) {
|
|
746
|
+
// The envelope rides the wasm tx (Slice I-ii): should this mid end non-applied,
|
|
747
|
+
// its durable outcome row echoes `name`/`args` on a DEOPT — the same
|
|
748
|
+
// self-contained-re-invoke rule as the frame (H-iv-b), but readable through the
|
|
749
|
+
// daemon after downgrade when no room socket exists to deliver one.
|
|
750
|
+
const begin = JSON.parse(room.beginMutation(entry.clientID, entry.mid, entry.name, entry.args === undefined ? undefined : JSON.stringify(entry.args)));
|
|
751
|
+
if (begin.begin === "dedup")
|
|
752
|
+
return { outcome: "dedup" };
|
|
753
|
+
// A journaled non-applied entry replays its RECORDED verdict without running: the
|
|
754
|
+
// live run already consumed the mid with no effects, and re-judging (a deopt that
|
|
755
|
+
// would now PASS against the moved base) would invent effects for a mutation the
|
|
756
|
+
// client was told to re-route elsewhere — a double apply. Passing the verdict
|
|
757
|
+
// through the reject IS the replay re-seed (I-ii): a recorded-but-never-flushed
|
|
758
|
+
// outcome re-enters the core buffer here, so the next flush carries its row —
|
|
759
|
+
// while a mid the boot probe covered dedups above and never re-seeds (its row
|
|
760
|
+
// co-committed with the flush that made it durable). The journal keeps no
|
|
761
|
+
// `reason`, so a replayed row carries kind + envelope only, like the replayed
|
|
762
|
+
// frame.
|
|
763
|
+
const journaled = journalEntryOutcome(entry);
|
|
764
|
+
if (journaled !== "applied") {
|
|
765
|
+
room.rejectMutation(journaled, undefined);
|
|
766
|
+
return { outcome: journaled };
|
|
767
|
+
}
|
|
768
|
+
const mutator = this.opts.downstream.writes?.mutators[entry.name];
|
|
769
|
+
if (!mutator) {
|
|
770
|
+
const reason = `unknown mutator \`${entry.name}\``;
|
|
771
|
+
room.rejectMutation("rejected", reason);
|
|
772
|
+
this.log(`mutation ${entry.clientID}:${entry.mid} rejected: ${reason}`);
|
|
773
|
+
return { outcome: "rejected", reason };
|
|
774
|
+
}
|
|
775
|
+
try {
|
|
776
|
+
// The ambient auth context (managed-writes §3.2): the journaled subject, so a
|
|
777
|
+
// replayed invocation sees exactly the identity the live one did (`""` for
|
|
778
|
+
// entries journaled before the identity plane — unauthenticated).
|
|
779
|
+
const returned = mutator(mutationTx(room, this.tableShapes), entry.args, { user: entry.sub ?? "" });
|
|
780
|
+
assertSyncMutatorReturn(returned, entry.name);
|
|
781
|
+
}
|
|
782
|
+
catch (e) {
|
|
783
|
+
// The shell owns the H-iv-b classification, so it rides the reject into the
|
|
784
|
+
// core's outcome record (the flush carries the durable row) exactly as it rides
|
|
785
|
+
// the frame below.
|
|
786
|
+
const message = String(e?.message ?? e);
|
|
787
|
+
if (isEnvironmentShortfall(e)) {
|
|
788
|
+
room.rejectMutation("deopt", "environment");
|
|
789
|
+
this.log(`mutation \`${entry.name}\` (${entry.clientID}:${entry.mid}) deopted: ` +
|
|
790
|
+
`environment shortfall: ${message}`);
|
|
791
|
+
return { outcome: "deopt", reason: "environment" };
|
|
792
|
+
}
|
|
793
|
+
room.rejectMutation("rejected", message);
|
|
794
|
+
this.log(`mutation \`${entry.name}\` (${entry.clientID}:${entry.mid}) rejected: ${message}`);
|
|
795
|
+
return { outcome: "rejected", reason: message };
|
|
796
|
+
}
|
|
797
|
+
const out = JSON.parse(room.commitMutation());
|
|
798
|
+
if (out.deopt !== undefined) {
|
|
799
|
+
// The gate refused at commit and consumed the tx INTERNALLY (mid burnt,
|
|
800
|
+
// watermark advanced, no head commit) — do NOT call rejectMutation here.
|
|
801
|
+
this.log(`mutation \`${entry.name}\` (${entry.clientID}:${entry.mid}) deopted: ` +
|
|
802
|
+
`${out.deopt.reason} on \`${out.deopt.table}\` pk ${JSON.stringify(out.deopt.pk)}`);
|
|
803
|
+
return { outcome: "deopt", reason: out.deopt.reason };
|
|
804
|
+
}
|
|
805
|
+
return { outcome: "applied", applyCv: out.headCv };
|
|
806
|
+
}
|
|
807
|
+
/** The live `pushMutation` path (§5.1): apply → fan the data out NOW → journal under
|
|
808
|
+
* the group commit → ack (the ledger advance) once the append resolves. */
|
|
809
|
+
handlePushMutation(ws, conn, envelope) {
|
|
810
|
+
const fail = (message) => send(ws, { t: "error", message });
|
|
811
|
+
if (!this.opts.downstream.writes || !this.journal) {
|
|
812
|
+
fail("this room is read-only (no write plane configured)");
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
815
|
+
if (!conn.clientID) {
|
|
816
|
+
fail("init with a clientID before pushMutation");
|
|
817
|
+
return;
|
|
818
|
+
}
|
|
819
|
+
if (envelope.clientID !== conn.clientID) {
|
|
820
|
+
// The envelope's clientID is bound to the connection identity — a session may
|
|
821
|
+
// not write another client's mid stream.
|
|
822
|
+
fail("envelope clientID does not match the connection identity");
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
825
|
+
if (!this.live || !this.room) {
|
|
826
|
+
this.pendingMutes.push({ ws, conn, envelope });
|
|
827
|
+
return;
|
|
828
|
+
}
|
|
829
|
+
// Checked at execution time (after the live gate), so a mutation queued behind a
|
|
830
|
+
// still-pending subscribe is judged AFTER that subscribe bound the subject.
|
|
831
|
+
if (conn.sub === null) {
|
|
832
|
+
fail("pushMutation requires an authenticated subject — subscribe with a lease token first");
|
|
833
|
+
return;
|
|
834
|
+
}
|
|
835
|
+
const room = this.room;
|
|
836
|
+
let result;
|
|
837
|
+
try {
|
|
838
|
+
result = this.runMutation(room, { ...envelope, sub: conn.sub });
|
|
839
|
+
}
|
|
840
|
+
catch (e) {
|
|
841
|
+
const message = String(e?.message ?? e);
|
|
842
|
+
if (room.isPoisoned()) {
|
|
843
|
+
this.violation(`mutation commit tore the head: ${message}`);
|
|
844
|
+
}
|
|
845
|
+
else {
|
|
846
|
+
// The mid-gap contract: the exact "mutation gap …" text the client's
|
|
847
|
+
// recovery keys on rides an error frame.
|
|
848
|
+
fail(message);
|
|
849
|
+
}
|
|
850
|
+
return;
|
|
851
|
+
}
|
|
852
|
+
if (result.outcome === "dedup") {
|
|
853
|
+
// Absorbed; the ledger already covers it (or its in-flight ack will). But a
|
|
854
|
+
// re-sent NON-APPLIED mid is ANSWERED with its recorded outcome (H-iv-b): the
|
|
855
|
+
// client may have missed the original frame (reconnect), and silence would
|
|
856
|
+
// leave its deopted mutation parked forever.
|
|
857
|
+
const recorded = this.outcomes.get(conn.clientID)?.get(envelope.mid);
|
|
858
|
+
if (recorded !== undefined)
|
|
859
|
+
this.sendOutcome(conn, envelope.mid, recorded);
|
|
860
|
+
return;
|
|
861
|
+
}
|
|
862
|
+
if (result.outcome === "applied") {
|
|
863
|
+
const held = this.unacked.get(conn.clientID) ?? [];
|
|
864
|
+
held.push({ mid: envelope.mid, applyCv: result.applyCv });
|
|
865
|
+
this.unacked.set(conn.clientID, held);
|
|
866
|
+
this.drainDownstream(); // §5.1 step 2: never gated on durability
|
|
867
|
+
}
|
|
868
|
+
else {
|
|
869
|
+
// Non-applied (deopt/rejected): record + answer NOW — synchronously, before the
|
|
870
|
+
// journal enqueue below, so on this ordered socket the `mutationOutcome` frame
|
|
871
|
+
// always precedes the lmid ack that burns the mid. Applied mutations send
|
|
872
|
+
// NOTHING (additive frame: old clients drop unknown `t`).
|
|
873
|
+
const recorded = {
|
|
874
|
+
kind: result.outcome,
|
|
875
|
+
...(result.reason !== undefined ? { reason: result.reason } : {}),
|
|
876
|
+
...(result.outcome === "deopt" ? { name: envelope.name, args: envelope.args } : {}),
|
|
877
|
+
};
|
|
878
|
+
this.recordOutcome(conn.clientID, envelope.mid, recorded);
|
|
879
|
+
this.sendOutcome(conn, envelope.mid, recorded);
|
|
880
|
+
}
|
|
881
|
+
this.enqueueJournal({
|
|
882
|
+
clientID: envelope.clientID,
|
|
883
|
+
mid: envelope.mid,
|
|
884
|
+
name: envelope.name,
|
|
885
|
+
args: envelope.args,
|
|
886
|
+
sub: conn.sub,
|
|
887
|
+
outcome: result.outcome,
|
|
888
|
+
// The legacy flag rides alongside BOTH non-applied kinds: a pre-H-iv-b reader
|
|
889
|
+
// replays either as a consumed-mid-no-effect — exactly right (journal.ts).
|
|
890
|
+
...(result.outcome !== "applied" ? { rejected: true } : {}),
|
|
891
|
+
});
|
|
892
|
+
// §5.1 step 5: the write-behind rides its own debounce — applied mutations dirty
|
|
893
|
+
// rows, rejected/deopted ones still advance an lmid the authority must eventually
|
|
894
|
+
// hold.
|
|
895
|
+
this.scheduleFlush();
|
|
896
|
+
}
|
|
897
|
+
/** Record a non-applied verdict for `(clientID, mid)` — the re-send answer sheet.
|
|
898
|
+
* Insertion-ordered per client; past {@link MAX_RECORDED_OUTCOMES_PER_CLIENT} the
|
|
899
|
+
* oldest evicts (see the constant's retention rationale). */
|
|
900
|
+
recordOutcome(clientID, mid, outcome) {
|
|
901
|
+
let byMid = this.outcomes.get(clientID);
|
|
902
|
+
if (byMid === undefined) {
|
|
903
|
+
byMid = new Map();
|
|
904
|
+
this.outcomes.set(clientID, byMid);
|
|
905
|
+
}
|
|
906
|
+
byMid.delete(mid); // re-recording refreshes recency
|
|
907
|
+
byMid.set(mid, outcome);
|
|
908
|
+
while (byMid.size > MAX_RECORDED_OUTCOMES_PER_CLIENT) {
|
|
909
|
+
byMid.delete(byMid.keys().next().value);
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
/** The `mutationOutcome` frame (H-iv-b): `{t, mid, kind, reason?, name?, args?}` on
|
|
913
|
+
* the mutating client's socket. DEOPT frames echo `name`/`args` so they are
|
|
914
|
+
* self-contained — a client that already retired the entry can re-invoke from the
|
|
915
|
+
* frame alone. Sent before the mid's journal append is even enqueued, so it always
|
|
916
|
+
* precedes the lmid ack on this ordered socket. */
|
|
917
|
+
sendOutcome(conn, mid, o) {
|
|
918
|
+
this.sendBudgeted(conn, {
|
|
919
|
+
t: "mutationOutcome",
|
|
920
|
+
mid,
|
|
921
|
+
kind: o.kind,
|
|
922
|
+
...(o.reason !== undefined ? { reason: o.reason } : {}),
|
|
923
|
+
...(o.kind === "deopt" ? { name: o.name, args: o.args } : {}),
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
/** Collect entries for the next group commit (§5.1 step 3: one append per window). */
|
|
927
|
+
enqueueJournal(entry) {
|
|
928
|
+
this.ackQueue.push(entry);
|
|
929
|
+
if (this.ackTimer === null) {
|
|
930
|
+
const window = this.opts.downstream.writes?.groupCommitMs ?? DEFAULT_GROUP_COMMIT_MS;
|
|
931
|
+
this.ackTimer = setTimeout(() => this.flushJournal(), window);
|
|
932
|
+
this.ackTimer.unref?.();
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
flushJournal() {
|
|
936
|
+
this.ackTimer = null;
|
|
937
|
+
const batch = this.ackQueue;
|
|
938
|
+
this.ackQueue = [];
|
|
939
|
+
if (batch.length === 0 || !this.journal)
|
|
940
|
+
return;
|
|
941
|
+
const journal = this.journal;
|
|
942
|
+
const bootAtFlush = this.incarnationBootId;
|
|
943
|
+
// One append in flight at a time; acks apply in append order.
|
|
944
|
+
this.ackChain = this.ackChain.then(async () => {
|
|
945
|
+
try {
|
|
946
|
+
await journal.append(batch);
|
|
947
|
+
}
|
|
948
|
+
catch (e) {
|
|
949
|
+
// An ack that might not survive must never be sent (§8.1): the incarnation
|
|
950
|
+
// dies loudly instead.
|
|
951
|
+
if (this.incarnationBootId === bootAtFlush && !this.closed) {
|
|
952
|
+
this.violation(`journal append failed: ${String(e)}`);
|
|
953
|
+
}
|
|
954
|
+
return;
|
|
955
|
+
}
|
|
956
|
+
if (this.incarnationBootId !== bootAtFlush || !this.room) {
|
|
957
|
+
return; // the incarnation died mid-append: its optimism died with it
|
|
958
|
+
}
|
|
959
|
+
this.applyAck(this.room, batch);
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
/** §5.1 step 4 — the ack: advance the ledger rows, ship the lmid frames, and raise
|
|
963
|
+
* every author's release point past its confirmed data. */
|
|
964
|
+
applyAck(room, batch) {
|
|
965
|
+
let headCv;
|
|
966
|
+
try {
|
|
967
|
+
const entries = batch.map(({ clientID, mid }) => ({ clientID, mid }));
|
|
968
|
+
headCv = JSON.parse(room.ack(JSON.stringify(entries)))
|
|
969
|
+
.headCv;
|
|
970
|
+
}
|
|
971
|
+
catch (e) {
|
|
972
|
+
this.violation(`ack failed: ${String(e)}`);
|
|
973
|
+
return;
|
|
974
|
+
}
|
|
975
|
+
const authors = new Set();
|
|
976
|
+
for (const { clientID, mid } of batch) {
|
|
977
|
+
authors.add(clientID);
|
|
978
|
+
const held = this.unacked.get(clientID);
|
|
979
|
+
if (held) {
|
|
980
|
+
const rest = held.filter((u) => u.mid > mid);
|
|
981
|
+
if (rest.length > 0)
|
|
982
|
+
this.unacked.set(clientID, rest);
|
|
983
|
+
else
|
|
984
|
+
this.unacked.delete(clientID);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
if (headCv !== null) {
|
|
988
|
+
this.drainDownstream(); // the lmid frames (+ progress to their receivers)
|
|
989
|
+
}
|
|
990
|
+
// Raise the release point for every author connection even if no frame reached
|
|
991
|
+
// it in this drain (e.g. its lmid slice was already current).
|
|
992
|
+
for (const meta of this.subs.values()) {
|
|
993
|
+
if (meta.conn.clientID && authors.has(meta.conn.clientID)) {
|
|
994
|
+
this.sendProgress(meta.conn);
|
|
995
|
+
authors.delete(meta.conn.clientID);
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
// --------------------------- the write-behind flush ---------------------------
|
|
1000
|
+
/** Arm the flush debounce (or fire now past the dirty-size threshold). Cheap and
|
|
1001
|
+
* idempotent — called after every applied/rejected mutation and every settlement. */
|
|
1002
|
+
scheduleFlush() {
|
|
1003
|
+
const writes = this.opts.downstream.writes;
|
|
1004
|
+
if (!writes?.authority || this.flushBusy || this.flushTimer !== null)
|
|
1005
|
+
return;
|
|
1006
|
+
if (this.closed || this.moved || !this.live || !this.room)
|
|
1007
|
+
return;
|
|
1008
|
+
const dirtyMax = writes.flushDirtyMax ?? DEFAULT_FLUSH_DIRTY_MAX;
|
|
1009
|
+
const delay = this.room.dirtyLen() >= dirtyMax ? 0 : writes.flushDebounceMs ?? DEFAULT_FLUSH_DEBOUNCE_MS;
|
|
1010
|
+
this.flushTimer = setTimeout(() => {
|
|
1011
|
+
this.flushTimer = null;
|
|
1012
|
+
this.fireFlush();
|
|
1013
|
+
}, delay);
|
|
1014
|
+
this.flushTimer.unref?.();
|
|
1015
|
+
}
|
|
1016
|
+
/** Fire the write-behind flush now and await its settlement (tests, drain paths). */
|
|
1017
|
+
flushNow() {
|
|
1018
|
+
if (this.flushTimer !== null) {
|
|
1019
|
+
clearTimeout(this.flushTimer);
|
|
1020
|
+
this.flushTimer = null;
|
|
1021
|
+
}
|
|
1022
|
+
this.fireFlush();
|
|
1023
|
+
return this.flushChain;
|
|
1024
|
+
}
|
|
1025
|
+
/** §4.2/§7.4 drain-before-downgrade: fire the write-behind repeatedly until pending AND dirty
|
|
1026
|
+
* are both empty, awaiting each settlement, then report the last COMMITTED flush seq — the
|
|
1027
|
+
* value a downgraded client's frozen ghost fences against
|
|
1028
|
+
* (`_rindle_room_watermark(doc) ≥ finalFlushSeq`). A never-flushed room reports 0. Idempotent:
|
|
1029
|
+
* re-draining an already-quiescent room re-reports the same seq without flushing. It does NOT
|
|
1030
|
+
* refuse later pushes (§4.2 R2: a straggler push after drain flushes at seq+1; the client
|
|
1031
|
+
* ghost's second conjunct — no sent room-domain pending — carries correctness). A room that
|
|
1032
|
+
* fails to quiesce within {@link DRAIN_MAX_ITERATIONS} throws LOUD (never silently). */
|
|
1033
|
+
async drainForDowngrade() {
|
|
1034
|
+
const writes = this.opts.downstream.writes;
|
|
1035
|
+
let iterations = 0;
|
|
1036
|
+
while (writes?.authority &&
|
|
1037
|
+
this.live &&
|
|
1038
|
+
!this.moved &&
|
|
1039
|
+
!this.closed &&
|
|
1040
|
+
this.room !== null &&
|
|
1041
|
+
(this.room.dirtyLen() > 0 || this.room.pendingLen() > 0)) {
|
|
1042
|
+
if (++iterations > DRAIN_MAX_ITERATIONS) {
|
|
1043
|
+
throw new Error(`drain: room ${this.opts.downstream.docId} failed to quiesce after ${DRAIN_MAX_ITERATIONS} flushes ` +
|
|
1044
|
+
`(dirty=${this.room.dirtyLen()}, pending=${this.room.pendingLen()})`);
|
|
1045
|
+
}
|
|
1046
|
+
await this.flushNow();
|
|
1047
|
+
}
|
|
1048
|
+
return this.lastCommittedFlushSeq;
|
|
1049
|
+
}
|
|
1050
|
+
/** §5.3: build the batch (synchronously — the build IS the snapshot), journal its
|
|
1051
|
+
* exact body bytes, then settle it against the authority. One in flight; the
|
|
1052
|
+
* settlement outcome drives the state machine (§5.4). */
|
|
1053
|
+
fireFlush() {
|
|
1054
|
+
const writes = this.opts.downstream.writes;
|
|
1055
|
+
const authority = writes?.authority;
|
|
1056
|
+
if (!authority || !this.journal || this.flushBusy)
|
|
1057
|
+
return;
|
|
1058
|
+
if (this.closed || this.moved || !this.live || !this.room)
|
|
1059
|
+
return;
|
|
1060
|
+
const room = this.room;
|
|
1061
|
+
let out;
|
|
1062
|
+
try {
|
|
1063
|
+
out = room.beginFlush();
|
|
1064
|
+
}
|
|
1065
|
+
catch (e) {
|
|
1066
|
+
this.violation(`beginFlush failed: ${String(e)}`);
|
|
1067
|
+
return;
|
|
1068
|
+
}
|
|
1069
|
+
if (!out)
|
|
1070
|
+
return; // nothing dirty, nothing unflushed
|
|
1071
|
+
const { changes, batchHash } = JSON.parse(out);
|
|
1072
|
+
const seq = this.flushSeq++;
|
|
1073
|
+
// Composed ONCE; journaled and sent as this exact string forever (§5.3 step 4).
|
|
1074
|
+
const body = JSON.stringify({
|
|
1075
|
+
source: `room:${this.opts.downstream.docId}:${this.placementEpoch}`,
|
|
1076
|
+
offset: padOffset(seq),
|
|
1077
|
+
doc: this.opts.downstream.docId,
|
|
1078
|
+
epoch: this.placementEpoch,
|
|
1079
|
+
batchHash,
|
|
1080
|
+
cas: true,
|
|
1081
|
+
changes,
|
|
1082
|
+
});
|
|
1083
|
+
const journal = this.journal;
|
|
1084
|
+
const bootAtFlush = this.incarnationBootId;
|
|
1085
|
+
this.flushBusy = true;
|
|
1086
|
+
this.flushChain = (async () => {
|
|
1087
|
+
try {
|
|
1088
|
+
await journal.appendFlush({ seq, epoch: this.placementEpoch, body });
|
|
1089
|
+
}
|
|
1090
|
+
catch (e) {
|
|
1091
|
+
// An unjournaled batch must never reach the wire: a retry could rebuild
|
|
1092
|
+
// different bytes under the same id (§8.3). Loud incarnation death.
|
|
1093
|
+
this.flushBusy = false;
|
|
1094
|
+
if (this.incarnationBootId === bootAtFlush && !this.closed) {
|
|
1095
|
+
this.violation(`flush journal append failed: ${String(e)}`);
|
|
1096
|
+
}
|
|
1097
|
+
return;
|
|
1098
|
+
}
|
|
1099
|
+
let res;
|
|
1100
|
+
try {
|
|
1101
|
+
res = await withNetRetry(() => authority.applyRowChangeTxn(body), () => this.closed || this.moved, this.log);
|
|
1102
|
+
}
|
|
1103
|
+
catch (e) {
|
|
1104
|
+
this.flushBusy = false;
|
|
1105
|
+
if (this.closed || this.moved)
|
|
1106
|
+
return;
|
|
1107
|
+
// Fatal-class apply error (identity mismatch): our bug, never retried.
|
|
1108
|
+
if (this.incarnationBootId === bootAtFlush) {
|
|
1109
|
+
this.violation(`flush apply failed fatally: ${String(e)}`);
|
|
1110
|
+
}
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
1113
|
+
this.flushBusy = false;
|
|
1114
|
+
const sameIncarnation = this.incarnationBootId === bootAtFlush && this.room !== null;
|
|
1115
|
+
if (res.kind === "ok") {
|
|
1116
|
+
this.flushesConfirmed += 1;
|
|
1117
|
+
this.lastCommittedFlushSeq = Math.max(this.lastCommittedFlushSeq, seq); // §4.2 fence input
|
|
1118
|
+
await journal.confirmFlush(seq);
|
|
1119
|
+
if (!sameIncarnation) {
|
|
1120
|
+
// Settled; the new incarnation replayed its own state — let it flush.
|
|
1121
|
+
this.scheduleFlush();
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
try {
|
|
1125
|
+
this.room.flushOk();
|
|
1126
|
+
}
|
|
1127
|
+
catch (e) {
|
|
1128
|
+
this.violation(`flushOk failed: ${String(e)}`);
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
this.scheduleFlush(); // in-flight re-dirties / new lmids
|
|
1132
|
+
}
|
|
1133
|
+
else if (res.kind === "conflict") {
|
|
1134
|
+
await journal.confirmFlush(seq); // nothing applied; the retry re-derives
|
|
1135
|
+
if (!sameIncarnation) {
|
|
1136
|
+
this.scheduleFlush();
|
|
1137
|
+
return;
|
|
1138
|
+
}
|
|
1139
|
+
try {
|
|
1140
|
+
const out = JSON.parse(this.room.flushConflict(JSON.stringify(res.conflicts)));
|
|
1141
|
+
if (out.headCv !== null) {
|
|
1142
|
+
this.drainDownstream(); // the corrective frames (§5.4: ordinary edits)
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
catch (e) {
|
|
1146
|
+
this.violation(`flushConflict failed: ${String(e)}`);
|
|
1147
|
+
return;
|
|
1148
|
+
}
|
|
1149
|
+
this.log(`flush ${seq} CAS-conflicted: converged to the authority, retrying the rest`);
|
|
1150
|
+
this.scheduleFlush();
|
|
1151
|
+
}
|
|
1152
|
+
else {
|
|
1153
|
+
// Fenced under OUR epoch: this room is superseded (§2.5) — terminal.
|
|
1154
|
+
await journal.confirmFlush(seq);
|
|
1155
|
+
this.roomMoved(`flush ${seq} fenced (authority epoch ${res.currentEpoch ?? "?"})`);
|
|
1156
|
+
}
|
|
1157
|
+
})();
|
|
1158
|
+
}
|
|
1159
|
+
/** §2.5 stale-room behavior: the authority fenced us — another placement owns the
|
|
1160
|
+
* doc. Final state discarded, downstream closed with `room_moved`, nothing
|
|
1161
|
+
* reconnects. Clients re-open through the API server onto the current epoch. */
|
|
1162
|
+
roomMoved(reason) {
|
|
1163
|
+
if (this.moved)
|
|
1164
|
+
return;
|
|
1165
|
+
this.moved = true;
|
|
1166
|
+
this.log(`room moved: ${reason}`);
|
|
1167
|
+
for (const meta of this.subs.values()) {
|
|
1168
|
+
meta.ws.close(4009, "room_moved");
|
|
1169
|
+
}
|
|
1170
|
+
void this.close();
|
|
1171
|
+
}
|
|
1172
|
+
// ----------------------------- downstream leg -----------------------------
|
|
1173
|
+
serveDownstream(ws) {
|
|
1174
|
+
const conn = {
|
|
1175
|
+
id: this.nextConnId++,
|
|
1176
|
+
ws,
|
|
1177
|
+
queries: new Map(),
|
|
1178
|
+
busy: Promise.resolve(),
|
|
1179
|
+
clientID: null,
|
|
1180
|
+
sub: null,
|
|
1181
|
+
};
|
|
1182
|
+
this.conns.add(conn);
|
|
1183
|
+
ws.on("message", (data) => {
|
|
1184
|
+
let msg;
|
|
1185
|
+
try {
|
|
1186
|
+
msg = JSON.parse(String(data));
|
|
1187
|
+
}
|
|
1188
|
+
catch {
|
|
1189
|
+
return;
|
|
1190
|
+
}
|
|
1191
|
+
// Serialize per connection: subscribe verification is async, and a client's
|
|
1192
|
+
// subscribe → unsubscribe order must hold. A per-message throw is isolated to
|
|
1193
|
+
// THIS connection (the reference server's #12).
|
|
1194
|
+
conn.busy = conn.busy.then(async () => {
|
|
1195
|
+
try {
|
|
1196
|
+
await this.handleDownstreamMsg(ws, conn, msg);
|
|
1197
|
+
}
|
|
1198
|
+
catch (err) {
|
|
1199
|
+
send(ws, {
|
|
1200
|
+
t: "error",
|
|
1201
|
+
queryId: msg.queryId,
|
|
1202
|
+
message: String(err?.message ?? err),
|
|
1203
|
+
});
|
|
1204
|
+
}
|
|
1205
|
+
});
|
|
1206
|
+
});
|
|
1207
|
+
ws.on("close", () => {
|
|
1208
|
+
const now = Date.now();
|
|
1209
|
+
for (const q of conn.queries.values()) {
|
|
1210
|
+
if (this.subs.get(q.subKey)?.ws === ws) {
|
|
1211
|
+
this.subs.delete(q.subKey);
|
|
1212
|
+
this.room?.unsubscribe(q.subKey, now);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
conn.queries.clear();
|
|
1216
|
+
this.conns.delete(conn);
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
async handleDownstreamMsg(ws, conn, msg) {
|
|
1220
|
+
switch (msg.t) {
|
|
1221
|
+
case "init": {
|
|
1222
|
+
// The connection identity: what the lmid subscribe and pushMutation key on.
|
|
1223
|
+
// No reply frame — exactly rindled's wire.
|
|
1224
|
+
if (typeof msg.clientID === "string" && msg.clientID.length > 0) {
|
|
1225
|
+
conn.clientID = msg.clientID;
|
|
1226
|
+
}
|
|
1227
|
+
break;
|
|
1228
|
+
}
|
|
1229
|
+
case "subscribe": {
|
|
1230
|
+
if (typeof msg.queryId !== "number")
|
|
1231
|
+
return;
|
|
1232
|
+
if (!this.live || !this.room) {
|
|
1233
|
+
// Not live yet: hold the subscribe until the seq-0 snapshot lands, so a
|
|
1234
|
+
// booting room doesn't refuse its first clients.
|
|
1235
|
+
this.pendingSubs.push({ ws, conn, msg: msg });
|
|
1236
|
+
return;
|
|
1237
|
+
}
|
|
1238
|
+
await this.handleSubscribe(ws, conn, msg);
|
|
1239
|
+
break;
|
|
1240
|
+
}
|
|
1241
|
+
case "unsubscribe": {
|
|
1242
|
+
if (typeof msg.queryId !== "number")
|
|
1243
|
+
return;
|
|
1244
|
+
const prev = conn.queries.get(msg.queryId);
|
|
1245
|
+
if (prev) {
|
|
1246
|
+
conn.queries.delete(msg.queryId);
|
|
1247
|
+
this.subs.delete(prev.subKey);
|
|
1248
|
+
this.room?.unsubscribe(prev.subKey, Date.now());
|
|
1249
|
+
}
|
|
1250
|
+
break;
|
|
1251
|
+
}
|
|
1252
|
+
case "pushMutation": {
|
|
1253
|
+
const e = msg.envelope;
|
|
1254
|
+
if (e === null ||
|
|
1255
|
+
typeof e !== "object" ||
|
|
1256
|
+
typeof e.clientID !== "string" ||
|
|
1257
|
+
typeof e.mid !== "number" ||
|
|
1258
|
+
!Number.isFinite(e.mid) ||
|
|
1259
|
+
typeof e.name !== "string") {
|
|
1260
|
+
send(ws, { t: "error", message: "malformed pushMutation envelope" });
|
|
1261
|
+
break;
|
|
1262
|
+
}
|
|
1263
|
+
this.handlePushMutation(ws, conn, {
|
|
1264
|
+
clientID: e.clientID,
|
|
1265
|
+
mid: e.mid,
|
|
1266
|
+
name: e.name,
|
|
1267
|
+
args: e.args,
|
|
1268
|
+
});
|
|
1269
|
+
break;
|
|
1270
|
+
}
|
|
1271
|
+
case "mutate":
|
|
1272
|
+
// Raw CRUD never crosses this wire: room writes are named mutators (§4.2),
|
|
1273
|
+
// validated against the owned set at the transaction boundary.
|
|
1274
|
+
send(ws, {
|
|
1275
|
+
t: "error",
|
|
1276
|
+
queryId: msg.queryId,
|
|
1277
|
+
message: "this room accepts named mutators only (pushMutation)",
|
|
1278
|
+
});
|
|
1279
|
+
break;
|
|
1280
|
+
default:
|
|
1281
|
+
break;
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
async handleSubscribe(ws, conn, msg) {
|
|
1285
|
+
if (!this.room)
|
|
1286
|
+
return; // raced an incarnation death; the socket is being closed
|
|
1287
|
+
const queryError = (message) => send(ws, { t: "queryError", queryId: msg.queryId, message });
|
|
1288
|
+
// The reserved lmid system query arrives BY NAME even in lease mode — the write
|
|
1289
|
+
// plane's confirmation stream. The room composes the AST itself from the
|
|
1290
|
+
// connection's `init` identity (client args are ignored, exactly as on rindled).
|
|
1291
|
+
if (typeof msg.name === "string") {
|
|
1292
|
+
if (msg.name !== LMID_QUERY_NAME) {
|
|
1293
|
+
queryError("subscribe requires a lease token");
|
|
1294
|
+
return;
|
|
1295
|
+
}
|
|
1296
|
+
if (!this.opts.downstream.writes) {
|
|
1297
|
+
queryError("this room is read-only (no write plane configured)");
|
|
1298
|
+
return;
|
|
1299
|
+
}
|
|
1300
|
+
if (!conn.clientID) {
|
|
1301
|
+
queryError("subscribe lmid query before init");
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1304
|
+
const prev = conn.queries.get(msg.queryId);
|
|
1305
|
+
const epoch = prev ? prev.epoch + 1 : 1;
|
|
1306
|
+
const subKey = `${conn.id}:${msg.queryId}`;
|
|
1307
|
+
if (prev)
|
|
1308
|
+
this.subs.delete(prev.subKey);
|
|
1309
|
+
let res;
|
|
1310
|
+
try {
|
|
1311
|
+
res = JSON.parse(this.room.lmidSubscribe(subKey, epoch, conn.clientID, Date.now()));
|
|
1312
|
+
}
|
|
1313
|
+
catch (e) {
|
|
1314
|
+
queryError(`materialize failed: ${String(e?.message ?? e)}`);
|
|
1315
|
+
return;
|
|
1316
|
+
}
|
|
1317
|
+
conn.queries.set(msg.queryId, { subKey, epoch });
|
|
1318
|
+
this.subs.set(subKey, {
|
|
1319
|
+
ws,
|
|
1320
|
+
conn,
|
|
1321
|
+
clientQid: msg.queryId,
|
|
1322
|
+
// Not lease-gated: it lives exactly as long as its connection — never
|
|
1323
|
+
// swept by exp, never a revocation key (revoking a user closes the socket).
|
|
1324
|
+
user: "",
|
|
1325
|
+
exp: Number.POSITIVE_INFINITY,
|
|
1326
|
+
});
|
|
1327
|
+
send(ws, {
|
|
1328
|
+
t: "nhello",
|
|
1329
|
+
queryId: msg.queryId,
|
|
1330
|
+
hello: res.hello,
|
|
1331
|
+
bootId: this.incarnationBootId,
|
|
1332
|
+
});
|
|
1333
|
+
send(ws, { t: "nbatch", queryId: msg.queryId, batch: res.snapshot });
|
|
1334
|
+
this.sendProgress(conn);
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
if (typeof msg.leaseToken !== "string") {
|
|
1338
|
+
queryError("subscribe requires a lease token");
|
|
1339
|
+
return;
|
|
1340
|
+
}
|
|
1341
|
+
// The §10.1 gate: signature, doc, expiry — then the §4.1 revocation check.
|
|
1342
|
+
const now = Date.now();
|
|
1343
|
+
let payload;
|
|
1344
|
+
try {
|
|
1345
|
+
payload = await verifyRoomToken(msg.leaseToken, {
|
|
1346
|
+
doc: this.opts.downstream.docId,
|
|
1347
|
+
keys: this.opts.downstream.tokenKeys,
|
|
1348
|
+
now,
|
|
1349
|
+
});
|
|
1350
|
+
}
|
|
1351
|
+
catch (e) {
|
|
1352
|
+
queryError(e instanceof RoomTokenError ? e.message : "lease token refused");
|
|
1353
|
+
return;
|
|
1354
|
+
}
|
|
1355
|
+
const revokedAt = this.revoked.get(payload.sub);
|
|
1356
|
+
if (revokedAt !== undefined && payload.iat <= revokedAt) {
|
|
1357
|
+
queryError("lease token refused: revoked");
|
|
1358
|
+
return;
|
|
1359
|
+
}
|
|
1360
|
+
// Scope-skew tripwire: a lease proving against scopes this room's gate did NOT arm
|
|
1361
|
+
// with means a profile was edited after this room booted (the gate arms once). The
|
|
1362
|
+
// subscribe is NOT refused — the gate is still sound, it will just deopt routed
|
|
1363
|
+
// writes until the room re-boots. Log once per distinct skew so the otherwise-silent
|
|
1364
|
+
// deopt loop is diagnosable. Both hashes present required (a pre-stamp token or v1
|
|
1365
|
+
// gate skips).
|
|
1366
|
+
if (this.armedScopesHash !== undefined &&
|
|
1367
|
+
payload.scopesHash !== undefined &&
|
|
1368
|
+
payload.scopesHash !== this.armedScopesHash) {
|
|
1369
|
+
const key = `${payload.scopesHash}${this.armedScopesHash}`;
|
|
1370
|
+
if (!this.loggedScopeSkew.has(key)) {
|
|
1371
|
+
this.loggedScopeSkew.add(key);
|
|
1372
|
+
this.log(`scope skew: this room's gate armed with scopes ${this.armedScopesHash} but a lease ` +
|
|
1373
|
+
`proves against ${payload.scopesHash} — a room profile was edited under this live ` +
|
|
1374
|
+
`room. Routed writes will deopt (safe, but degraded) until the room re-boots.`);
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
// Subject binding (managed-writes §3.1): the FIRST verified lease binds the
|
|
1378
|
+
// connection's authenticated subject; a later lease whose `sub` differs is refused
|
|
1379
|
+
// — one principal per connection, so "the connection's user" is well-defined and
|
|
1380
|
+
// privilege-mixing is closed.
|
|
1381
|
+
if (conn.sub === null) {
|
|
1382
|
+
conn.sub = payload.sub;
|
|
1383
|
+
}
|
|
1384
|
+
else if (conn.sub !== payload.sub) {
|
|
1385
|
+
queryError("lease token refused: connection is bound to another subject");
|
|
1386
|
+
return;
|
|
1387
|
+
}
|
|
1388
|
+
// A re-subscribe (gap recovery) replaces the prior envelope and bumps the
|
|
1389
|
+
// DOWNSTREAM epoch — the same rule rindled applies per (conn, queryId).
|
|
1390
|
+
const prev = conn.queries.get(msg.queryId);
|
|
1391
|
+
const epoch = prev ? prev.epoch + 1 : 1;
|
|
1392
|
+
const subKey = `${conn.id}:${msg.queryId}`;
|
|
1393
|
+
if (prev)
|
|
1394
|
+
this.subs.delete(prev.subKey);
|
|
1395
|
+
let res;
|
|
1396
|
+
try {
|
|
1397
|
+
res = JSON.parse(this.room.subscribe(subKey, epoch, JSON.stringify(payload.ast), now));
|
|
1398
|
+
}
|
|
1399
|
+
catch (e) {
|
|
1400
|
+
queryError(`materialize failed: ${String(e?.message ?? e)}`);
|
|
1401
|
+
return;
|
|
1402
|
+
}
|
|
1403
|
+
conn.queries.set(msg.queryId, { subKey, epoch });
|
|
1404
|
+
this.subs.set(subKey, {
|
|
1405
|
+
ws,
|
|
1406
|
+
conn,
|
|
1407
|
+
clientQid: msg.queryId,
|
|
1408
|
+
user: payload.sub,
|
|
1409
|
+
exp: payload.exp,
|
|
1410
|
+
});
|
|
1411
|
+
send(ws, {
|
|
1412
|
+
t: "nhello",
|
|
1413
|
+
queryId: msg.queryId,
|
|
1414
|
+
hello: res.hello,
|
|
1415
|
+
bootId: this.incarnationBootId,
|
|
1416
|
+
});
|
|
1417
|
+
send(ws, { t: "nbatch", queryId: msg.queryId, batch: res.snapshot });
|
|
1418
|
+
// The release point for the snapshot: an optimistic client buffers every nbatch
|
|
1419
|
+
// until a progress frame's cvMin covers its cv.
|
|
1420
|
+
this.sendProgress(conn);
|
|
1421
|
+
}
|
|
1422
|
+
/** The periodic enforcement tick: drop subscriptions whose lease passed `exp`
|
|
1423
|
+
* unrenewed (§4.1's TTL backstop), reclaim idle pipelines, prune old revocations. */
|
|
1424
|
+
sweep() {
|
|
1425
|
+
const now = Date.now();
|
|
1426
|
+
for (const [subKey, meta] of [...this.subs]) {
|
|
1427
|
+
if (meta.exp <= now) {
|
|
1428
|
+
send(meta.ws, {
|
|
1429
|
+
t: "queryError",
|
|
1430
|
+
queryId: meta.clientQid,
|
|
1431
|
+
message: "expired lease — renew through the API server",
|
|
1432
|
+
});
|
|
1433
|
+
this.dropSub(subKey, meta, now);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
if (this.room && this.live) {
|
|
1437
|
+
this.room.sweep(now);
|
|
1438
|
+
}
|
|
1439
|
+
const window = this.opts.downstream.revocationWindowMs ?? DEFAULT_REVOCATION_WINDOW_MS;
|
|
1440
|
+
for (const [user, at] of [...this.revoked]) {
|
|
1441
|
+
if (now - at > window)
|
|
1442
|
+
this.revoked.delete(user);
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
dropSub(subKey, meta, now) {
|
|
1446
|
+
this.subs.delete(subKey);
|
|
1447
|
+
meta.conn.queries.delete(meta.clientQid);
|
|
1448
|
+
this.room?.unsubscribe(subKey, now);
|
|
1449
|
+
}
|
|
1450
|
+
/** §4.1 layer 2: synchronous revocation. Terminates every subscription and socket of
|
|
1451
|
+
* `user` and refuses their pre-revocation tokens from here on. */
|
|
1452
|
+
revokeUser(user) {
|
|
1453
|
+
const now = Date.now();
|
|
1454
|
+
this.revoked.set(user, now);
|
|
1455
|
+
const sockets = new Set();
|
|
1456
|
+
let dropped = 0;
|
|
1457
|
+
for (const [subKey, meta] of [...this.subs]) {
|
|
1458
|
+
if (meta.user !== user)
|
|
1459
|
+
continue;
|
|
1460
|
+
send(meta.ws, {
|
|
1461
|
+
t: "queryError",
|
|
1462
|
+
queryId: meta.clientQid,
|
|
1463
|
+
message: "lease revoked",
|
|
1464
|
+
});
|
|
1465
|
+
this.dropSub(subKey, meta, now);
|
|
1466
|
+
sockets.add(meta.ws);
|
|
1467
|
+
dropped++;
|
|
1468
|
+
}
|
|
1469
|
+
// A conn BOUND to the subject is write-capable even with no live subscription
|
|
1470
|
+
// (managed-writes §3.1/§3.4) — its socket goes too.
|
|
1471
|
+
for (const conn of this.conns) {
|
|
1472
|
+
if (conn.sub === user)
|
|
1473
|
+
sockets.add(conn.ws);
|
|
1474
|
+
}
|
|
1475
|
+
for (const ws of sockets) {
|
|
1476
|
+
ws.close(1008, "revoked");
|
|
1477
|
+
}
|
|
1478
|
+
this.log(`revoked ${user}: ${dropped} subscription(s)`);
|
|
1479
|
+
return dropped;
|
|
1480
|
+
}
|
|
1481
|
+
// ----------------------------- control plane ------------------------------
|
|
1482
|
+
startControl(control) {
|
|
1483
|
+
const server = createServer((req, res) => {
|
|
1484
|
+
const reply = (status, body) => {
|
|
1485
|
+
res.writeHead(status, { "content-type": "application/json" });
|
|
1486
|
+
res.end(JSON.stringify(body));
|
|
1487
|
+
};
|
|
1488
|
+
if (req.headers.authorization !== `Bearer ${control.authToken}`) {
|
|
1489
|
+
return reply(401, { error: "unauthorized" });
|
|
1490
|
+
}
|
|
1491
|
+
if (req.method === "GET" && req.url === "/stats") {
|
|
1492
|
+
return reply(200, {
|
|
1493
|
+
live: this.live,
|
|
1494
|
+
connections: this.wss.clients.size,
|
|
1495
|
+
subscriptions: this.subs.size,
|
|
1496
|
+
materializations: this.room?.materializationCount() ?? 0,
|
|
1497
|
+
pendingMutations: this.room?.pendingLen() ?? 0,
|
|
1498
|
+
dirtyKeys: this.room?.dirtyLen() ?? 0,
|
|
1499
|
+
placementEpoch: this.placementEpoch,
|
|
1500
|
+
flushesConfirmed: this.flushesConfirmed,
|
|
1501
|
+
flushInFlight: this.flushBusy,
|
|
1502
|
+
});
|
|
1503
|
+
}
|
|
1504
|
+
if (req.method === "POST" && req.url === "/revoke") {
|
|
1505
|
+
let body = "";
|
|
1506
|
+
req.on("data", (c) => (body += c));
|
|
1507
|
+
req.on("end", () => {
|
|
1508
|
+
try {
|
|
1509
|
+
const { userId } = JSON.parse(body);
|
|
1510
|
+
if (typeof userId !== "string" || userId.length === 0) {
|
|
1511
|
+
return reply(400, { error: "userId required" });
|
|
1512
|
+
}
|
|
1513
|
+
reply(200, { revoked: this.revokeUser(userId) });
|
|
1514
|
+
}
|
|
1515
|
+
catch {
|
|
1516
|
+
reply(400, { error: "invalid JSON" });
|
|
1517
|
+
}
|
|
1518
|
+
});
|
|
1519
|
+
return;
|
|
1520
|
+
}
|
|
1521
|
+
if (req.method === "POST" && req.url === "/drain") {
|
|
1522
|
+
// §4.2 drain-before-downgrade: the api-server's `drainRoom` hook lands here. No body.
|
|
1523
|
+
this.drainForDowngrade().then((finalFlushSeq) => reply(200, { finalFlushSeq, drained: true }), (e) => reply(500, { error: String(e?.message ?? e) }));
|
|
1524
|
+
return;
|
|
1525
|
+
}
|
|
1526
|
+
reply(404, { error: "unknown endpoint" });
|
|
1527
|
+
});
|
|
1528
|
+
this.control = server;
|
|
1529
|
+
return new Promise((resolve) => {
|
|
1530
|
+
server.listen(control.port ?? 0, "127.0.0.1", () => resolve());
|
|
1531
|
+
});
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
/** A flush seq as its wire offset: zero-padded so `rindled`'s lexicographic keyset
|
|
1535
|
+
* compare (`run_already_applied`) orders it like the number it is. */
|
|
1536
|
+
function padOffset(seq) {
|
|
1537
|
+
return String(seq).padStart(20, "0");
|
|
1538
|
+
}
|
|
1539
|
+
/** Retry `op` on network-class failures (backoff, same inputs — for the flush that
|
|
1540
|
+
* means the same journaled bytes) until it settles, `stop()` says quit, or the error
|
|
1541
|
+
* is fatal (`{fatal: true}` — retrying cannot help). */
|
|
1542
|
+
async function withNetRetry(op, stop, log) {
|
|
1543
|
+
let delay = FLUSH_RETRY_MIN_MS;
|
|
1544
|
+
for (;;) {
|
|
1545
|
+
try {
|
|
1546
|
+
return await op();
|
|
1547
|
+
}
|
|
1548
|
+
catch (e) {
|
|
1549
|
+
if (e?.fatal === true || stop())
|
|
1550
|
+
throw e;
|
|
1551
|
+
log(`authority call failed (retrying in ${delay}ms): ${String(e)}`);
|
|
1552
|
+
await new Promise((r) => {
|
|
1553
|
+
const t = setTimeout(r, delay);
|
|
1554
|
+
t.unref?.();
|
|
1555
|
+
});
|
|
1556
|
+
if (stop())
|
|
1557
|
+
throw e;
|
|
1558
|
+
delay = Math.min(delay * 2, FLUSH_RETRY_MAX_MS);
|
|
1559
|
+
}
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
/** Positional table shapes from the upstream hello — the keyed MutationTx layer's
|
|
1563
|
+
* schema. (The hello was already validated by `WasmRoom.open` before this runs.) */
|
|
1564
|
+
function shapesOf(hello) {
|
|
1565
|
+
const shapes = new Map();
|
|
1566
|
+
const tables = hello?.tables;
|
|
1567
|
+
if (Array.isArray(tables)) {
|
|
1568
|
+
for (const t of tables) {
|
|
1569
|
+
shapes.set(t.name, { columns: t.columns, primaryKey: t.primaryKey });
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
return shapes;
|
|
1573
|
+
}
|
|
1574
|
+
/** Boot a room shell: init the wasm, mint the upstream lease, connect the upstream leg,
|
|
1575
|
+
* and serve the downstream ws (+ the private control plane, if configured). Returns
|
|
1576
|
+
* once the ports are bound and the upstream connection is underway — await
|
|
1577
|
+
* `shell.awaitLive()` for the seed. */
|
|
1578
|
+
export async function createRoomShell(opts) {
|
|
1579
|
+
const shell = new Shell(opts);
|
|
1580
|
+
try {
|
|
1581
|
+
await shell.start();
|
|
1582
|
+
}
|
|
1583
|
+
catch (e) {
|
|
1584
|
+
await shell.close();
|
|
1585
|
+
throw e;
|
|
1586
|
+
}
|
|
1587
|
+
return shell;
|
|
1588
|
+
}
|
|
1589
|
+
//# sourceMappingURL=shell.js.map
|