@rindle/optimistic 0.4.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/backend.ts CHANGED
@@ -48,6 +48,7 @@ import {
48
48
  LMID_QUERY_NAME,
49
49
  localTableNames,
50
50
  normalizedTableSchemas,
51
+ rowsEqual,
51
52
  tableSpec,
52
53
  toCell,
53
54
  } from "@rindle/client";
@@ -58,11 +59,14 @@ import type {
58
59
  ChangeEvent,
59
60
  ColsMap,
60
61
  ColType,
62
+ Condition,
61
63
  IsoTx,
62
64
  KeyedRow,
63
65
  Mutation,
66
+ MutationEnvelope,
64
67
  MutationGen,
65
68
  MutationOp,
69
+ MutationOutcomeFrame,
66
70
  MutatorCtx,
67
71
  NormalizedEvent,
68
72
  NormalizedOp,
@@ -78,9 +82,21 @@ import type {
78
82
  WireValue,
79
83
  } from "@rindle/client";
80
84
  import { aggTableSchemas, NormalizedSync, rewriteAggregates, type ColCounts, type PkCols } from "@rindle/normalized";
81
- import { WasmBackend, type ServerDeltaOp, type WasmWriteTxn } from "@rindle/wasm";
85
+ import { WasmBackend, type ServerDeltaOp, type WasmWriteTxn, type WritableDescriptor } from "@rindle/wasm";
82
86
 
83
87
  import { AggOverlay, type ChildOp, collectAggDefs } from "./agg-overlay.ts";
88
+ import {
89
+ decodeOutcomeRow,
90
+ LIFECYCLE_TABLE_SCHEMAS,
91
+ ROOM_CLIENT_MUTATIONS_TABLE,
92
+ ROOM_MUTATION_OUTCOMES_TABLE,
93
+ ROOM_WATERMARK_TABLE,
94
+ roomDomainKey,
95
+ SCOPE_SESSIONS_TABLE,
96
+ type SystemStreamSpec,
97
+ } from "./system-streams.ts";
98
+
99
+ export type { SystemStreamSpec, SystemStreamTable } from "./system-streams.ts";
84
100
 
85
101
  /** A keyed row: column name → cell. The ergonomic shape — column names are validated against the
86
102
  * schema at runtime, so a typo throws immediately with the valid names. Re-exported from
@@ -149,16 +165,242 @@ export type { ResultType };
149
165
  * are assigned by the `Store` starting at 1, so 0 never collides. */
150
166
  const LMID_QID: QueryId = 0;
151
167
 
168
+ /** One captured write, pk-granular (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §3.2 #1: "the
169
+ * writers already receive `(table, row)`... this is pure capture, no semantic change"). `row` is
170
+ * the post-write image (positional wire cells, schema column order); `undefined` for a `remove` —
171
+ * no row survives it, and `row === undefined` stays the remove marker.
172
+ *
173
+ * `oldRow` is the PRE-IMAGE, captured for a `remove` (G-iii, §7.3 tombstone) AND — since H-ii —
174
+ * an `edit` (it feeds H-iii's join-key no-change rule, old vs new join-key cells, and mirrors the
175
+ * engine, which routes an Edit by its OLD row — the H-i handoff): the full-width row read via
176
+ * `tx.get` immediately BEFORE the write staged — read-your-writes, so a write to the same pk
177
+ * earlier in the SAME invocation shows through — falling back to the caller's asserted old row
178
+ * when the pk is not txn-visible (a raw remove/edit of an absent row; incl. the pk-MOVING raw
179
+ * edit, whose record is keyed by the NEW pk yet whose pre-image is the caller's OLD row — the row
180
+ * H-i's `provenanceOf` routes by). A captured remove/edit thus always carries a full-width
181
+ * pre-image (the engine width-checks `holdBackAbsent`; Slice H evaluates writable predicates on
182
+ * it) — with ONE exception: a record that collapses to a (re-)insert has none, see the matrix.
183
+ *
184
+ * Coalescing within one invocation is last-write-wins per pk on `row` (the final image matches
185
+ * the engine head's own semantics for that pk) with `oldRow` pinned to the TXN-ENTRY BASE — the
186
+ * H-ii matrix:
187
+ * - edit-after-add / edit-after-remove: the record collapses to a (re-)insert — post-image
188
+ * only, NO `oldRow` (the pk did not pre-exist this invocation's base; presence hold-back
189
+ * uses `row`).
190
+ * - edit-after-edit: keeps the FIRST pre-image (the txn-entry base — the chain nets to ONE
191
+ * edit from the base to the final image).
192
+ * - remove-after-edit / remove-after-remove: keeps the ORIGINAL pre-image (the first write's
193
+ * captured base), NOT the edited transient — the net effect is a remove of the row the
194
+ * external world last knew.
195
+ * - remove-after-add: keeps the txn-visible pre-image (the transient added row — the pk had
196
+ * no base, and this is the only truthful full-width row there is; G-iii pinned it).
197
+ * - add-after-remove (a re-insert): drops `oldRow` (presence hold-back uses `row`).
198
+ * Across rebase re-invocations the write-set is union-never-shrink ({@link mergeWriteSet}): a
199
+ * re-run that no-ops keeps the prior record — and its pre-image — intact. */
200
+ export interface WriteRecord {
201
+ table: string;
202
+ pk: WireValue[];
203
+ row: WireValue[] | undefined;
204
+ oldRow?: WireValue[];
205
+ }
206
+
207
+ /** The pk-granular write-set captured over ONE mutator invocation: table → pk-key (a stable-JSON
208
+ * encoding of the pk cells, {@link stableJson}) → that pk's write, LAST-WRITE-WINS within the
209
+ * invocation — an add-then-edit or edit-then-edit of the SAME pk collapses to its final image,
210
+ * matching the engine head's own semantics for that pk. Chosen (over a flat array) because the
211
+ * later routing proof needs "is pk P in the writable scope" / "did we already see this pk in this
212
+ * invocation" as cheap lookups, and rebase re-invocation needs to MERGE a fresh write-set into an
213
+ * accumulated one ({@link mergeWriteSet}) — both are Map operations, not scans.
214
+ *
215
+ * `touched` (the pre-existing table-granular `Set<string>` the pending axis reads, §7.2) is
216
+ * exactly `new Set(writeSet.keys())` — derived from this, never separately populated, so the two
217
+ * can never drift. */
218
+ export type WriteSet = Map<string, Map<string, WriteRecord>>;
219
+
220
+ /** Whether a recorded point read (`tx.get`/`tx.row`) found a row. */
221
+ export type ReadOutcome = "present" | "absent";
222
+
223
+ /** One recorded point read, pk-granular (§3.2 #2). Recording-mode only — see {@link ReadLog}.
224
+ * Since H-ii this covers BOTH the public reads (`tx.get`/`tx.row`) and the keyed writers'
225
+ * internal pre-existence probes (§3.2 #3 — see the `rawGet` note in {@link trackingTx}). */
226
+ export interface ReadRecord {
227
+ table: string;
228
+ pk: WireValue[];
229
+ outcome: ReadOutcome;
230
+ /** Per-read provenance (H-ii, §3.2 #3): the engine's `provenanceOf` answer AT THE MOMENT OF THE
231
+ * READ — which source ("daemon" / a room key) served the row the mutator saw. Live pre-commit
232
+ * state: an open txn's staged writes are not consulted (H-i), so a PRESENT read of a pk this
233
+ * txn itself created reports `undefined` — recorded honestly, key omitted (the router treats
234
+ * self-reads via write-set membership, not provenance). ABSENT reads carry no source (the
235
+ * mutator saw no row; there is no full-width row to probe with — a pk this txn staged-removed
236
+ * thus also records `undefined`, again a write-set-membership case for the router). Probed only
237
+ * on a PROMOTED table ({@link promotedTables}): a never-promoted (Collapsed) table is
238
+ * daemon-owned by construction, so a pure single-domain app pays zero wasm calls per read and
239
+ * `undefined` is still correct (everything is daemon-owned). One probe call per recorded
240
+ * present read, at read time. */
241
+ source?: string;
242
+ }
243
+
244
+ /** The read-log captured over ONE mutator invocation when recording is armed (RINDLE-REALTIME-
245
+ * QUERY-ENABLEMENT-DESIGN.md §3.2 #2): every point read (`reads` — the public `tx.get`/`tx.row`
246
+ * and, since H-ii, the keyed writers' internal pre-existence probes) plus every resolved query AST
247
+ * (`queries`, from `tx.query`). A SIBLING of the folded read TRAP (`FoldReadError` below) — the
248
+ * trap arms on the folded path and throws before any read completes (recording never runs there);
249
+ * recording arms on the ordinary (non-folded) prediction run and never throws. Pure capture: which
250
+ * row/predicate a read is actually PROVEN covered by (§3.1's table) is a later slice's job. */
251
+ export interface ReadLog {
252
+ reads: ReadRecord[];
253
+ queries: Ast[];
254
+ }
255
+
256
+ /** One promoted `(sourceKey, table)`'s client-held routing spec — THE per-table routing record the
257
+ * §3 prove-or-slow-path router reads (H-iii). Recorded by {@link OptimisticBackend.promoteRoomTable}
258
+ * in the same breath as the engine attach, from the lease's `RealtimeLeaseTableSpec` (the
259
+ * api-server's `RoomTableSpec`, one compiler with the boot wire).
260
+ *
261
+ * - `writable`: whether the room may write this table AT ALL — `true` iff the registered engine
262
+ * {@link WritableDescriptor} is not `none` (a context/"followed" table is `none`; the daemon
263
+ * stays write-authoritative for it, §2.2). Rule #1 of the write proof.
264
+ * - `joinKeyCols`: the correlation columns the footprint binds on this table — the "room mutators
265
+ * never change join keys" input (an edit with a moved join-key cell would re-parent the row
266
+ * across the footprint's correlation and is refused room-routing). Deliberately client-side —
267
+ * they never cross the wasm {@link WritableDescriptor} ABI (the room GATE enforces them
268
+ * engine-side, H-iv).
269
+ * - `where`: the lease's row-local writable predicate (advisory here — the router evaluates the
270
+ * ENGINE's compiled copy via `writableMatches`, never this one; kept for introspection parity
271
+ * with the wire).
272
+ * - `footprintWhere`: the EXACT footprint-membership predicate (H-iv-b: root-node-only, lossless
273
+ * extraction; the vacuous-true `{type:"and",conditions:[]}` for an exact unconstrained root;
274
+ * ABSENT for child/correlated tables). The read proof's pk-membership test input — see
275
+ * {@link OptimisticBackend.deriveDomain}. */
276
+ export interface RoomTableRouting {
277
+ writable: boolean;
278
+ joinKeyCols: string[];
279
+ where?: Condition;
280
+ footprintWhere?: Condition;
281
+ }
282
+
283
+ /** The per-table spec {@link OptimisticBackend.promoteRoomTable} receives beside the engine
284
+ * descriptor (H-iii spec plumbing) — the wire-facing half of {@link RoomTableRouting}
285
+ * (`writable` is derived from the descriptor's kind, never passed separately). */
286
+ export interface RoomTableRoutingSpec {
287
+ joinKeyCols?: readonly string[];
288
+ where?: Condition;
289
+ footprintWhere?: Condition;
290
+ }
291
+
292
+ /** The §3 router's Q6 measurement hook ({@link OptimisticBackend.__inspectDomains}`().routing`):
293
+ * how often derivation succeeded (per derived room) and why it fell back to the daemon (per
294
+ * failure reason). `reasons` counts PER-CANDIDATE proof failures plus the terminal reasons
295
+ * (`no-candidates` / `tx-query` / `ambiguous`) — with multiple candidate rooms one derivation can
296
+ * bump several reason counters (one per failing candidate), and a candidate's failure is counted
297
+ * even when a sibling candidate ultimately proved. Explicit-policy pins bump NOTHING (the router
298
+ * never ran). */
299
+ export interface RoutingInspect {
300
+ /** Successful DERIVED routes, per room key. */
301
+ derived: Record<string, number>;
302
+ /** Derivation failures, per {@link RoutingFailureReason}. */
303
+ reasons: Record<string, number>;
304
+ }
305
+
306
+ /** Why a §3 derivation (or one candidate's proof) fell to the daemon — the {@link RoutingInspect}
307
+ * counter keys. */
308
+ export type RoutingFailureReason =
309
+ /** No connected room gate with a promoted table — the single-domain fast path. */
310
+ | "no-candidates"
311
+ /** The invocation ran `tx.query` — a declarative read is not room-provable client-side
312
+ * (rindle-cover is native-only BY DESIGN), so the whole derivation fails unconditionally. */
313
+ | "tx-query"
314
+ /** Two or more candidate rooms both proved. Principled disambiguation is a §9.2 (multi-room
315
+ * clients + budgets) NON-goal, deferred past Slice J; routing slow is always sound (the §3.3
316
+ * gate is the contract). Reachable essentially only for zero-write mutations — real writes
317
+ * can't be writable in two rooms (§2.2 exactly-one-writer per doc). */
318
+ | "ambiguous"
319
+ /** A write touched a table not promoted for the candidate, or promoted `writable: none`
320
+ * (context) — the room may not write it at all. */
321
+ | "write-unwritable-table"
322
+ /** `writableMatches` refused the write's post-image (add/edit) or pre-image (remove). */
323
+ | "write-scope-miss"
324
+ /** An edit moved a join-key cell (old vs new differ on a `joinKeyCols` column). */
325
+ | "write-join-key-change"
326
+ /** The engine's provenance for the write's probe row names a DIFFERENT room. */
327
+ | "write-cross-room-provenance"
328
+ /** A recorded read was served by a DIFFERENT room. */
329
+ | "read-cross-room"
330
+ /** A read needed the pk-membership test but the table has no `footprintWhere` (child /
331
+ * correlated / inexact extraction / never-promoted table). */
332
+ | "read-no-footprint-where"
333
+ /** `footprintWhere` references a non-pk column — not decidable from the key alone. */
334
+ | "read-not-key-decidable"
335
+ /** `footprintWhere` is key-decidable but outside the deliberately-minimal client evaluator
336
+ * (an op other than `=`/`!=`, a null/type-mixed comparison, column-vs-column, …). */
337
+ | "read-not-evaluable"
338
+ /** `footprintWhere` evaluated FALSE on the read's pk — the room provably never holds it. */
339
+ | "read-outside-footprint"
340
+ /** NOT a derivation failure (H-v): a routed mutation came back `mutationOutcome
341
+ * {kind:"deopt"}` — the room GATE refused it at commit and the client re-enqueued it onto the
342
+ * daemon. Counted per processed deopt frame so Q6's picture is complete: derived-and-deopted
343
+ * routes are visible beside derived successes (a rising `deopt` count against a rising
344
+ * `derived` count means the client proof and the gate disagree — a compiler/evaluator skew
345
+ * worth investigating; each one costs a room round-trip + a burnt room mid, never
346
+ * correctness). */
347
+ | "deopt";
348
+
152
349
  interface PendingMutation {
153
350
  /** The wire mutation id. A FOLDED entry carries `null` until its window flushes — the `mid`
154
351
  * is dealt from `nextMid` in SEND order, never reserved at invoke, so the wire sequence stays
155
352
  * gapless under debounce (FOLDED-MUTATIONS-DESIGN §4.1). A `null` entry is never confirmable
156
- * (the confirm-drop retains it) and re-invokes AFTER every assigned mid (sorted by `mid ?? ∞`). */
353
+ * (the confirm-drop retains it) and re-invokes AFTER every assigned mid. */
157
354
  mid: number | null;
355
+ /** The client-global deal sequence, stamped in the same breath as {@link mid} (`null` while the
356
+ * mid is). Mids are PER-DOMAIN (each authority numbers its own confirms, §7.1), so mids from
357
+ * different domains are incomparable — a daemon mid 5 and a room mid 1 say nothing about which
358
+ * was sent first. `seq` is the ONE total order across domains: **confirmation order is
359
+ * per-domain; replay order is client-global** — the reconcile's re-invocation sort keys on
360
+ * `seq`, never on `mid`. Within one domain `seq` order equals `mid` order (both dealt at the
361
+ * same send-time choke point) EXCEPT across an H-v deopt re-enqueue: a flipped entry keeps its
362
+ * ORIGINAL seq while its fresh daemon mid is dealt later, so its seq may undercut daemon
363
+ * entries with smaller mids. That is the point — seq is the REPLAY order and the flip must not
364
+ * move the entry's overlay position (a read-dependent sibling invoked after it replays on its
365
+ * value); the only consumer of the ordering is the seq-keyed reconcile sort, which wants
366
+ * exactly this. Single-domain behavior without deopts is unchanged. */
367
+ seq: number | null;
158
368
  name: string;
159
369
  args: unknown;
160
- /** Tables this mutator touched at its LAST invocation (drives the pending axis, §7.2). */
370
+ /** The confirming stream (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §7.1): which domain's ledger
371
+ * dealt this entry's `mid` and whose confirm watermark can retire it. Resolved from the injectable
372
+ * `domainPolicy` when the mid is dealt (for a folded entry, re-resolved at flush). `"daemon"` in
373
+ * the single-domain configuration. An un-flushed fold (`mid == null`) carries its provisional
374
+ * domain but is never confirmable until the flush stamps a real mid. */
375
+ domain: string;
376
+ /** Tables this mutator touched at its LAST invocation (drives the pending axis, §7.2). Derived
377
+ * from `writes.keys()` — see {@link WriteSet}. */
161
378
  touched: Set<string>;
379
+ /** The pk-granular write-set captured at this entry's LAST invocation (§3.2 #1). A rebase
380
+ * re-invocation MERGES its fresh write-set into this one ({@link mergeWriteSet}), mirroring the
381
+ * `touched` union: the key set only grows across re-invocations (a re-run that no-ops must not
382
+ * shrink it, §7.2), each key's value is always the newest. */
383
+ writes: WriteSet;
384
+ /** The physical SOURCES this mutator's writes staged onto (RINDLE-REALTIME-QUERY-ENABLEMENT-
385
+ * DESIGN.md §5.3, from `commitTracked`) — `{"daemon"}` in single-domain. Drives the per-source
386
+ * reconcile FILTER: `runReconcileCycle(sourceKey)` re-invokes an entry only if it wrote onto
387
+ * `sourceKey` (its writes there were un-applied by that source's rewind; other-source writes are
388
+ * intact). Union-merged across re-invocations, never shrunk (like `touched`). DISTINCT from
389
+ * {@link domain} (which stream CONFIRMS it): a room-domain mutation whose write routed to the
390
+ * daemon slice (a fresh pk) has `domain === "room:doc:X"` but `touchedSources ∋ "daemon"`. */
391
+ touchedSources: Set<string>;
392
+ /** Per staged (table, pk) → the source it routed onto ({@link writeSourceKey}), aligned from
393
+ * `commitTracked`'s staged-order `sources[]` (last write to a pk wins). Read at the confirm-drop
394
+ * to fire the §7.3 echo hold-back for a write whose source ≠ its confirming {@link domain}.
395
+ * All `"daemon"` in single-domain (inert). Merged never-shrunk across re-invocations. */
396
+ writeSources: Map<string, string>;
397
+ /** The read-log captured at this entry's LAST *recorded* invocation (§3.2 #2). Empty for a
398
+ * FOLDED entry (the trap, not recording, arms on that path — nothing is ever recorded there)
399
+ * and left as the ORIGINAL invoke's log across a rebase re-invocation: recording is armed only
400
+ * on the initial `invoke`, not the reconcile replay (a re-invocation runs against a rebased
401
+ * base, so its read outcomes would need re-proving against the NEW base anyway — re-arming
402
+ * recording there is future work, not required for pure capture). */
403
+ reads: ReadLog;
162
404
  }
163
405
 
164
406
  /** A virtual-clock seam for the fold debounce/maxWait timers (FOLDED-MUTATIONS-DESIGN §9): the
@@ -179,6 +421,15 @@ export interface FoldOptions {
179
421
  /** Hard cap so a never-idle drag still persists periodically (trailing throttle). Unbounded if
180
422
  * omitted — an idle gap of `debounceMs` is then the only thing that flushes. */
181
423
  maxWaitMs?: number;
424
+ /** §9.3 room-aware cadence. When this fold's write ROUTES INTO A ROOM (a collaborator is live on
425
+ * the shared head), flush at this (short) interval instead of `debounceMs`/`maxWaitMs`, so the
426
+ * intermediate frames STREAM to the room rather than collapsing to last-value-wins — the pen is
427
+ * watched, so its growth matters. OFF the room (solo / daemon-served) this is ignored and the
428
+ * caller's `debounceMs` collapse governs. `0` ⇒ per-frame (never coalesce while in a room); a
429
+ * small value (e.g. 40ms) animates while still capping the write rate. Absent ⇒ same cadence
430
+ * room or not (today's behavior). The room decision is probed from the write-set at the fold
431
+ * window's first invoke; it stays fixed for that window. */
432
+ roomDebounceMs?: number;
182
433
  /** Keep deferring across overlapping non-fold writes for maximum economy, accepting the §4.2
183
434
  * read-dependent reorder snap. Default `false` (flush-on-enqueue — correct-and-boring). */
184
435
  deferAcrossWrites?: boolean;
@@ -223,6 +474,112 @@ interface BufferedFrame {
223
474
  seq: number;
224
475
  }
225
476
 
477
+ /** ONE authority channel's coherence gate (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §5.1): the
478
+ * per-source generalization of what used to be the backend's single `(buffer, appliedCv)` pair.
479
+ * Each connected source — the daemon always; a `room:doc:X` once Slice G wires the live feed —
480
+ * buffers its own cv-tagged frames and releases on its OWN `cvMin`, an independent cycle feeding
481
+ * the one `applyRelease`: **coherent within a source; eventual across sources** (no joint
482
+ * barrier, by design). `key` doubles as the lmid fold domain (§7.1) and the physical source the
483
+ * release rebases (§5.3). `sync` is this source's OWN refcount/baseline space — folding two
484
+ * sources through one `NormalizedSync` would refcount an overlapping row 1→2 and emit NOTHING
485
+ * for the second source, silently starving its per-source baseline (§5.2 "a real store: its own
486
+ * baseline"). */
487
+ interface SourceGate {
488
+ key: string;
489
+ source: OptimisticSource;
490
+ sync: NormalizedSync;
491
+ buffer: BufferedFrame[];
492
+ /** Arrival counter behind {@link BufferedFrame.seq}, scoped to THIS gate's buffer. */
493
+ nextSeq: number;
494
+ appliedCv: number;
495
+ /** The channel's latest UPSTREAM-absorption advert (§301 direction B, a ROOM gate only): "I
496
+ * have absorbed upstream daemon commits through cv `upstreamCv` of boot `upstreamBoot`" —
497
+ * recorded verbatim from each {@link ProgressFrame} ({@link OptimisticBackend.onGateProgress})
498
+ * so the drop pass that runs at that release's tail reads exactly this release's advert.
499
+ * `undefined` until the shell advertises (an old shell never does — the §2.5 skew fallback);
500
+ * cleared on a channel restart (the fresh incarnation re-advertises). */
501
+ upstreamBoot?: string;
502
+ upstreamCv?: number;
503
+ }
504
+
505
+ /** One I-iv doorbell event ({@link OptimisticBackend.onScopeSessions}, §4.1): a release folded
506
+ * scope-session rows for `scope`, and `others` is the count of OTHER clients' unexpired sessions
507
+ * there — {@link OptimisticBackend.otherScopeSessions} evaluated at fold time (the same one rule,
508
+ * on the injectable {@link FoldClock}, so a virtual-clock harness gets deterministic verdicts).
509
+ * The consumer (client.ts) triggers its one debounced re-lease on the 0→≥1 transition; expired
510
+ * and own-clientID rows never count, so a solo client's own row can never ring its own bell. */
511
+ export interface ScopeSessionsEvent {
512
+ scope: string;
513
+ others: number;
514
+ }
515
+
516
+ /** One demoted room source's §4.2 ghost record (Slice I-v): the room slice stays FROZEN in the
517
+ * merge (wins-if-present — still supplying the visible rows, accepting nothing) until the
518
+ * daemon plane has provably absorbed the room's final flush. The drop condition is evaluated
519
+ * after every release ({@link OptimisticBackend.evaluateGhosts}):
520
+ *
521
+ * `roomWatermarks[doc] ≥ finalFlushSeq` (0 ⇒ trivially true — a never-flushed room)
522
+ * AND no pending mutation with `domain === sourceKey` remains (sent-pins-domain, §7.5 —
523
+ * room-domain entries retire ONLY through the outcome-resolved daemon-carried folds, I-iii).
524
+ */
525
+ interface RoomGhost {
526
+ doc: string;
527
+ finalFlushSeq: number;
528
+ /** Every table promoted for the sourceKey at demote time — the {@link dropGhost} removal set
529
+ * (and, while the ghost lives, its hold on the {@link promotedTables} record). */
530
+ tables: string[];
531
+ /** Whether the ONE stuck-downgrade event already fired for this ghost. */
532
+ stuckReported: boolean;
533
+ }
534
+
535
+ /** The I-v stuck-downgrade event ({@link OptimisticBackend.onDowngradeStuck}): the ghost's fence
536
+ * is satisfied but these SENT room-domain mids never resolved (an entry that never reached the
537
+ * room — sent-but-undelivered when the socket died — is undecidable in general, §7.5). The ghost
538
+ * HOLDS (fail LOUD, never silent; no timeout-retire is invented) and the mids are surfaced once,
539
+ * actionably. */
540
+ export interface DowngradeStuckEvent {
541
+ sourceKey: string;
542
+ doc: string;
543
+ mids: number[];
544
+ }
545
+
546
+ /** One §301 echo-fence pin (`301-ECHO-FENCE-DESIGN.md` §2.1): the client-side registry entry for
547
+ * a parked engine hold-back, carrying the DELIVERY-fence inputs the engine deliberately does not
548
+ * know. Parked in the same breath as `holdBack`/`holdBackAbsent` at the §7.3 confirm-drop; keyed
549
+ * `${table}\0${sourceKey}\0${pkKey}` so a re-park of the same slice-pk overwrites (matching the
550
+ * engine's `held_back.insert`). The fence per direction (§1):
551
+ *
552
+ * - **A** (`sourceKey === "daemon"`, a room-domain write staged on the daemon slice): drops
553
+ * when the DAEMON-CARRIED room ledger covers `mid` — {@link OptimisticBackend}'s
554
+ * `daemonCarriedLmid[domain] ≥ mid` (the I-ii co-commit puts the flush data in the same
555
+ * coherent release, §1.1).
556
+ * - **B** (a room `sourceKey`, `domain === "daemon"`): drops when the room gate advertises
557
+ * absorption at-or-past the parking daemon release — `(upstreamBoot, upstreamCv)` covers
558
+ * `(daemonBoot, daemonCv)` under the §2.4 boot rule.
559
+ * - a room-staged pin confirmed by ANOTHER room has no fence (out of today's two-tier
560
+ * topology); it drops only via the engine's state-match fallback. */
561
+ interface EchoFencePin {
562
+ table: string;
563
+ /** The staged slice holding the engine pin — what to drop ON. */
564
+ sourceKey: string;
565
+ /** The held row / pre-image (full-width; the pk derives from it) — what to drop AT. */
566
+ probeRow: WireValue[];
567
+ /** The confirming domain (direction A's ledger to watch). */
568
+ domain: string;
569
+ /** The confirmed wire mid (direction A's fence input). */
570
+ mid: number;
571
+ /** Direction B stamps (301 §1.2): the parking daemon release's coherence position + the
572
+ * client-observed daemon boot it belongs to. Absent on direction A / the no-fence shape. */
573
+ daemonBoot?: string;
574
+ daemonCv?: number;
575
+ /** The slice's confirmed-baseline row at park — the §2.5 stuck-pin tripwire's "has the
576
+ * baseline CHANGED VALUE since park" reference (that change is the suspicious state that
577
+ * precedes every forever-pin). */
578
+ baselineAtPark?: WireValue[];
579
+ /** The tripwire's log-once latch. */
580
+ tripwired?: boolean;
581
+ }
582
+
226
583
  export interface OptimisticBackendOptions {
227
584
  /** Stable per-client identity for the upstream envelopes (§8.1). */
228
585
  clientID: string;
@@ -236,6 +593,23 @@ export interface OptimisticBackendOptions {
236
593
  /** Virtual-clock seam for the fold debounce timers (FOLDED-MUTATIONS-DESIGN §9). Defaults to
237
594
  * real `setTimeout`/`clearTimeout`/`Date.now`; the fold oracle injects a deterministic clock. */
238
595
  clock?: FoldClock;
596
+ /** The explicit confirming-stream OVERRIDE (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §7.1 /
597
+ * §3). Since H-iii the live default is the §3 prove-or-slow-path DERIVATION
598
+ * ({@link OptimisticBackend.deriveDomain}): a configured policy returning a string pins that
599
+ * domain VERBATIM — no proof runs (the test suites and the dual-source e2e pin by name);
600
+ * returning `undefined` (or configuring no policy) derives the domain from the prediction run's
601
+ * write/read capture against the connected room gates' routing table. With no room gate
602
+ * connected the derivation short-circuits to `"daemon"` — every single-domain app is
603
+ * byte-for-byte as before. */
604
+ domainPolicy?: (name: string, args: unknown) => string | undefined;
605
+ /** A FINAL (authz/validation) mutation rejection's reason surface — the room plane's twin of the
606
+ * HTTP mutate route's `onRejected` (H-v; the H-iv-b `mutationOutcome {kind:"rejected"}` frame).
607
+ * The prediction's snap-back is NOT this callback's job: the room burns the mid and its lmid
608
+ * release drops the entry exactly as a daemon-path rejection does (processed-as-no-op) — this
609
+ * is where the REASON reaches the app, same contract as the queue's callback. Also invoked when
610
+ * a DEOPT's fresh re-invocation (the already-retired arm) throws — that mutation is dead on the
611
+ * current base with no stream left to confirm it, the closest thing to a rejection there is. */
612
+ onRejected?: (envelope: MutationEnvelope, reason: string) => void;
239
613
  }
240
614
 
241
615
  // --- dev-only introspection (DEBUG-TOOLS-BROWSER-DESIGN §2/§4.1) -----------------
@@ -267,6 +641,13 @@ export interface PendingInspect {
267
641
  args: unknown;
268
642
  /** Tables this mutator touched at its last (re)invocation — the pending-axis basis (§7.2). */
269
643
  tables: string[];
644
+ /** The pk-granular write-set captured at this entry's LAST invocation (RINDLE-REALTIME-QUERY-
645
+ * ENABLEMENT-DESIGN.md §3.2 #1), flattened from the {@link WriteSet} map for inspection — one
646
+ * entry per `(table, pk)` currently held. Pure capture; no routing consumer yet. */
647
+ writes: WriteRecord[];
648
+ /** The read-log captured at this entry's LAST *recorded* invocation (§3.2 #2). Empty for a
649
+ * folded entry — the read TRAP arms there, not recording (see {@link PendingMutation.reads}). */
650
+ reads: ReadLog;
270
651
  /** Present iff this entry is a folded (debounced) write. */
271
652
  fold?: FoldInspect;
272
653
  }
@@ -298,6 +679,15 @@ const REAL_CLOCK: FoldClock = {
298
679
  now: () => Date.now(),
299
680
  };
300
681
 
682
+ /** The (shared, frozen-by-convention) empty map {@link OptimisticBackend.roomTablesFor} answers
683
+ * for a room with no promoted tables. */
684
+ const EMPTY_ROUTING: ReadonlyMap<string, RoomTableRouting> = new Map();
685
+
686
+ /** Per-domain retention cap for the processed-outcome set (H-v) — mirrors the shell's
687
+ * `MAX_RECORDED_OUTCOMES_PER_CLIENT`: the sender caps what it can re-answer at 512 per client,
688
+ * so remembering more than 512 processed mids per domain buys nothing. */
689
+ const MAX_PROCESSED_OUTCOMES_PER_DOMAIN = 512;
690
+
301
691
  export class OptimisticBackend<S extends ColsMap> implements Backend {
302
692
  private readonly local: WasmBackend<S>;
303
693
  private readonly sync: NormalizedSync;
@@ -320,6 +710,9 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
320
710
  * its width check. */
321
711
  private readonly colCounts: ColCounts;
322
712
  private readonly colIndex: Record<string, Map<string, number>>;
713
+ /** Per-table pk column indices — held so `connectSource` can build a fresh per-source
714
+ * `NormalizedSync` with the same layout the daemon's uses. */
715
+ private readonly pkCols: PkCols;
323
716
  /** The client's OWN typed per-table schemas + the reserved lmid table — the fixed base
324
717
  * of the expected-schema set (CRIT#4 validation). Synthetic agg tables are appended as
325
718
  * queries arrive (`ensureSyntheticTables`). */
@@ -338,6 +731,18 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
338
731
  * pending delta `displayed = server_base ⊕ local_pending_delta` is applied from. */
339
732
  private readonly overlay = new AggOverlay();
340
733
  private handler: (qid: QueryId, ev: ChangeEvent) => void = () => {};
734
+ // The local qids whose reconcile-cycle batch delivery this release is their FIRST hydration (empty
735
+ // → full): their batch is the initial result set arriving as a delta, so it is stamped `catchUp`
736
+ // and the Store maps it to the `snapshot` change-phase (a narrator ignores it by default). Non-null
737
+ // only for the duration of the reconcile cycle in `onProgress`; a re-hydrate after a drop is a real
738
+ // footprint diff (genuine change) and is NOT remapped. See {@link ChangeEvent} `catchUp`.
739
+ private catchUpQids: Set<QueryId> | null = null;
740
+ /** Newly-hydrated qids whose reconcile ACTUALLY emitted a (catch-up-stamped) batch — recorded by the
741
+ * local-event forwarder alongside {@link catchUpQids}. After the reconcile, any newly-hydrated qid
742
+ * NOT in here folded nothing (0 rows, or its result already present via a sibling → 0 net muts, or
743
+ * the reconcile was skipped), so `onProgress` sends it an explicit empty catch-up — else its SSR
744
+ * seed would never retire (the view freezes). Non-null only for the reconcile's duration. */
745
+ private catchUpEmitted: Set<QueryId> | null = null;
341
746
  /** The Store's commit-boundary handler ({@link Backend.onCommitBoundary}), forwarded from the
342
747
  * local engine's `dispatch` brackets so the Store folds every affected view before notifying any
343
748
  * subscriber (cross-view-atomic notification). All this backend's data deltas originate from the
@@ -346,18 +751,142 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
346
751
  private readonly devObservers = new Set<BackendDevObserver>();
347
752
 
348
753
  private pendingMutations: PendingMutation[] = [];
349
- private nextMid = 1;
754
+ /** The next mid to deal, PER DOMAIN (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §7.1): a client
755
+ * writing through room + daemon concurrently must not alias one lmid counter. Seeded with the
756
+ * `"daemon"` stream at 1; a domain absent from the map starts at 1. In the single-domain
757
+ * configuration only `"daemon"` is ever touched, so the sequence is byte-for-byte as before. */
758
+ private nextMid = new Map<string, number>([["daemon", 1]]);
759
+ /** The client-global deal counter behind {@link PendingMutation.seq}: one sequence across ALL
760
+ * domains, bumped whenever any domain's mid is dealt. The replay order (mids are per-domain and
761
+ * incomparable across domains — see the `seq` field doc). */
762
+ private dealSeq = 0;
763
+ /** The explicit confirming-stream override (§7.1/§3) — see
764
+ * {@link OptimisticBackendOptions.domainPolicy}. `undefined` from it ⇒ H-iii derivation. */
765
+ private readonly domainPolicy: (name: string, args: unknown) => string | undefined;
766
+ /** The final-rejection reason surface ({@link OptimisticBackendOptions.onRejected}). */
767
+ private readonly rejectedHandler: (envelope: MutationEnvelope, reason: string) => void;
768
+ /** Processed `(domain, mid)` outcome frames (H-v) — the deopt handshake's idempotence guard: a
769
+ * duplicate frame (the original plus a reconnect re-send's re-answer, or two re-answers across
770
+ * two reconnects) must not double-invoke. Needed precisely because a deopt frame can arrive for
771
+ * an ALREADY-RETIRED mid (the replay gotcha) — "no matching entry" alone cannot distinguish
772
+ * "handle it fresh" from "already handled". Per-domain FIFO, capped like the shell's
773
+ * recorded-outcome map ({@link MAX_PROCESSED_OUTCOMES_PER_DOMAIN}); past the cap a duplicate of
774
+ * an evicted mid would be re-processed — the same bounded-window trade the shell makes, and it
775
+ * takes 512 interleaving non-applied outcomes on one domain to open it. */
776
+ private readonly outcomesProcessed = new Map<string, Set<number>>();
777
+ /** THE routing table (H-iii §3): per connected room `sourceKey` → per promoted table → its
778
+ * client-held {@link RoomTableRouting}. Written ONLY by {@link promoteRoomTable} (same breath
779
+ * as the engine attach + {@link promotedTables}); read by {@link deriveDomain} and — via
780
+ * {@link roomTablesFor} — the client's `__realtimeInspect` bookkeeping. */
781
+ private readonly roomRouting = new Map<string, Map<string, RoomTableRouting>>();
782
+ /** Q6 counters ({@link RoutingInspect}): successful derivations per room. */
783
+ private readonly routingDerived = new Map<string, number>();
784
+ /** Q6 counters ({@link RoutingInspect}): derivation failures per {@link RoutingFailureReason}. */
785
+ private readonly routingReasons = new Map<string, number>();
350
786
  /** The live fold entries, by fold key `${name}\0${identityJSON}` — at most one per key
351
787
  * (FOLDED-MUTATIONS-DESIGN §8). Insertion order is creation order (the drain/flush tiebreak). */
352
788
  private readonly folds = new Map<string, FoldRecord>();
353
789
  /** The fold debounce clock (real timers by default; the oracle injects a virtual one). */
354
790
  private readonly clock: FoldClock;
355
- /** The high-water confirmed mutation id, folded from the lmid system query's
356
- * RELEASED ops (lmid-as-data) never from a frame. */
357
- private confirmedLmid = 0;
358
- private buffer: BufferedFrame[] = [];
359
- private nextSeq = 0;
360
- private appliedCv = 0;
791
+ /** The high-water confirmed mutation id, PER DOMAIN (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md
792
+ * §7.2 per-domain confirm-drop): an entry with `mid <= watermark[entry.domain]` has been
793
+ * confirmed. The `"daemon"` domain is folded from the lmid system query's RELEASED ops
794
+ * (lmid-as-data) never from a frame; a room domain will fold from its own lmid stream (later
795
+ * slice). Seeded with `"daemon"` at 0; the daemon scalar `confirmedLmid` (devtools) is
796
+ * `watermark.get("daemon")`. */
797
+ private watermark = new Map<string, number>([["daemon", 0]]);
798
+ /** The per-source coherence gates (§5.1), by source key. Seeded with the daemon gate at
799
+ * construction; a room channel attaches later (`connectSource`). Single-domain: one entry,
800
+ * and every gate-generalized path degenerates to the old single-buffer code. NOT the same
801
+ * space as {@link watermark}/{@link nextMid}: a DOMAIN can confirm with no gate connected
802
+ * (the `__testRelease` seam); a gate's `key` names the domain its lmid stream folds into. */
803
+ private readonly gates = new Map<string, SourceGate>();
804
+ /** The daemon's gate — the always-present channel (constructor-attached). The devtools
805
+ * scalars (`__inspect`) read it directly; its `sync` IS {@link sync} (the agg overlay and
806
+ * synthetic tables are daemon-tracked by design). */
807
+ private readonly daemonGate: SourceGate;
808
+ /** Tables promoted to a MERGED multi-source engine (a room source attached). Written by the one
809
+ * promotion seam ({@link promoteRoomTable}; the test-named {@link __addRoomSource} and Slice
810
+ * G-v's lease-driven promotion from `RoomTableSpec[]` both flow through it). Read by the §7.3
811
+ * hold-back trigger in
812
+ * {@link applyRelease}: parking on a never-promoted (Collapsed) table is SKIPPED — inert anyway
813
+ * today (`rewind_collapsed` never consults `held_back`), and a live hazard if the table later
814
+ * promotes (a stale Collapsed-era entry turns overlay-first-visible and pins forever). Slice H's
815
+ * prove-or-slow-path routing keeps the real scenario — a room-confirmed mutation writing a
816
+ * Collapsed table — impossible; this gate covers the window until then. */
817
+ private readonly promotedTables = new Set<string>();
818
+ /** The §301 pin registry ({@link EchoFencePin}): every parked §7.3 hold-back's delivery-fence
819
+ * inputs, keyed `${table}\0${sourceKey}\0${pkKey}`. Written in the same breath as the
820
+ * `holdBack`/`holdBackAbsent` park; read by {@link dropEchoFencePins} at the tail of every
821
+ * release. Empty on every single-domain client — the drop pass is then a structural no-op. */
822
+ private readonly pins = new Map<string, EchoFencePin>();
823
+ /** The §301 direction-A fence input: per room domain, the highest lmid the DAEMON-CARRIED
824
+ * ledger rows have folded ({@link foldSystemFrames} step 2 — and ONLY that path: the room
825
+ * socket's own lmid stream confirms long before the flush reaches the daemon, so it folds
826
+ * into the shared {@link watermark} but never here, 301 §1.1). When this covers a pin's mid,
827
+ * the same coherent release (or an earlier one) folded the flush data that carried it into
828
+ * the daemon baseline — the fence. */
829
+ private readonly daemonCarriedLmid = new Map<string, number>();
830
+ /** The §301 §2.4 boot rule's inputs: the daemon boot ids this client has OBSERVED, in order
831
+ * (id → ordinal), plus the current one. Boot ids are opaque — ordering is the client's own
832
+ * observation ({@link OptimisticSource.onBootId} on the daemon gate). Direction-B pins stamp
833
+ * the current boot; a room advertising a LATER-observed boot proves absorption (its
834
+ * re-snapshot came from a daemon state that durably includes the confirmed write). */
835
+ private readonly daemonBootOrdinals = new Map<string, number>();
836
+ private daemonBootId: string | undefined;
837
+ /** The per-read provenance probe `invoke` hands `trackingTx` when recording is armed (H-ii,
838
+ * §3.2 #3): the engine's `provenanceOf` — the VISIBLE overlay-first winner for the row's pk,
839
+ * read from LIVE pre-commit state (an open txn's staged writes are not consulted; H-i). Gated
840
+ * on PER-TABLE {@link promotedTables} membership — the cheapest existing signal, and the
841
+ * correct one: provenance is an ENGINE-merge question, not a channel question ({@link gates}
842
+ * can hold a connected-but-unpromoted room feed, and the oracle harness promotes with no gate),
843
+ * and a never-promoted (Collapsed) table is daemon-owned by construction. So a pure
844
+ * single-domain app pays ZERO wasm calls per recorded read; the recorded `source` is then
845
+ * `undefined`, which is also correct — everything is daemon-owned (see {@link ReadRecord}). */
846
+ private readonly readProvenance = (table: string, row: WireValue[]): string | undefined =>
847
+ this.promotedTables.has(table) ? this.local.provenanceOf(table, row) : undefined;
848
+
849
+ // --- the §4 lifecycle SYSTEM-STREAM plane (Slice I-iii) --------------------------------
850
+ /** System retains by source qid ({@link retainSystemQuery}): a subscription with NO store view
851
+ * and NO user-visible table — its frames buffer on its gate exactly like {@link LMID_QID}'s and
852
+ * fold at RELEASE time ({@link foldSystemFrames}), never entering the sync layer or the local
853
+ * engine. The spec names which system table the qid serves and the scope/doc it was minted for
854
+ * (the fold's row filter). Empty on every non-lifecycle client — every partition below is then
855
+ * a structural no-op and the release path is byte-identical to before. */
856
+ private readonly systemQids = new Map<QueryId, SystemStreamSpec>();
857
+ /** The §4.2 fence state: room doc → highest `flush_seq` delivered through the daemon plane
858
+ * (monotone max-fold; a remove never regresses it). Slice I-v's ghost-drop consumer — I-iii
859
+ * only maintains + exposes it (`__inspectDomains().lifecycle`). */
860
+ private readonly roomWatermarks = new Map<string, number>();
861
+ /** The §4.1 occupancy state: scope → (client_id → expires_at) from the doorbell stream. Slice
862
+ * I-iv's doorbell consumer (the 1→2 re-lease reaction) — I-iii only maintains + exposes it.
863
+ * A snapshot REPLACES the scope's map (authoritative re-hydrate); a batch folds add/edit/remove
864
+ * incrementally (the age-out sweep's deletes arrive as removes). */
865
+ private readonly scopeSessions = new Map<string, Map<string, number>>();
866
+ /** The I-iv doorbell event sink ({@link onScopeSessions}) — fired once per scope a release's
867
+ * scope-session fold touched, AFTER the whole release applied. Default no-op: a client that
868
+ * never registers (no lifecycle plane) pays nothing. */
869
+ private scopeSessionsHandler: (event: ScopeSessionsEvent) => void = () => {};
870
+ /** Deferred old-channel row GC for in-flight upgrade retargets ({@link retargetRemoteQuery}):
871
+ * sub sourceQid → the channel it left. The rows the OLD gate's sync holds for the qid stay
872
+ * visible (merge: daemon tier) until the sub's first snapshot RELEASES on its new room channel
873
+ * ({@link flushRetargetGc}) — dropping them at retarget time would emit net removes ahead of
874
+ * the room's re-adds, the flicker the two-phase cutover exists to avoid. Doubles as the
875
+ * wrong-channel GRACE window in {@link onFrame}: a frame already in flight from the old
876
+ * channel when the sub moved is stale, not a wiring bug. Empty on every non-upgrade client —
877
+ * every consultation below is then a structural no-op. */
878
+ private readonly pendingRetargetGc = new Map<QueryId, string>();
879
+ /** The §4.2 GHOSTS (Slice I-v): demoted room sources awaiting their watermark fence, by
880
+ * sourceKey. Written only by {@link demoteRoomSource}; evaluated after every release
881
+ * ({@link evaluateGhosts}) and dropped by {@link dropGhost} once the fence clears with no
882
+ * sent room-domain pending left. Empty on every non-downgrade client — the per-release
883
+ * evaluation is then a structural no-op. */
884
+ private readonly ghosts = new Map<string, RoomGhost>();
885
+ /** The I-v stuck-downgrade surface ({@link onDowngradeStuck}) — fired AT MOST ONCE per ghost
886
+ * when its fence is satisfied but sent room-domain mids remain unresolved (§7.5: they retire
887
+ * only through outcome resolution; the ghost holds rather than inventing a timeout-retire).
888
+ * Default no-op. */
889
+ private downgradeStuckHandler: (event: DowngradeStuckEvent) => void = () => {};
361
890
 
362
891
  private readonly asts = new Map<QueryId, Ast>();
363
892
  /** Per query: the base tables its result can draw from (from the AST tree). */
@@ -376,6 +905,9 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
376
905
  * touches its tables. Cached so `onPending` fires only on transitions (invoke ↔ confirm). */
377
906
  private readonly pendingState = new Map<QueryId, boolean>();
378
907
  private pendingHandler: (qid: QueryId, pending: boolean) => void = () => {};
908
+ /** The local-persistence write-through tap (`207-LOCAL-TABLE-PERSISTENCE-DESIGN.md` §5.1):
909
+ * {@link writeLocal} invokes it post-commit; {@link applyLocalReplica} deliberately does not. */
910
+ private localWriteObserver: ((mutations: Mutation[]) => void) | null = null;
379
911
 
380
912
  constructor(
381
913
  schema: Schema<S>,
@@ -384,13 +916,22 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
384
916
  opts: OptimisticBackendOptions,
385
917
  ) {
386
918
  this.local = new WasmBackend(schema);
387
- this.local.onEvent((qid, ev) => this.handler(qid, ev));
919
+ this.local.onEvent((qid, ev) => {
920
+ // Stamp a newly-hydrating query's reconcile batch as a catch-up (initial-hydration) delivery,
921
+ // so the Store phases it as a `snapshot` rather than narrating the whole first result set. Record
922
+ // that we emitted a hydration batch for this qid, so `onProgress` knows which newly-hydrated qids
923
+ // still need an explicit empty catch-up (they folded nothing — see {@link catchUpEmitted}).
924
+ const stamp = ev.type === "batch" && this.catchUpQids?.has(qid) === true;
925
+ if (stamp) this.catchUpEmitted?.add(qid);
926
+ this.handler(qid, stamp ? { ...ev, catchUp: true } : ev);
927
+ });
388
928
  // Forward the local engine's commit brackets up to the Store (cross-view-atomic notification):
389
929
  // every data delta this backend emits comes from `this.local`, so its commit boundaries are ours.
390
930
  this.local.onCommitBoundary((phase) => this.boundaryHandler(phase));
391
931
  this.colCounts = colCountsFromSchema(schema);
392
932
  this.colIndex = colIndexFromSchema(schema);
393
- this.sync = new NormalizedSync(pkColsFromSchema(schema), this.colCounts);
933
+ this.pkCols = pkColsFromSchema(schema);
934
+ this.sync = new NormalizedSync(this.pkCols, this.colCounts);
394
935
  this.specs = tableSpecsFromSchema(schema);
395
936
  this.localTables = localTableNames(schema);
396
937
  this.source = source;
@@ -399,23 +940,83 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
399
940
  this.user = opts.user ?? (() => "");
400
941
  this.bufferCap = opts.bufferCap ?? 1024;
401
942
  this.clock = opts.clock ?? REAL_CLOCK;
402
- // Validate each server hello against our OWN typed schema reject a schema skew
403
- // (CRIT#4). The reserved lmid table is part of the expected set so the system
404
- // query's hello passes. Synthetic agg tables join the set as queries register them.
405
- this.clientTablesBase = [...normalizedTableSchemas(schema), CLIENT_MUTATIONS_SCHEMA];
406
- this.source.expectClientSchema?.(this.clientTablesBase);
407
- this.source.onNormalized((qid, ev) => this.onNormalized(qid, ev));
408
- this.source.onProgress((frame) => this.onProgress(frame));
409
- this.source.onRestart?.(() => this.resetForRestart());
410
- // The lmid system query (lmid-as-data): our confirmations arrive on this stream,
411
- // cv-tagged, released by the same cvMin as the data they belong to. The server
412
- // derives the identity from the connection; args are advisory.
413
- this.source.registerQuery(LMID_QID, { name: LMID_QUERY_NAME, args: {} });
943
+ // No policy configured every route DERIVES (H-iii §3). With no room gate connected the
944
+ // derivation short-circuits to "daemon", so a single-domain app is byte-for-byte as before.
945
+ this.domainPolicy = opts.domainPolicy ?? (() => undefined);
946
+ this.rejectedHandler = opts.onRejected ?? (() => {});
947
+ // The reserved lmid table + (I-iii) the four lifecycle system tables join the expected set so
948
+ // a system subscription's hello passes CRIT#4 validation. Extra CLIENT-side entries are inert
949
+ // for every other server hello (validation only checks tables a server advertises), so a
950
+ // client that never receives a lifecycle block is byte-identical.
951
+ this.clientTablesBase = [...normalizedTableSchemas(schema), CLIENT_MUTATIONS_SCHEMA, ...LIFECYCLE_TABLE_SCHEMAS];
952
+ // The daemon is the always-present channel: its gate is attached at construction, and its
953
+ // per-source refcount space IS `this.sync` (the agg overlay reads it directly). A room
954
+ // channel attaches through the same seam later (§5.1; Slice G).
955
+ this.daemonGate = this.attachGate("daemon", source, this.sync);
956
+ }
957
+
958
+ /** Wire one authority channel into its own coherence gate (§5.1): every frame the channel
959
+ * delivers buffers on THIS gate's cv timeline, its progress frames release THIS buffer, its
960
+ * restart resets THIS gate alone, and its reserved lmid stream folds into `watermark[key]`.
961
+ * Validates each server hello against our OWN typed schema → reject a schema skew (CRIT#4);
962
+ * the reserved lmid table is part of the expected set so the system query's hello passes, and
963
+ * synthetic agg tables join the set as queries register them. */
964
+ private attachGate(key: string, source: OptimisticSource, sync: NormalizedSync): SourceGate {
965
+ const gate: SourceGate = { key, source, sync, buffer: [], nextSeq: 0, appliedCv: 0 };
966
+ this.gates.set(key, gate);
967
+ source.expectClientSchema?.([...this.clientTablesBase, ...this.synthetic.values()]);
968
+ source.onNormalized((qid, ev) => this.onFrame(gate, qid, ev));
969
+ source.onProgress((frame) => this.onGateProgress(gate, frame));
970
+ source.onRestart?.(() => this.resetGate(gate));
971
+ // §301 §2.4: the DAEMON channel's observed boot-id stream seeds the client-local boot
972
+ // ordinals the direction-B fence's boot rule compares against (boot ids are opaque; order is
973
+ // the client's own observation). Only the daemon gate's boots matter — a room's advertised
974
+ // `upstreamBoot` names a DAEMON boot, and room-channel incarnations are handled by
975
+ // `resetGate`. Optional: an in-process source never reports one.
976
+ if (key === "daemon") source.onBootId?.((bootId) => this.observeDaemonBoot(bootId));
977
+ // The deopt handshake's client half (H-v §3.3): the channel's `mutationOutcome` frames arrive
978
+ // as `(domain = gate.key, frame)`. OUT-OF-BAND — the source dispatches on arrival and this
979
+ // handler runs immediately, NEVER behind the gate's cv buffer: a deopt must migrate its entry
980
+ // BEFORE the buffered lmid release that would otherwise retire it as a success (and the §7.3
981
+ // hold-back trigger, keyed on `p.domain`, would park its staged writes the wrong way).
982
+ source.onMutationOutcome?.((frame) => this.handleMutationOutcome(gate.key, frame));
983
+ // §7.5 rule 3 (H-v): a re-established session re-sends this DOMAIN's unconfirmed pending
984
+ // envelopes with their ORIGINAL mids — the authority's own ledger dedups (an applied mid is
985
+ // silent; a non-applied one is re-answered from the recorded-outcome map into the handler
986
+ // above). This is the deopt crash-window closer: a frame lost with its socket is re-earned.
987
+ source.onResync?.(() => this.resendPending(gate.key));
988
+ // The lmid system query (lmid-as-data): confirmations arrive on this channel's stream,
989
+ // cv-tagged, released by the same cvMin as the data they belong to. The server derives
990
+ // the identity from the connection; args are advisory. Qid 0 is reserved PER CHANNEL —
991
+ // it never collides with Store-dealt qids and never enters the sync layer.
992
+ source.registerQuery(LMID_QID, { name: LMID_QUERY_NAME, args: {} });
993
+ return gate;
994
+ }
995
+
996
+ /** Attach a SECOND authority channel (§5.1) — the seam Slice G's room upgrade calls with the
997
+ * ws-backed room feed. Rooms speak the daemon protocol verbatim (§2.4: the client cannot tell
998
+ * a room from the daemon), so the argument is a full {@link OptimisticSource} — exactly what
999
+ * `@rindle/remote` builds from `{roomUrl, leaseToken}`. The channel buffers/releases on its
1000
+ * own cv timeline (an independent §5.1 gate: coherent within, eventual across) and its
1001
+ * reserved lmid stream folds into `watermark[sourceKey]` — so `sourceKey` must equal the
1002
+ * `domainPolicy` name for the mutations this authority confirms. The converse is NOT required:
1003
+ * a domain may exist with no connected gate (`__testRelease` drives confirms gate-less); the
1004
+ * live production path stays daemon-only until G calls this. */
1005
+ connectSource(sourceKey: string, source: OptimisticSource): void {
1006
+ if (this.gates.has(sourceKey)) {
1007
+ throw new Error(`optimistic backend: source ${sourceKey} is already connected`);
1008
+ }
1009
+ this.attachGate(sourceKey, source, new NormalizedSync(this.pkCols, this.colCounts));
414
1010
  }
415
1011
 
416
1012
  // --- the Backend seam ---------------------------------------------------------
417
1013
 
418
- registerQuery(qid: QueryId, ast: Ast, remote?: RemoteQuery): void {
1014
+ /** `channel` (G-iii registration-time routing) names the authority channel the remote sub
1015
+ * registers on — a `connectSource`d gate key; default `"daemon"` (every existing caller is
1016
+ * byte-identical). Slice G-v threads the lease's `realtime.sourceKey` here. Validated FIRST
1017
+ * (like the E3 check below): a bad channel must throw before any per-query state is recorded. */
1018
+ registerQuery(qid: QueryId, ast: Ast, remote?: RemoteQuery, channel?: string): void {
1019
+ if (remote) this.requireGate(channel ?? "daemon");
419
1020
  // queryTables is derived from the ORIGINAL ast — its `count(comments)` subquery names
420
1021
  // `comment`, so an optimistic comment mutation flips this query to `unknown` (§6). The
421
1022
  // local engine, by contrast, runs the REWRITTEN ast (reads the synthetic `__agg_*`).
@@ -450,7 +1051,7 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
450
1051
  if (remote) {
451
1052
  // A remote query is `unknown` until its first server snapshot lands (hydration); retainRemote
452
1053
  // attaches it to the sub and sets the lifecycle against the sub's hydration state.
453
- this.retainRemote(qid, remote);
1054
+ this.retainRemote(qid, remote, qid, channel);
454
1055
  } else {
455
1056
  // No server stream (a purely local AST view — or the local half of a split retain whose
456
1057
  // remote attaches separately via `retainRemoteQuery`): local data is synchronous, so it is
@@ -517,9 +1118,18 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
517
1118
 
518
1119
  unregisterQuery(qid: QueryId): void {
519
1120
  const remoteQid = this.releaseRemote(qid);
520
- if (remoteQid !== undefined) this.buffer = this.buffer.filter((f) => f.qid !== remoteQid);
521
- // GC: rows this remote footprint SOLELY referenced fall to refcount 0 net removes.
522
- const gc = remoteQid === undefined ? [] : this.sync.dropQuery(remoteQid);
1121
+ // GC: rows this remote footprint SOLELY referenced fall to refcount 0 → net removes. A qid
1122
+ // lives on ONE channel, so at most one gate's dropQuery is non-empty (dropQuery of an
1123
+ // unknown qid returns []) but sweep every gate so this needs no ownership lookup.
1124
+ const gcs: [string, Mutation[]][] = [];
1125
+ if (remoteQid !== undefined) {
1126
+ this.pendingRetargetGc.delete(remoteQid); // the sweep below covers a mid-retarget teardown
1127
+ for (const gate of this.gates.values()) {
1128
+ gate.buffer = gate.buffer.filter((f) => f.qid !== remoteQid);
1129
+ const gc = gate.sync.dropQuery(remoteQid);
1130
+ if (gc.length) gcs.push([gate.key, gc]);
1131
+ }
1132
+ }
523
1133
  // Tear down the local pipeline+view first so the reconcile cycle below skips it.
524
1134
  this.local.unregisterQuery(qid);
525
1135
  // The GC removals must leave BOTH head AND the engine's `sync` baseline. A plain
@@ -527,8 +1137,9 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
527
1137
  // optimistic REMOVE: the next release's rewind diffs head against sync+D and RESURRECTS
528
1138
  // them, GC never frees anything, and a later query is served the stale/deleted row
529
1139
  // forever (CRIT#2). Deliver them as a coherent SERVER delta instead — the same
530
- // sync-moving boundary `onProgress` uses — so head and sync both drop the rows.
531
- if (gc.length) this.runReconcileCycle(gc);
1140
+ // sync-moving boundary the release gate uses — so head and sync both drop the rows,
1141
+ // against the SOURCE whose baseline held them.
1142
+ for (const [key, gc] of gcs) this.runReconcileCycle(key, gc);
532
1143
  // The local pipeline is gone (no live conn) and the remote footprint's `__agg` rows were
533
1144
  // GC'd above, so any synthetic table this was the last reader of can now be freed (§4).
534
1145
  this.releaseSyntheticTables(qid);
@@ -540,9 +1151,14 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
540
1151
  // Store counter — re-materialize gets a fresh id, never this one again), so drop it on teardown.
541
1152
  }
542
1153
 
543
- retainRemoteQuery(qid: QueryId, remote: RemoteQuery, localQueryId?: QueryId, ast?: Ast): void {
1154
+ /** `channel` as in {@link registerQuery} (G-iii): the gate the remote sub registers on; default
1155
+ * `"daemon"`. This is the split-retain seam G-v's resolve-then-register drives — resolve the
1156
+ * lease, learn `realtime.sourceKey`, `connectSource` it, then retain the query on that channel.
1157
+ * Validated FIRST so a bad channel throws before any synthetic-table refcount moves. */
1158
+ retainRemoteQuery(qid: QueryId, remote: RemoteQuery, localQueryId?: QueryId, ast?: Ast, channel?: string): void {
1159
+ this.requireGate(channel ?? "daemon");
544
1160
  if (ast) this.ensureSyntheticTables(qid, ast);
545
- this.retainRemote(qid, remote, localQueryId);
1161
+ this.retainRemote(qid, remote, localQueryId, channel);
546
1162
  }
547
1163
 
548
1164
  releaseRemoteQuery(qid: QueryId): void {
@@ -553,9 +1169,305 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
553
1169
  this.hydrated.delete(qid);
554
1170
  }
555
1171
  if (remoteQid === undefined) return;
556
- this.buffer = this.buffer.filter((f) => f.qid !== remoteQid);
557
- const gc = this.sync.dropQuery(remoteQid);
558
- if (gc.length) this.runReconcileCycle(gc);
1172
+ // A mid-retarget release: the every-gate sweep below IS the deferred old-channel GC
1173
+ // (dropQuery hits the old gate's sync too), so retire the pending record — and its
1174
+ // wrong-channel grace — with it.
1175
+ this.pendingRetargetGc.delete(remoteQid);
1176
+ // Per-gate sweep, like `unregisterQuery`: at most one gate owned this qid's frames/rows.
1177
+ for (const gate of this.gates.values()) {
1178
+ gate.buffer = gate.buffer.filter((f) => f.qid !== remoteQid);
1179
+ const gc = gate.sync.dropQuery(remoteQid);
1180
+ if (gc.length) this.runReconcileCycle(gate.key, gc);
1181
+ }
1182
+ }
1183
+
1184
+ /** The Slice I-iv upgrade retarget (§4.1 "Retarget" / the doorbell reaction): move a LIVE
1185
+ * (name, args) sub — every retain of it and every local view it feeds, wholesale — from the
1186
+ * channel it lives on onto `sourceKey`'s (already-`connectSource`d, already-promoted) room
1187
+ * channel, WITHOUT the view ever dropping its rows. Returns the sub's wire `sourceQid` (the
1188
+ * identity the client's renewal loop re-subscribes with).
1189
+ *
1190
+ * Why a dedicated primitive: the one-channel-per-(name,args) invariant ({@link retainRemote}'s
1191
+ * loud throw) is correct — a sub's frames must never split across two cv timelines — so the
1192
+ * upgrade cannot simply retain a second sub on the room and release the daemon one; and the
1193
+ * naive release-then-retain order GCs the daemon sync's rows synchronously (net removes emit,
1194
+ * the view flashes empty) a full ws round trip before the room's seq-0 snapshot refills it.
1195
+ * The cutover is therefore TWO-PHASE around the room's first release:
1196
+ *
1197
+ * 1. NOW (here): unsubscribe the old channel's wire sub, sweep its still-buffered frames for
1198
+ * this qid (their cv timeline continues without the sub — the hello-supersession
1199
+ * precedent), flip `sub.channel`, re-arm `sub.hydrated` (the room's own snapshot is the
1200
+ * cutover point), and register on the room source (its resolver presents the handed
1201
+ * roomToken). The old gate's SYNC rows are deliberately NOT dropped: they keep the view's
1202
+ * rows visible through the window (merge: the daemon-tier holder). Local views keep their
1203
+ * `hydrated` membership — the view stays `complete` on its authoritative daemon rows.
1204
+ * 2. AT THE ROOM'S FIRST RELEASED SNAPSHOT ({@link flushRetargetGc}): the room source now
1205
+ * holds the footprint rows (folded via `serverBatchBegin(sourceKey)` — value-equal rows
1206
+ * re-hydrate as a net-zero diff, RT §3.4 per channel), so the deferred
1207
+ * `dropQuery`+reconcile on the OLD gate flips each pk's winner daemon→room value-equal:
1208
+ * net-zero again. No emission carries a disappearance at any point.
1209
+ *
1210
+ * Idempotent per target channel: a sub already on `sourceKey` returns immediately (the
1211
+ * double-doorbell / re-entrancy guard — one retarget per (query, sourceKey), mirroring
1212
+ * {@link promoteRoomTable}'s caller-side per-(sourceKey, table) idempotence). Validates before
1213
+ * mutating: a throw here leaves the sub fully daemon-attached (the client's fail-open). */
1214
+ retargetRemoteQuery(remote: RemoteQuery, sourceKey: string): QueryId {
1215
+ const newGate = this.requireGate(sourceKey); // throw loudly BEFORE any sub state moves
1216
+ const key = remoteKey(remote);
1217
+ const sub = this.remoteSubs.get(key);
1218
+ if (!sub) {
1219
+ throw new Error(
1220
+ `optimistic backend: no live sub for query "${remote.name}" — nothing to retarget`,
1221
+ );
1222
+ }
1223
+ if (sub.channel === sourceKey) return sub.sourceQid; // already there — idempotent
1224
+ const oldGate = this.gates.get(sub.channel) ?? this.daemonGate;
1225
+ oldGate.source.unregisterQuery(sub.sourceQid);
1226
+ oldGate.buffer = oldGate.buffer.filter((f) => f.qid !== sub.sourceQid);
1227
+ this.pendingRetargetGc.set(sub.sourceQid, sub.channel);
1228
+ sub.channel = sourceKey;
1229
+ sub.hydrated = false;
1230
+ newGate.source.registerQuery(sub.sourceQid, remote);
1231
+ return sub.sourceQid;
1232
+ }
1233
+
1234
+ /** Phase 2 of {@link retargetRemoteQuery}, run at the end of every gate release: once a
1235
+ * retargeted sub's first snapshot has RELEASED on its new channel (`sub.hydrated` re-armed at
1236
+ * retarget, re-set by {@link markSubHydrated} inside this very release), drop the qid's rows
1237
+ * from the OLD gate's sync and reconcile them out — after the room's rows are already applied,
1238
+ * so the winner flip is value-equal (net-zero; see the phase table above). A sub torn down
1239
+ * mid-window was already swept by `releaseRemoteQuery`/`unregisterQuery` (which delete the
1240
+ * record); a vanished record here is pruned defensively. */
1241
+ private flushRetargetGc(gate: SourceGate): void {
1242
+ if (this.pendingRetargetGc.size === 0) return; // every non-upgrade release: structural no-op
1243
+ for (const [sourceQid, oldGateKey] of this.pendingRetargetGc) {
1244
+ const key = this.sourceToRemote.get(sourceQid);
1245
+ const sub = key !== undefined ? this.remoteSubs.get(key) : undefined;
1246
+ if (!sub) {
1247
+ this.pendingRetargetGc.delete(sourceQid);
1248
+ continue;
1249
+ }
1250
+ if (sub.channel !== gate.key || !sub.hydrated) continue; // not this gate / not yet cut over
1251
+ this.pendingRetargetGc.delete(sourceQid);
1252
+ const oldGate = this.gates.get(oldGateKey);
1253
+ if (!oldGate) continue;
1254
+ const gc = oldGate.sync.dropQuery(sourceQid);
1255
+ if (gc.length) this.runReconcileCycle(oldGateKey, gc);
1256
+ }
1257
+ }
1258
+
1259
+ // --- the §4.2 downgrade: demote → ghost → fence → drop (Slice I-v) ----------------------
1260
+
1261
+ /** The I-v downgrade orchestration primitive (§4.2/§7.4): retire room `sourceKey` behind the
1262
+ * watermark fence. The caller has ALREADY retargeted every live sub off the channel
1263
+ * ({@link retargetRemoteQuery} room→daemon — validated loudly below) and holds the fence from
1264
+ * the api-server's downgrade response (`finalFlushSeq` = the room's last COMMITTED flush seq;
1265
+ * `doc` keys the §4.2 watermark fold, {@link roomWatermarks}). Steps, in order:
1266
+ *
1267
+ * 1. **De-candidacy NOW**: the sourceKey's {@link roomRouting} entries are removed, so
1268
+ * {@link deriveDomain} stops proposing the room (zero candidates ⇒ `"daemon"`). Unsent
1269
+ * pending (`mid === null`) re-routes for free — a fold's flush re-derives from the live
1270
+ * candidate set (§7.5 rule 1). The engine's frozen scope still ANSWERS (`writableMatches`
1271
+ * / `provenanceOf` — freezing gates serving, not the scope definition); routing is
1272
+ * TS-gated here.
1273
+ * 2. **Freeze** the WRITABLE promoted tables (`freezeSource`): the room slice becomes the
1274
+ * §4.2 ghost — wins-if-present in the merge (D4), so its rows (at-or-ahead of the daemon
1275
+ * until the flush echoes) stay visible while the slice accepts nothing. Context tables
1276
+ * (`writable: false`) are deliberately NOT frozen: the daemon is authoritative for them
1277
+ * LIVE (§5.2's context tier) and freezing would invert that, pinning a possibly-BEHIND
1278
+ * relayed copy over fresher daemon rows; unfrozen they keep exactly the live tiering.
1279
+ * 3. **Disconnect** the channel ({@link disconnectSource}): handlers detached, gate + buffer
1280
+ * dropped. `nextMid`/`watermark`/processed-outcomes for the domain are KEPT FOREVER (§7.1:
1281
+ * an assigned mid pins its domain; a later re-upgrade of the same doc continues the
1282
+ * sequence — {@link connectSource} attaches a fresh gate and the lmid snapshot max-folds
1283
+ * into the surviving watermark). Disconnecting BEFORE the daemon sub's first release is
1284
+ * load-bearing: it makes {@link flushRetargetGc}'s deferred old-channel GC a no-op (gate
1285
+ * gone ⇒ record deleted, nothing dropped) — running that GC would rewind the room slice's
1286
+ * rows at daemon hydration, i.e. BEFORE the fence, surfacing a lagging follower's stale
1287
+ * images (the exact regression §4.2 exists to prevent). The ghost's rows leave only
1288
+ * through {@link dropGhost}'s `removeRoomSource`, value-equal under the fence.
1289
+ * 4. **Ghost + first evaluation**: the record joins {@link ghosts} and is evaluated once
1290
+ * immediately — `finalFlushSeq === 0` (a never-flushed room) with no room-domain pending
1291
+ * drops on the spot, the single-daemon first-frame case.
1292
+ *
1293
+ * In-flight discipline (§7.5): entries with `mid !== null` on `sourceKey` stay PINNED (rule
1294
+ * 2 — never re-route a sent mutation); their resolution arrives via the daemon-carried
1295
+ * ledger+outcome folds (I-iii) and blocks the drop until then. Idempotent per sourceKey (a
1296
+ * second labeled query sharing the room demotes into the existing ghost). */
1297
+ demoteRoomSource(sourceKey: string, doc: string, finalFlushSeq: number): void {
1298
+ if (sourceKey === "daemon") {
1299
+ throw new Error("optimistic backend: the daemon source cannot be demoted");
1300
+ }
1301
+ if (this.ghosts.has(sourceKey)) return; // idempotent — one ghost per source
1302
+ // Validate FIRST (nothing mutated yet): a live sub still on the channel would silently
1303
+ // starve once the gate detaches — the caller must retarget every sub off the room first.
1304
+ for (const sub of this.remoteSubs.values()) {
1305
+ if (sub.channel === sourceKey) {
1306
+ throw new Error(
1307
+ `optimistic backend: cannot demote ${JSON.stringify(sourceKey)} — query "${sub.remote.name}" is still retained on it (retarget it to the daemon first)`,
1308
+ );
1309
+ }
1310
+ }
1311
+ const routing = this.roomRouting.get(sourceKey);
1312
+ const tables = routing ? [...routing.keys()] : [];
1313
+ const writable = routing ? tables.filter((t) => routing.get(t)!.writable) : [];
1314
+ this.roomRouting.delete(sourceKey); // (1) de-candidacy
1315
+ for (const table of writable) this.local.freezeSource(table, sourceKey); // (2) the ghost
1316
+ this.disconnectSource(sourceKey); // (3) the channel
1317
+ this.ghosts.set(sourceKey, { doc, finalFlushSeq, tables, stuckReported: false }); // (4)
1318
+ this.evaluateGhosts();
1319
+ }
1320
+
1321
+ /** Detach one connected room channel (Slice I-v step 3): the source's handlers are replaced
1322
+ * with no-ops (the {@link OptimisticSource} handler seam is single-registration, so this IS
1323
+ * the detach — a late frame from a dying socket can no longer touch any bookkeeping), its
1324
+ * reserved lmid sub is unregistered, and the gate — buffer, per-source sync, cv watermark —
1325
+ * is dropped from {@link gates}. The DOMAIN state deliberately survives forever:
1326
+ * `nextMid[sourceKey]`, `watermark[sourceKey]`, and the processed-outcome set are untouched
1327
+ * (§7.1 — an assigned mid pins its domain; a re-upgrade must continue, never restart, the mid
1328
+ * sequence; {@link connectSource} then attaches a fresh gate whose lmid snapshot max-folds
1329
+ * into the surviving watermark via {@link foldConfirm}). Closing the underlying transport is
1330
+ * the caller's job. Idempotent (a missing gate is a no-op). */
1331
+ disconnectSource(sourceKey: string): void {
1332
+ if (sourceKey === "daemon") {
1333
+ throw new Error("optimistic backend: the daemon source cannot be disconnected");
1334
+ }
1335
+ const gate = this.gates.get(sourceKey);
1336
+ if (!gate) return;
1337
+ this.gates.delete(sourceKey);
1338
+ gate.source.onNormalized(() => {});
1339
+ gate.source.onProgress(() => {});
1340
+ gate.source.onRestart?.(() => {});
1341
+ gate.source.onMutationOutcome?.(() => {});
1342
+ gate.source.onResync?.(() => {});
1343
+ gate.source.unregisterQuery(LMID_QID);
1344
+ }
1345
+
1346
+ /** Register the I-v stuck-downgrade sink — see {@link DowngradeStuckEvent}. One handler (a
1347
+ * later registration replaces it, the {@link onScopeSessions} convention); client.ts maps it
1348
+ * onto the loud anomaly surface. */
1349
+ onDowngradeStuck(handler: (event: DowngradeStuckEvent) => void): void {
1350
+ this.downgradeStuckHandler = handler;
1351
+ }
1352
+
1353
+ /** The I-v ghost-drop watcher (§4.2), run after every applied release ({@link applyRelease} —
1354
+ * the seam where {@link roomWatermarks} has just folded and the confirm-drop has just run) and
1355
+ * once at demote time. For each ghost: the fence must be satisfied
1356
+ * (`roomWatermarks[doc] ≥ finalFlushSeq`; 0 is trivially satisfied) AND no SENT room-domain
1357
+ * pending may remain (§7.5 — such entries resolve only through the daemon-carried
1358
+ * outcome/ledger folds; an entry that never reached the room is undecidable, so the ghost
1359
+ * HOLDS and the stuck event fires exactly once, naming the mids). Both satisfied ⇒
1360
+ * {@link dropGhost}. */
1361
+ private evaluateGhosts(): void {
1362
+ if (this.ghosts.size === 0) return; // every non-downgrade release: structural no-op
1363
+ for (const [sourceKey, ghost] of [...this.ghosts]) {
1364
+ if ((this.roomWatermarks.get(ghost.doc) ?? 0) < ghost.finalFlushSeq) continue; // fence holds
1365
+ const stuck = this.pendingMutations.filter((p) => p.domain === sourceKey && p.mid !== null);
1366
+ if (stuck.length > 0) {
1367
+ if (!ghost.stuckReported) {
1368
+ ghost.stuckReported = true;
1369
+ this.downgradeStuckHandler({ sourceKey, doc: ghost.doc, mids: stuck.map((p) => p.mid as number) });
1370
+ }
1371
+ continue; // hold — never a timeout-retire (§7.5 rule 2)
1372
+ }
1373
+ this.dropGhost(sourceKey, ghost);
1374
+ }
1375
+ }
1376
+
1377
+ /** Drop one cleared ghost (§4.2 end state): detach every promoted table's room source (D2
1378
+ * stay-merged — `removeRoomSource` reconciles each held pk to daemon-or-vanish, value-equal
1379
+ * under the fence, so visually a no-op), release the {@link promotedTables} record for tables
1380
+ * no OTHER room/ghost still holds, then run ONE daemon reconcile so entries whose writes had
1381
+ * staged onto the ghost slice re-invoke and re-stage by provenance onto the daemon slice (the
1382
+ * removal took their staged copies with the tree; provenance now answers daemon). The whole
1383
+ * drop runs under one commit boundary so the removal's compensating deltas and the re-staged
1384
+ * predictions notify as ONE step. After this, a FUTURE upgrade of the same doc promotes
1385
+ * again from scratch — the round trip is pinned by the §8.5 lanes. */
1386
+ private dropGhost(sourceKey: string, ghost: RoomGhost): void {
1387
+ this.ghosts.delete(sourceKey);
1388
+ // Re-stage eligibility FIRST: a pending entry staged (only) on the ghost slice must join the
1389
+ // daemon reconcile below (the cycle filter is `touchedSources.has(sourceKey)`) — union, never
1390
+ // shrink, the §5.3 rule.
1391
+ let restage = false;
1392
+ for (const p of this.pendingMutations) {
1393
+ if (p.touchedSources.has(sourceKey)) {
1394
+ p.touchedSources.add("daemon");
1395
+ restage = true;
1396
+ }
1397
+ }
1398
+ this.inOneCommit(() => {
1399
+ for (const table of ghost.tables) this.local.removeRoomSource(table, sourceKey);
1400
+ for (const table of ghost.tables) {
1401
+ let held = false;
1402
+ for (const byTable of this.roomRouting.values()) if (byTable.has(table)) held = true;
1403
+ for (const g of this.ghosts.values()) if (g.tables.includes(table)) held = true;
1404
+ if (!held) this.promotedTables.delete(table);
1405
+ }
1406
+ // One daemon reconcile replays the re-staged entries onto the daemon slice. Run whenever
1407
+ // any pending exists: the removal above may have taken a staged write with the tree even
1408
+ // when `touchedSources` never recorded the ghost (defensive; an empty-pending drop skips).
1409
+ if (restage || this.pendingMutations.length > 0) this.runReconcileCycle("daemon", []);
1410
+ });
1411
+ this.refreshPending(); // the reconcile may have dropped a throwing re-invocation
1412
+ }
1413
+
1414
+ // --- the §4 lifecycle SYSTEM-STREAM retains (Slice I-iii) ------------------------------
1415
+
1416
+ /** Retain one minted SYSTEM subscription (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §4, Slice
1417
+ * I-iii): a wire sub with NO store view and NO user-visible table. Registered through the same
1418
+ * {@link RemoteSub} bookkeeping as any remote retain — so qid→channel ownership, the overflow
1419
+ * re-subscribe, and refcounted release all work unchanged — but with an EMPTY `localQids` set
1420
+ * (no hydration/resultType coupling) and a {@link systemQids} record telling the release path
1421
+ * which system table this qid's frames carry (`spec.table`) and which scope/doc it was minted
1422
+ * for (the fold's row filter). Its frames then buffer on the channel's gate exactly like
1423
+ * {@link LMID_QID}'s and fold at RELEASE time in {@link foldSystemFrames} — riding the SAME
1424
+ * buffered cv path as the data they co-committed with (fence coherence: an out-of-band
1425
+ * shortcut would break I-ii's co-commit ordering guarantee).
1426
+ *
1427
+ * `channel` defaults to `"daemon"` — the system tables live in the DAEMON store (that is the
1428
+ * point: outcome/ledger/watermark rows must be readable with no room socket alive, §7.1
1429
+ * "load-bearing for §7.5"). Idempotence per (table, scope/doc) is the CALLER's job (client.ts
1430
+ * keys its retains on exactly that); a duplicate retain of the SAME remote identity refcounts
1431
+ * like any sub. */
1432
+ retainSystemQuery(retainQid: QueryId, remote: RemoteQuery, spec: SystemStreamSpec, channel = "daemon"): void {
1433
+ const gate = this.requireGate(channel); // throw loudly BEFORE any sub state moves
1434
+ const key = remoteKey(remote);
1435
+ let sub = this.remoteSubs.get(key);
1436
+ if (sub) {
1437
+ if (sub.channel !== channel) {
1438
+ throw new Error(
1439
+ `optimistic backend: system query "${remote.name}" is already retained on channel ${JSON.stringify(sub.channel)} — cannot retain it on ${JSON.stringify(channel)}`,
1440
+ );
1441
+ }
1442
+ sub.refCount++;
1443
+ this.localToRemote.set(retainQid, key);
1444
+ this.remoteRetainToLocal.set(retainQid, undefined);
1445
+ return;
1446
+ }
1447
+ // A fresh sub: deliberately NOT `retainRemote` — its `localQueryId` default would couple this
1448
+ // retain's qid to the view-hydration machinery (`hydrated`/`resultType`), and a system stream
1449
+ // has no view to hydrate.
1450
+ sub = { sourceQid: retainQid, remote, refCount: 1, localQids: new Map(), hydrated: false, channel };
1451
+ this.remoteSubs.set(key, sub);
1452
+ this.sourceToRemote.set(retainQid, key);
1453
+ this.localToRemote.set(retainQid, key);
1454
+ this.remoteRetainToLocal.set(retainQid, undefined);
1455
+ this.systemQids.set(retainQid, { ...spec });
1456
+ gate.source.registerQuery(retainQid, remote);
1457
+ }
1458
+
1459
+ /** Release a {@link retainSystemQuery} retain. Refcounted like any sub; the LAST release
1460
+ * unregisters from the owning channel, sweeps its buffered frames, and drops the
1461
+ * {@link systemQids} record. The folded lifecycle STATE (`roomWatermarks`/`scopeSessions`/
1462
+ * processed outcomes) deliberately survives — the fence is monotone truth about the store, not
1463
+ * about the subscription (a re-retained fence must not forget a cleared watermark). */
1464
+ releaseSystemQuery(retainQid: QueryId): void {
1465
+ const remoteQid = this.releaseRemote(retainQid);
1466
+ if (remoteQid === undefined) return; // still refcounted (or unknown)
1467
+ for (const gate of this.gates.values()) {
1468
+ gate.buffer = gate.buffer.filter((f) => f.qid !== remoteQid);
1469
+ }
1470
+ this.systemQids.delete(remoteQid);
559
1471
  }
560
1472
 
561
1473
  /** Raw CRUD has no optimistic story (§9 replaces it with named mutators). Register a
@@ -570,9 +1482,33 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
570
1482
  * straight to the local engine, OUTSIDE the optimistic pending stack — a local table is
571
1483
  * untracked, so it never rebases, reverts, or waits on a confirmation. Rejects a synced/tracked
572
1484
  * table (the local engine's `writeLocal` is the chokepoint). Reconcile is synchronous and
573
- * non-reentrant (A5), so a local write can never interleave with an open server cycle. */
1485
+ * non-reentrant (A5), so a local write can never interleave with an open server cycle.
1486
+ *
1487
+ * Fires the {@link onLocalWrite} observer AFTER the engine commit but BEFORE subscriber
1488
+ * delivery — i.e. for exactly the batches that passed the M2 guard and committed (the
1489
+ * persistence layer's write-through tap, `207-LOCAL-TABLE-PERSISTENCE-DESIGN.md` §5.1). A
1490
+ * subscriber throwing during delivery re-raises out of this call, but only after the tap has
1491
+ * seen the batch: a committed write can never be invisible to the persistence plane. */
574
1492
  writeLocal(mutations: Mutation[]): void {
575
- this.local.writeLocal(mutations);
1493
+ this.local.writeLocal(mutations, () => this.localWriteObserver?.(mutations));
1494
+ }
1495
+
1496
+ /** The write-through tap for the local-persistence layer (207 §5.1): `observer` sees every
1497
+ * {@link writeLocal} batch post-commit. One observer (the layer); a later registration
1498
+ * replaces it. The observer must not throw — a persistence failure degrades durability, never
1499
+ * the write path (P9); the layer catches internally. */
1500
+ onLocalWrite(observer: (mutations: Mutation[]) => void): void {
1501
+ this.localWriteObserver = observer;
1502
+ }
1503
+
1504
+ /** Apply a REPLICATED local batch (a restore snapshot / a leader `commit` — 207 §5.1): delegates
1505
+ * to the engine's `writeLocal`, so the M2 locality guard still fires (P8 — a corrupt record
1506
+ * naming a synced table dies loudly here), but does NOT invoke the {@link onLocalWrite}
1507
+ * observer — the echo guard is structural, so the persistence layer can never re-enter itself.
1508
+ * `onCommitted` fires post-commit pre-delivery (same anchor as {@link writeLocal}'s tap): the
1509
+ * layer updates its mirror there, so a subscriber throw can never desync mirror from engine. */
1510
+ applyLocalReplica(mutations: Mutation[], onCommitted?: () => void): void {
1511
+ this.local.writeLocal(mutations, onCommitted);
576
1512
  }
577
1513
 
578
1514
  onEvent(handler: (qid: QueryId, ev: ChangeEvent) => void): void {
@@ -625,10 +1561,223 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
625
1561
  }
626
1562
  }
627
1563
 
1564
+ /** Deal the next wire mid from `domain`'s ledger (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md
1565
+ * §7.1) and advance that counter. A domain absent from the map starts at 1. Per-domain, so a
1566
+ * client writing through room + daemon concurrently keeps two gapless, non-aliasing sequences.
1567
+ * The client-global `seq` is stamped in the same breath — the ONE cross-domain total order
1568
+ * (confirmation order is per-domain; replay order is client-global). Bundled here so no call
1569
+ * site can deal a mid without its seq. ONE caller discards the seq deliberately: the H-v deopt
1570
+ * flip ({@link handleMutationOutcome}) keeps the entry's ORIGINAL seq — its replay position —
1571
+ * and takes only the fresh mid (the dealSeq bump is harmless: seq consumers order, never
1572
+ * count). */
1573
+ private dealMid(domain: string): { mid: number; seq: number } {
1574
+ const mid = this.nextMid.get(domain) ?? 1;
1575
+ this.nextMid.set(domain, mid + 1);
1576
+ return { mid, seq: ++this.dealSeq };
1577
+ }
1578
+
1579
+ // --- the §3 prove-or-slow-path ROUTER (H-iii) -----------------------------------
1580
+ //
1581
+ // RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §3: a mutation routes to room R iff every row its
1582
+ // prediction run WROTE is provably inside R's writable scope and every read it RECORDED is
1583
+ // provably covered by R's footprint — ANY unproven condition routes slow (the §0 asymmetry:
1584
+ // mis-guessing "slow" costs one round-trip; mis-guessing "room" would commit divergent data).
1585
+ // Failure is therefore SILENT "daemon" — never a throw, never a warning per call — plus the Q6
1586
+ // per-reason counters ({@link RoutingInspect}).
1587
+ //
1588
+ // The proof evaluates against the ENGINE wherever the engine has an answer: `writableMatches`
1589
+ // (the registered compiled writable scope — the same predicate the merge's winner tiering uses)
1590
+ // and `provenanceOf` (the visible overlay-first winner). It runs AFTER the prediction committed,
1591
+ // so the probes see the prediction's own staged effects — which only ever strengthens the
1592
+ // conservative direction (a fresh pk reports "daemon"/undefined and neither disqualifies).
1593
+ //
1594
+ // ** THE ONE CLIENT-SIDE NON-ENGINE EVALUATOR — deliberately minimal (the TS evaluation
1595
+ // caveat). ** Evaluating `footprintWhere` on a read's pk cells ({@link evalFootprintOnPk}) is
1596
+ // the single place Slice H allows a non-engine evaluator, and it is restricted to STRICT cell
1597
+ // equality only: `simple` `=` / `!=` between a pk column and a same-primitive-type non-null
1598
+ // literal, composed under and/or (the empty AND is the vacuous-true exact-unconstrained-root
1599
+ // emission and evaluates true). ANY other op or shape — LIKE, <, >=, IS, IN, null operands,
1600
+ // cross-type comparisons, column-vs-column — is NOT EVALUABLE and fails to the daemon.
1601
+ // Justification: a mis-evaluation here cannot commit divergence — the room GATE (H-iv) re-proves
1602
+ // every write and absent read engine-side at commit and deopts, so a wrong client verdict only
1603
+ // costs a wasted hop (route room → gate deopt) or a skipped optimization (route daemon); and
1604
+ // restricting to strict equality on non-null, same-typed pk cells removes the entire
1605
+ // null/collation/coercion divergence space between this evaluator and the engine's comparators.
1606
+ // The gate remains the contract; this router is the optimization.
1607
+ //
1608
+ // ** H-v (SHIPPED — the router's missing recovery half). ** A derived (or pinned) room route
1609
+ // whose gate DEOPTS (the H-iv-b `mutationOutcome {kind:"deopt"}` frame — the mid is burnt in
1610
+ // the room ledger, no effects committed) is re-enqueued onto the daemon stream by
1611
+ // {@link handleMutationOutcome}: the entry flips in place (fresh daemon mid, ORIGINAL seq,
1612
+ // prediction applied throughout), or — already retired — re-invokes from the frame's echoed
1613
+ // name/args pinned to the daemon. A reconnect re-sends unconfirmed mids so a frame lost with
1614
+ // its socket is re-answered ({@link resendPending}). A wrong client verdict therefore costs
1615
+ // exactly one room round-trip plus one burnt room-mid (§3.3), never a stranded prediction.
1616
+
1617
+ /** The route decision for one invocation (invoke, and again at fold flush): the explicit
1618
+ * `domainPolicy` is the OVERRIDE — a string pins that domain verbatim, no proof runs;
1619
+ * `undefined` (or no policy configured) derives per §3. */
1620
+ private resolveDomain(name: string, args: unknown, writes: WriteSet, reads: ReadLog): string {
1621
+ const pinned = this.domainPolicy(name, args);
1622
+ if (pinned !== undefined) return pinned;
1623
+ return this.deriveDomain(writes, reads);
1624
+ }
1625
+
1626
+ /** §9.3: does this fold's write-set route into a room (vs the daemon)? Used ONLY to pick the
1627
+ * {@link FoldOptions.roomDebounceMs} cadence at a fold window's start. It probes
1628
+ * {@link resolveDomain} but RESTORES the Q6 routing diagnostics afterward — the authoritative
1629
+ * route and its single counter bump belong to the flush ({@link flushFold}), not to this
1630
+ * interval hint. A fold's reads are provably empty (the read-trap arms on that path), so the
1631
+ * write-set alone decides. */
1632
+ private routesToRoom(name: string, args: unknown, writes: WriteSet): boolean {
1633
+ const reasons = [...this.routingReasons];
1634
+ const derived = [...this.routingDerived];
1635
+ const domain = this.resolveDomain(name, args, writes, { reads: [], queries: [] });
1636
+ this.routingReasons.clear();
1637
+ for (const [k, v] of reasons) this.routingReasons.set(k, v);
1638
+ this.routingDerived.clear();
1639
+ for (const [k, v] of derived) this.routingDerived.set(k, v);
1640
+ return domain !== "daemon";
1641
+ }
1642
+
1643
+ /** Count one derivation failure ({@link RoutingInspect}). Returns `false` so the per-candidate
1644
+ * proof's call sites read `return this.failDerivation(...)`. */
1645
+ private failDerivation(reason: RoutingFailureReason): false {
1646
+ this.routingReasons.set(reason, (this.routingReasons.get(reason) ?? 0) + 1);
1647
+ return false;
1648
+ }
1649
+
1650
+ /** The §3 derivation. Candidates are the connected room gates with at least one promoted table
1651
+ * (a promoted table with no gate cannot confirm; a gate with no promoted table holds no data to
1652
+ * prove against). Zero candidates ⇒ `"daemon"` (the single-domain fast path). The proof runs
1653
+ * per candidate and EXACTLY ONE must survive — two rooms both proving routes slow (principled
1654
+ * disambiguation is a §9.2 multi-room NON-goal, deferred past Slice J; slow is always sound). */
1655
+ private deriveDomain(writes: WriteSet, reads: ReadLog): string {
1656
+ const candidates: string[] = [];
1657
+ for (const key of this.gates.keys()) {
1658
+ if (key === "daemon") continue;
1659
+ if ((this.roomRouting.get(key)?.size ?? 0) > 0) candidates.push(key);
1660
+ }
1661
+ if (candidates.length === 0) {
1662
+ this.failDerivation("no-candidates");
1663
+ return "daemon";
1664
+ }
1665
+ // tx.query is not room-executable client-side: predicate containment (rindle-cover) is
1666
+ // native-only BY DESIGN, so a declarative read fails the WHOLE derivation unconditionally.
1667
+ // This is required, not conservative.
1668
+ if (reads.queries.length > 0) {
1669
+ this.failDerivation("tx-query");
1670
+ return "daemon";
1671
+ }
1672
+ const proven = candidates.filter((room) => this.provesRoom(room, writes, reads));
1673
+ if (proven.length === 1) {
1674
+ const room = proven[0];
1675
+ this.routingDerived.set(room, (this.routingDerived.get(room) ?? 0) + 1);
1676
+ return room;
1677
+ }
1678
+ if (proven.length > 1) this.failDerivation("ambiguous");
1679
+ return "daemon"; // zero survivors: each candidate already counted its own failure reason
1680
+ }
1681
+
1682
+ /** One candidate room's §3 proof over the captured write-set + read-log. Every write must pass
1683
+ * ALL the write rules; every read must pass ONE of the read rules. First failure wins (and is
1684
+ * counted); order is deterministic (write-set map order, then read-log order). */
1685
+ private provesRoom(room: string, writes: WriteSet, reads: ReadLog): boolean {
1686
+ const specs = this.roomRouting.get(room);
1687
+ // --- write rules: every WriteRecord passes ALL of #1–#4 -------------------------------
1688
+ for (const [table, byPk] of writes) {
1689
+ // #1: the table is promoted for R with a writable (predicate-kind) spec. A context table
1690
+ // (`writable: none`), an un-promoted table, and a local table (unreachable — the mutator
1691
+ // guard refuses local writes at stage time) all fail here.
1692
+ const spec = specs?.get(table);
1693
+ if (spec === undefined || !spec.writable) return this.failDerivation("write-unwritable-table");
1694
+ const colIx = this.colIndex[table];
1695
+ for (const rec of byPk.values()) {
1696
+ const removeShape = rec.row === undefined;
1697
+ // #2: the registered writable scope, evaluated BY THE ENGINE on the post-image for
1698
+ // add/edit shapes and on the pre-image for removes (the row the room would delete). A
1699
+ // remove record always carries its full-width pre-image (the H-ii capture contract);
1700
+ // a violated contract fails closed rather than probing nothing.
1701
+ const scopeRow = removeShape ? rec.oldRow : rec.row;
1702
+ if (scopeRow === undefined || !this.local.writableMatches(table, room, scopeRow)) {
1703
+ return this.failDerivation("write-scope-miss");
1704
+ }
1705
+ // #3: join-key no-change. An edit-shape (row + oldRow — the oldRow is the txn-entry base)
1706
+ // must keep every joinKeyCols cell STRICTLY identical old-vs-new; an add-shape may SET
1707
+ // them (it creates the correlation); removes are exempt. A join-key column missing from
1708
+ // the schema is a spec bug — fail closed.
1709
+ if (!removeShape && rec.oldRow !== undefined) {
1710
+ for (const col of spec.joinKeyCols) {
1711
+ const i = colIx?.get(col);
1712
+ if (i === undefined || !identicalCell(rec.oldRow[i], rec.row![i])) {
1713
+ return this.failDerivation("write-join-key-change");
1714
+ }
1715
+ }
1716
+ }
1717
+ // #4: provenance corroboration — the engine routes an Edit by its OLD row (H-i), so probe
1718
+ // the pre-image for edit/remove shapes and the post-image for add-shapes. A winner naming
1719
+ // a DIFFERENT room disqualifies; "daemon"/undefined does NOT (a thin slice, or a fresh pk
1720
+ // — fresh pks stage to the daemon slice by the E2 decision, kept permanently: the
1721
+ // room-side authoritative run is the committing execution either way).
1722
+ const probeRow = rec.oldRow ?? rec.row;
1723
+ if (probeRow !== undefined) {
1724
+ const src = this.local.provenanceOf(table, probeRow);
1725
+ if (src !== undefined && src !== "daemon" && src !== room) {
1726
+ return this.failDerivation("write-cross-room-provenance");
1727
+ }
1728
+ }
1729
+ }
1730
+ }
1731
+ // --- read rules: every ReadRecord passes ONE ------------------------------------------
1732
+ // (tx.query already failed the whole derivation in deriveDomain — never reaches here.)
1733
+ for (const r of reads.reads) {
1734
+ // Self-read: the pk is in THIS invocation's write-set for that table — the write rules
1735
+ // above already judged it (covers the keyed writers' probe-then-write shape). NOTE the
1736
+ // read-log reflects the ORIGINAL invoke only (the pre-existing capture caveat).
1737
+ if (writes.get(r.table)?.has(stableJson(r.pk))) continue;
1738
+ if (r.source !== undefined) {
1739
+ if (r.source === room) continue; // the room served the row — in-footprint by construction
1740
+ if (r.source !== "daemon") return this.failDerivation("read-cross-room");
1741
+ }
1742
+ // Present-with-daemon/undefined source, or ABSENT (source is never recorded for those —
1743
+ // including the un-promoted-table meaning of an absent key): the footprint-membership test
1744
+ // on the PK ALONE. R's footprintWhere must exist, be key-decidable (every column it reads is
1745
+ // a pk column), and evaluate TRUE on the read's pk cells:
1746
+ // present-daemon + TRUE ⇒ the row is in the room's COMPLETE footprint ⇒ covered;
1747
+ // absent + TRUE ⇒ absent-in-room = absent-in-truth ⇒ covered;
1748
+ // FALSE (either outcome) ⇒ fail — present-daemon-FALSE means the room lacks the row;
1749
+ // absent-FALSE is arguably provable (decidably-outside ⇒ the room never sees the pk)
1750
+ // but the room-side run would then read absent for a DIFFERENT reason than truth's —
1751
+ // stay CONSERVATIVE and fail;
1752
+ // no footprintWhere / not key-decidable / not evaluable ⇒ fail (see the evaluator caveat
1753
+ // on the router block comment).
1754
+ const fw = specs?.get(r.table)?.footprintWhere;
1755
+ if (fw === undefined) return this.failDerivation("read-no-footprint-where");
1756
+ const tspec = this.specs[r.table];
1757
+ const pkCols = tspec.primaryKey.map((i) => tspec.columns[i]);
1758
+ if (!keyDecidable(fw, new Set(pkCols))) return this.failDerivation("read-not-key-decidable");
1759
+ const pkCells = new Map<string, WireValue>(pkCols.map((c, j) => [c, r.pk[j]]));
1760
+ const verdict = evalFootprintOnPk(fw, pkCells);
1761
+ if (verdict === undefined) return this.failDerivation("read-not-evaluable");
1762
+ if (!verdict) return this.failDerivation("read-outside-footprint");
1763
+ }
1764
+ return true;
1765
+ }
1766
+
628
1767
  /** Run the named client mutator optimistically: the prediction applies to the live
629
1768
  * engine now (affected views update synchronously), `(mid, name, args)` joins the
630
1769
  * pending stack, and the envelope ships upstream. Returns the assigned `mid`. */
631
1770
  invoke(name: string, args: unknown): number {
1771
+ return this.invokeWith(name, args);
1772
+ }
1773
+
1774
+ /** {@link invoke} with an optional PINNED confirming domain (H-v): the deopt handshake's
1775
+ * already-retired arm re-invokes the frame's echoed `(name, args)` as a FRESH invocation pinned
1776
+ * to `"daemon"` — an honest re-prediction on the current base, never derived (`pin` bypasses
1777
+ * {@link resolveDomain} entirely, so the router never runs and no Q6 counter moves). Every
1778
+ * other step is `invoke` verbatim: prediction now, capture, drainOverlapping, mid dealt from
1779
+ * the pinned domain's ledger, envelope on its channel. */
1780
+ private invokeWith(name: string, args: unknown, pin?: string): number {
632
1781
  const mutator = this.registry[name];
633
1782
  if (!mutator) throw new Error(`unknown client mutator: ${name}`);
634
1783
  // One commit boundary spans the prediction AND the `__agg`-head reconcile below, so their views
@@ -638,18 +1787,42 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
638
1787
  // the staged write is discarded (the wasm txn is a clean no-op until commit) and the throw
639
1788
  // propagates with NO mid consumed — a burnt mid is a permanent server-side gap that
640
1789
  // silently refuses every later mutation from this client (#10).
641
- const touched = new Set<string>();
1790
+ const writes: WriteSet = new Map();
1791
+ const reads: ReadLog = { reads: [], queries: [] };
642
1792
  const ops: ChildOp[] = [];
643
- this.local.writeWith((tx) => {
644
- this.runMutator(mutator, trackingTx(tx, touched, this.specs, this.localTables, this.opCollector(ops)), args);
1793
+ // Per-staged-write source keys (staged order), zipped with the `commitTracked` `sources[]`
1794
+ // below to learn which physical source each pk routed onto (§5.3 filter + §7.3 hold-back).
1795
+ const stagedKeys: string[] = [];
1796
+ const sources = this.local.writeWith((tx) => {
1797
+ this.runMutator(
1798
+ mutator,
1799
+ // `readProvenance` rides only with recording (H-ii §3.2 #3): the folded path (trap, no
1800
+ // readLog) and the reconcile replay (no readLog) never probe — see trackingTx.
1801
+ trackingTx(tx, writes, this.specs, this.localTables, this.opCollector(ops), false, reads, stagedKeys, this.readProvenance),
1802
+ args,
1803
+ );
645
1804
  });
1805
+ // `touched` is DERIVED, never separately populated (§3.2 #1) — see {@link WriteSet}.
1806
+ const touched = new Set(writes.keys());
646
1807
  // Flush-on-enqueue (§4.2): a fold whose tables overlap this write must take its mid NOW, BEFORE
647
1808
  // this write does, so wire order == local-apply order for any pair that can observe each other
648
1809
  // (a read-dependent write reading a folded cell sees the same value optimistically and on the
649
1810
  // wire — no snap). Drained folds ship with smaller mids; this write's mid is dealt after.
650
1811
  this.drainOverlapping(touched);
651
- const mid = this.nextMid++;
652
- this.pendingMutations.push({ mid, name, args, touched });
1812
+ // The confirming stream (§7.1): its ledger deals the mid and its watermark alone retires the
1813
+ // entry. THE §3 ROUTER RUNS HERE (H-iii) after the prediction (write/read capture is
1814
+ // complete, and the engine holds the committed prediction the probes read) and BEFORE the
1815
+ // mid is dealt: an assigned mid pins its domain forever (§7.1) — a re-invocation never
1816
+ // re-routes. An explicit `domainPolicy` string pins verbatim (no proof); an H-v deopt
1817
+ // re-invocation pins via `pin` (see {@link invokeWith}) and the router never runs.
1818
+ const domain = pin ?? this.resolveDomain(name, args, writes, reads);
1819
+ const { mid, seq } = this.dealMid(domain);
1820
+ // The write-source axes (§5.3): which physical source each write staged onto. All `"daemon"`
1821
+ // in single-domain (Collapsed) — so the filter and hold-back below are inert on the live path.
1822
+ const touchedSources = new Set(sources);
1823
+ const writeSources = new Map<string, string>();
1824
+ mergeWriteSources(writeSources, stagedKeys, sources);
1825
+ this.pendingMutations.push({ mid, seq, name, args, domain, touched, writes, reads, touchedSources, writeSources });
653
1826
  // The prediction stuck — fold its child ops into the optimistic agg delta and push it onto
654
1827
  // the `__agg` head rows (§4). No reset here (this is the §1.3 trivial case, no rewind): the
655
1828
  // delta accumulates on top of the prior pending set, and `reconcileAggHead` recomputes each
@@ -657,7 +1830,7 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
657
1830
  for (const op of ops) this.overlay.observe(op);
658
1831
  this.reconcileAggHead();
659
1832
  this.refreshPending(); // §7.2: this write now touches its queries' pending axis (NOT ResultType).
660
- void this.source.pushMutation({ clientID: this.clientID, mid, name, args });
1833
+ void this.channelFor(domain).pushMutation({ clientID: this.clientID, mid, name, args });
661
1834
  return mid;
662
1835
  });
663
1836
  }
@@ -675,12 +1848,15 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
675
1848
  return this.inOneCommit(() => {
676
1849
  // Apply the prediction with the read trap armed (§5): a folded mutator that reads state to
677
1850
  // compute its write is non-absorbing and refused. A throw discards the staged write (clean
678
- // no-op) and consumes no mid — exactly `invoke`'s guarantee.
679
- const touched = new Set<string>();
1851
+ // no-op) and consumes no mid — exactly `invoke`'s guarantee. NO `readLog` here — the trap
1852
+ // path stays byte-for-byte as it was; recording (§3.2 #2) never arms alongside the trap.
1853
+ const writes: WriteSet = new Map();
680
1854
  const ops: ChildOp[] = [];
1855
+ const stagedKeys: string[] = [];
1856
+ let sources: string[] = [];
681
1857
  try {
682
- this.local.writeWith((tx) => {
683
- this.runMutator(mutator, trackingTx(tx, touched, this.specs, this.localTables, this.opCollector(ops), true), args);
1858
+ sources = this.local.writeWith((tx) => {
1859
+ this.runMutator(mutator, trackingTx(tx, writes, this.specs, this.localTables, this.opCollector(ops), true, undefined, stagedKeys), args);
684
1860
  });
685
1861
  } catch (e) {
686
1862
  if (e instanceof FoldReadError) {
@@ -693,6 +1869,13 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
693
1869
  for (const op of ops) this.overlay.observe(op);
694
1870
  this.reconcileAggHead();
695
1871
 
1872
+ // `touched` is DERIVED, never separately populated (§3.2 #1) — see {@link WriteSet}.
1873
+ const touched = new Set(writes.keys());
1874
+ // The write-source axes (§5.3), like `touched`/`writes` — REPLACED wholesale on an in-place
1875
+ // fold overwrite (the latest invocation supersedes, absorbing), initialized on a new entry.
1876
+ const touchedSources = new Set(sources);
1877
+ const writeSources = new Map<string, string>();
1878
+ mergeWriteSources(writeSources, stagedKeys, sources);
696
1879
  const now = this.clock.now();
697
1880
  let f = this.folds.get(foldKey);
698
1881
  if (f) {
@@ -701,10 +1884,24 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
701
1884
  // only the LATEST args, which is what a rebase re-derives from and what the flush ships.
702
1885
  f.entry.args = args;
703
1886
  f.entry.touched = touched;
1887
+ f.entry.writes = writes;
1888
+ f.entry.touchedSources = touchedSources;
1889
+ f.entry.writeSources = writeSources;
704
1890
  f.args = args;
705
1891
  this.clock.clearTimeout(f.timer);
706
1892
  } else {
707
- const entry: PendingMutation = { mid: null, name, args, touched };
1893
+ // Provisional domain (§7.1): re-resolved from the final args/write-set at flush, when the
1894
+ // mid is dealt. Never read before then — an un-flushed fold (`mid == null`) is
1895
+ // unconditionally retained — so the full (counting) derivation deliberately does NOT run
1896
+ // here (it would double-bump the Q6 counters for one logical route); the flush derives.
1897
+ const domain = this.domainPolicy(name, args) ?? "daemon";
1898
+ // §9.3: pick the window's cadence. Routing into a room ⇒ flush at roomDebounceMs so
1899
+ // intermediates stream to the shared head; off the room, the caller's collapse debounce
1900
+ // governs. The probe restores the Q6 diagnostics (the flush owns the authoritative bump).
1901
+ const inRoom = opts.roomDebounceMs !== undefined && this.routesToRoom(name, args, writes);
1902
+ const debounceMs = inRoom ? opts.roomDebounceMs! : opts.debounceMs ?? DEFAULT_FOLD_DEBOUNCE_MS;
1903
+ const maxWaitMs = inRoom ? opts.roomDebounceMs! : opts.maxWaitMs;
1904
+ const entry: PendingMutation = { mid: null, seq: null, name, args, domain, touched, writes, reads: { reads: [], queries: [] }, touchedSources, writeSources };
708
1905
  this.pendingMutations.push(entry);
709
1906
  let resolveMid!: (mid: number) => void;
710
1907
  const midPromise = new Promise<number>((res) => (resolveMid = res));
@@ -713,8 +1910,8 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
713
1910
  args,
714
1911
  timer: undefined,
715
1912
  firstAt: now,
716
- debounceMs: opts.debounceMs ?? DEFAULT_FOLD_DEBOUNCE_MS,
717
- maxWaitMs: opts.maxWaitMs,
1913
+ debounceMs,
1914
+ maxWaitMs,
718
1915
  deferAcrossWrites: opts.deferAcrossWrites ?? false,
719
1916
  midPromise,
720
1917
  resolveMid,
@@ -754,12 +1951,175 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
754
1951
  if (!f) return;
755
1952
  this.clock.clearTimeout(f.timer);
756
1953
  this.folds.delete(foldKey);
757
- const mid = this.nextMid++;
1954
+ // Resolve the confirming stream from the FINAL args (§7.1) and deal the mid from that domain's
1955
+ // ledger — SEND order, never reserved, so gapless within the domain. Under H-iii derivation
1956
+ // this RE-DERIVES from the entry's LATEST write-set (each in-place fold overwrite replaced it,
1957
+ // so it reflects the final absorbed value): same §3 rules minus reads — a fold's read-set is
1958
+ // provably empty by construction (the FoldReadError trap, not recording, arms on that path),
1959
+ // which is the STRONGEST read proof there is. The mid dealt below then pins this domain.
1960
+ const domain = this.resolveDomain(f.entry.name, f.args, f.entry.writes, f.entry.reads);
1961
+ f.entry.domain = domain;
1962
+ const { mid, seq } = this.dealMid(domain);
758
1963
  f.entry.mid = mid;
759
- void this.source.pushMutation({ clientID: this.clientID, mid, name: f.entry.name, args: f.args });
1964
+ f.entry.seq = seq;
1965
+ void this.channelFor(domain).pushMutation({ clientID: this.clientID, mid, name: f.entry.name, args: f.args });
760
1966
  f.resolveMid(mid);
761
1967
  }
762
1968
 
1969
+ /** The transport a `domain`-confirmed mutation ships on (§7.5 sent-pins-domain: only the
1970
+ * domain's own authority can confirm it, so its channel is the only correct transport). A
1971
+ * domain with NO connected gate ships on the daemon channel — the gate-less configurations
1972
+ * (`__testRelease`-driven tests) and today's entire live path resolve `"daemon"` anyway. */
1973
+ private channelFor(domain: string): OptimisticSource {
1974
+ return (this.gates.get(domain) ?? this.daemonGate).source;
1975
+ }
1976
+
1977
+ /** The gate a channel-keyed retain registers through (G-iii registration-time routing). The
1978
+ * channel MUST already be connected (`connectSource`; the daemon is constructor-attached) —
1979
+ * loud by design: a typo'd or not-yet-connected sourceKey must throw at retain time, never
1980
+ * silently register on the daemon and split the query's frames across channels. */
1981
+ private requireGate(channel: string): SourceGate {
1982
+ const gate = this.gates.get(channel);
1983
+ if (!gate) {
1984
+ throw new Error(
1985
+ `optimistic backend: no source connected for channel ${JSON.stringify(channel)} — call connectSource(${JSON.stringify(channel)}, source) before retaining a query on it`,
1986
+ );
1987
+ }
1988
+ return gate;
1989
+ }
1990
+
1991
+ /** The channel that owns `sourceQid` — {@link RemoteSub.channel}, the ONE source of truth for
1992
+ * qid routing (G-iii). `undefined` when no sub owns the qid (a harness-delivered raw feed, or
1993
+ * a just-released sub): such frames buffer on whatever gate they arrive at. */
1994
+ private channelOf(sourceQid: QueryId): string | undefined {
1995
+ const key = this.sourceToRemote.get(sourceQid);
1996
+ return key ? this.remoteSubs.get(key)?.channel : undefined;
1997
+ }
1998
+
1999
+ // --- the §3.3 deopt handshake, client half (H-v) ---------------------------------
2000
+ //
2001
+ // THE NAMED INVARIANT (Slice I inherits it): **never retire a room-domain entry off a
2002
+ // daemon-carried lmid without outcome resolution.** On the room socket it holds by
2003
+ // construction: every room lmid folds through the room's OWN gate, whose socket also carries
2004
+ // the outcome frames — same-socket ordering puts the frame before the ack, and the reconnect
2005
+ // re-send re-earns a lost frame, so a room-domain entry is only ever retired as a success when
2006
+ // the room really applied it. Slice I's downgrade path breaks that coupling: the doc-scoped
2007
+ // ledger row becomes readable THROUGH THE DAEMON with no room socket alive (§7.1 "load-bearing
2008
+ // for §7.5"), and an lmid adopted that way covers burnt non-applied mids with no frame to say
2009
+ // so — retiring a deopted entry there as a silent success is exactly the lost-write this
2010
+ // handshake exists to prevent. ENFORCED since I-iii by {@link foldSystemFrames}: the I-ii
2011
+ // outcome ROWS (co-committed, in ONE daemon transaction, with the ledger row that covers them)
2012
+ // are synthesized into frames and routed through THIS machine BEFORE the ledger fold advances
2013
+ // the domain watermark — one verdict path for frames and rows, with the processed set as the
2014
+ // cross-release resolved-verdict memory, and absence-under-a-covering-lmid = applied (I-ii's
2015
+ // atomicity makes that the sound default).
2016
+
2017
+ /** Record `(domain, mid)` as processed; `false` if it already was (a duplicate frame —
2018
+ * ignore it). FIFO-capped per domain ({@link MAX_PROCESSED_OUTCOMES_PER_DOMAIN}). */
2019
+ private markOutcomeProcessed(domain: string, mid: number): boolean {
2020
+ let mids = this.outcomesProcessed.get(domain);
2021
+ if (!mids) this.outcomesProcessed.set(domain, (mids = new Set()));
2022
+ if (mids.has(mid)) return false;
2023
+ mids.add(mid);
2024
+ while (mids.size > MAX_PROCESSED_OUTCOMES_PER_DOMAIN) {
2025
+ mids.delete(mids.values().next().value as number);
2026
+ }
2027
+ return true;
2028
+ }
2029
+
2030
+ /** One `mutationOutcome` frame from `domain`'s channel (H-v — the §3.3 handshake's client
2031
+ * half). The frame arrives OUT-OF-BAND (see {@link attachGate}); the state machine:
2032
+ *
2033
+ * 1. `mid` never issued on `domain` ⇒ ignore (a confused/foreign frame must not invent work).
2034
+ * 2. `(domain, mid)` already processed ⇒ ignore — idempotence under duplicate frames (the
2035
+ * original + a re-send's re-answer; a deopt for a mid whose entry ALREADY FLIPPED also
2036
+ * lands here harmlessly on its second frame).
2037
+ * 3. `kind:"rejected"` ⇒ FINAL. Surface the reason through {@link rejectedHandler} (room-plane
2038
+ * parity with the HTTP queue's callback) and STOP — the drop + snap-back is the EXISTING
2039
+ * failed-mutation machinery: the room burnt the mid, its lmid release retires the entry
2040
+ * per-domain and the reconcile rewinds the prediction, exactly the daemon path's
2041
+ * processed-as-no-op rejection. No new drop path.
2042
+ * 4. `kind:"deopt"`, entry found (pending `(domain, mid)`) ⇒ FLIP IN PLACE: `domain` becomes
2043
+ * `"daemon"`, a fresh daemon mid is dealt and the envelope ships NOW on the daemon channel
2044
+ * ("deal-and-send-now" — the conforming §3.3 re-enqueue: there is no flush machinery for
2045
+ * non-fold entries, so the design's "mid: null until the daemon flush" is satisfied
2046
+ * momentarily inside this call). THE ENTRY'S `seq` IS KEPT — settled (§5.3, commit
2047
+ * 68141096): `seq` is the client-global REPLAY order; re-sequencing would move the entry's
2048
+ * overlay position and change read-dependent SIBLINGS' replay base. Everything else stays
2049
+ * (writes/reads/touched/touchedSources/writeSources — union-never-shrink), the prediction
2050
+ * stays applied (the entry never leaves `pendingMutations`, so no rewind fires), and the
2051
+ * router does NOT re-run nor does `drainOverlapping` (§3.3 re-enqueues, never re-derives;
2052
+ * any open overlapping fold was invoked later and flushes later with a larger mid).
2053
+ * 5. `kind:"deopt"`, entry NOT found ⇒ the burnt-mid confirm won the race, or the frame is a
2054
+ * replay re-answer for an entry a previous session retired (the replay gotcha): re-invoke
2055
+ * the frame's echoed `name`/`args` as a FRESH invocation PINNED to `"daemon"` — an honest
2056
+ * re-prediction on the current base, never a derived route ({@link invokeWith}). A frame
2057
+ * without `name` (not self-contained) has nothing to re-invoke and is dropped; a re-invoke
2058
+ * that THROWS (the base moved from under it) is surfaced through {@link rejectedHandler} —
2059
+ * the mutation is dead with no stream left to confirm it.
2060
+ *
2061
+ * A `"deopt"` bump joins the Q6 routing counters either way (`routing.reasons.deopt`) —
2062
+ * derived-and-deopted routes are visible beside derived successes. */
2063
+ private handleMutationOutcome(domain: string, frame: MutationOutcomeFrame): void {
2064
+ if (frame.mid >= (this.nextMid.get(domain) ?? 1)) return; // never issued here — not ours
2065
+ if (!this.markOutcomeProcessed(domain, frame.mid)) return; // duplicate frame
2066
+ if (frame.kind === "rejected") {
2067
+ const entry = this.pendingMutations.find((p) => p.domain === domain && p.mid === frame.mid);
2068
+ this.rejectedHandler(
2069
+ {
2070
+ clientID: this.clientID,
2071
+ mid: frame.mid,
2072
+ name: entry?.name ?? frame.name ?? "",
2073
+ args: entry !== undefined ? entry.args : frame.args,
2074
+ },
2075
+ frame.reason ?? "mutation rejected",
2076
+ );
2077
+ return;
2078
+ }
2079
+ // kind === "deopt": Q6's completion — how often a routed mutation came back refused.
2080
+ this.routingReasons.set("deopt", (this.routingReasons.get("deopt") ?? 0) + 1);
2081
+ const entry = this.pendingMutations.find((p) => p.domain === domain && p.mid === frame.mid);
2082
+ if (entry) {
2083
+ entry.domain = "daemon";
2084
+ // Deal the fresh daemon mid but DISCARD its seq — the entry keeps its own (state-machine
2085
+ // step 4 above; the harmless dealSeq bump is accepted). This is the ONE place a dealt seq
2086
+ // is dropped, so "within one domain seq order == mid order" weakens to "except deopt
2087
+ // re-enqueues" — see the {@link PendingMutation.seq} doc.
2088
+ const { mid } = this.dealMid("daemon");
2089
+ entry.mid = mid;
2090
+ void this.channelFor("daemon").pushMutation({ clientID: this.clientID, mid, name: entry.name, args: entry.args });
2091
+ return;
2092
+ }
2093
+ if (frame.name === undefined) return; // not self-contained — nothing to re-invoke
2094
+ try {
2095
+ this.invokeWith(frame.name, frame.args, "daemon");
2096
+ } catch (err) {
2097
+ this.rejectedHandler(
2098
+ { clientID: this.clientID, mid: frame.mid, name: frame.name, args: frame.args },
2099
+ `deopt re-invocation failed: ${String((err as Error)?.message ?? err)}`,
2100
+ );
2101
+ }
2102
+ }
2103
+
2104
+ /** §7.5 rule 3 (H-v): re-send `domain`'s unconfirmed pending envelopes with their ORIGINAL
2105
+ * mids, in mid order, on the domain's own channel. Folds with `mid === null` are excluded —
2106
+ * nothing was ever sent for them (the flush deals their mid). Envelopes are reconstructed from
2107
+ * the pending entries exactly as `invoke` shipped them (`clientID`/`mid`/`name`/`args` —
2108
+ * entries carry everything the wire needs). Idempotent under the domain's ledger: an APPLIED
2109
+ * mid dedups silently and its lmid coverage retires the entry; a NON-APPLIED mid is re-answered
2110
+ * from the shell's recorded-outcome map into {@link handleMutationOutcome}. Confirmed entries
2111
+ * are already gone from `pendingMutations`, so no filter against the watermark is needed. */
2112
+ private resendPending(domain: string): void {
2113
+ const unconfirmed = this.pendingMutations
2114
+ .filter((p) => p.domain === domain && p.mid !== null)
2115
+ .sort((a, b) => a.mid! - b.mid!);
2116
+ if (unconfirmed.length === 0) return;
2117
+ const channel = this.channelFor(domain);
2118
+ for (const p of unconfirmed) {
2119
+ void channel.pushMutation({ clientID: this.clientID, mid: p.mid!, name: p.name, args: p.args });
2120
+ }
2121
+ }
2122
+
763
2123
  /** Drain every outstanding fold immediately (FOLDED-MUTATIONS-DESIGN §3): the explicit
764
2124
  * `app.flushFolds()` and the `beforeunload`/`close` hook. Creation (insertion) order. */
765
2125
  flushFolds(): void {
@@ -858,7 +2218,17 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
858
2218
  const pending: PendingInspect[] = this.pendingMutations.map((p) => {
859
2219
  const folded = foldByEntry.get(p);
860
2220
  const key = p.mid != null ? `m:${p.mid}` : folded ? `f:${folded[0]}` : `?:${p.name}`;
861
- const out: PendingInspect = { key, mid: p.mid, name: p.name, args: p.args, tables: [...p.touched] };
2221
+ const writes: WriteRecord[] = [];
2222
+ for (const byPk of p.writes.values()) for (const rec of byPk.values()) writes.push(rec);
2223
+ const out: PendingInspect = {
2224
+ key,
2225
+ mid: p.mid,
2226
+ name: p.name,
2227
+ args: p.args,
2228
+ tables: [...p.touched],
2229
+ writes,
2230
+ reads: { reads: [...p.reads.reads], queries: [...p.reads.queries] },
2231
+ };
862
2232
  if (folded) {
863
2233
  const f = folded[1];
864
2234
  out.fold = {
@@ -873,14 +2243,119 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
873
2243
  });
874
2244
  return {
875
2245
  pending,
876
- confirmedLmid: this.confirmedLmid,
877
- nextMid: this.nextMid,
878
- appliedCv: this.appliedCv,
879
- bufferedFrames: this.buffer.length,
2246
+ // Back-compat scalars for the devtools `OptimisticInspect` mirror (unchanged shape): the DAEMON
2247
+ // domain's ledger — the only one in single-domain. Per-domain state is `__inspectDomains()`.
2248
+ confirmedLmid: this.watermark.get("daemon") ?? 0,
2249
+ nextMid: this.nextMid.get("daemon") ?? 1,
2250
+ appliedCv: this.daemonGate.appliedCv,
2251
+ bufferedFrames: this.daemonGate.buffer.length,
880
2252
  pendingTables: [...this.pendingTables()],
881
2253
  };
882
2254
  }
883
2255
 
2256
+ /** Test-only per-domain ledger snapshot (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §7.1/§8.5).
2257
+ * Kept separate from {@link __inspect} so the devtools `OptimisticInspect` mirror stays byte-for-
2258
+ * byte identical (daemon-scalar-only). Exposes the per-domain `nextMid`/`watermark` maps plus each
2259
+ * pending entry's confirming domain — the axes the §8.5 ledger-isolation assertion checks. */
2260
+ __inspectDomains(): {
2261
+ nextMid: Record<string, number>;
2262
+ watermark: Record<string, number>;
2263
+ /** Per connected CHANNEL (§5.1): its release watermark + buffered-frame depth — the axis the
2264
+ * gate-isolation assertions read (one source's laggy cvMin must never move the other's) —
2265
+ * plus the §301 upstream-absorption advert last recorded from its progress frames (a ROOM
2266
+ * gate with a new shell; absent otherwise). */
2267
+ gates: Record<string, { appliedCv: number; bufferedFrames: number; upstreamCv?: number; upstreamBoot?: string }>;
2268
+ /** The §3 router's counters (H-iii) — Q6's measurement hook: how often derivation succeeded
2269
+ * (per derived room) and why it fell to the daemon (per reason). See {@link RoutingInspect}
2270
+ * for the counting discipline (per-candidate failures; pins bump nothing). */
2271
+ routing: RoutingInspect;
2272
+ /** The §4 lifecycle plane's folded state (Slice I-iii introspection): the per-doc §4.2 fence
2273
+ * value (`roomWatermarks`, I-v's ghost-drop input), the per-scope §4.1 occupancy map
2274
+ * (`scopeSessions`: scope → client_id → expires_at, I-iv's doorbell input), and the live
2275
+ * I-v ghosts (demoted room sources still awaiting their fence). */
2276
+ lifecycle: {
2277
+ roomWatermarks: Record<string, number>;
2278
+ scopeSessions: Record<string, Record<string, number>>;
2279
+ ghosts: Record<string, { doc: string; finalFlushSeq: number; tables: string[] }>;
2280
+ };
2281
+ /** The live §301 pin registry (`301-ECHO-FENCE-DESIGN.md` §2.5 — the devtools/Q6 surface):
2282
+ * every parked hold-back's fence inputs plus its tripwire state, keyed
2283
+ * `${table}\0${sourceKey}\0${pkKey}`. Pruned lazily, so an entry here may briefly outlive
2284
+ * its engine pin (never the reverse). */
2285
+ pins: Record<
2286
+ string,
2287
+ {
2288
+ table: string;
2289
+ sourceKey: string;
2290
+ domain: string;
2291
+ mid: number;
2292
+ daemonBoot?: string;
2293
+ daemonCv?: number;
2294
+ tripwired: boolean;
2295
+ }
2296
+ >;
2297
+ /** The §301 direction-A fence map: per room domain, the highest DAEMON-CARRIED ledger lmid. */
2298
+ daemonCarriedLmid: Record<string, number>;
2299
+ pending: { mid: number | null; seq: number | null; name: string; domain: string; touchedSources: string[]; writeSources: Record<string, string> }[];
2300
+ } {
2301
+ return {
2302
+ nextMid: Object.fromEntries(this.nextMid),
2303
+ watermark: Object.fromEntries(this.watermark),
2304
+ gates: Object.fromEntries(
2305
+ [...this.gates].map(([k, g]) => [
2306
+ k,
2307
+ {
2308
+ appliedCv: g.appliedCv,
2309
+ bufferedFrames: g.buffer.length,
2310
+ ...(g.upstreamCv !== undefined ? { upstreamCv: g.upstreamCv } : {}),
2311
+ ...(g.upstreamBoot !== undefined ? { upstreamBoot: g.upstreamBoot } : {}),
2312
+ },
2313
+ ]),
2314
+ ),
2315
+ routing: {
2316
+ derived: Object.fromEntries(this.routingDerived),
2317
+ reasons: Object.fromEntries(this.routingReasons),
2318
+ },
2319
+ lifecycle: {
2320
+ roomWatermarks: Object.fromEntries(this.roomWatermarks),
2321
+ scopeSessions: Object.fromEntries(
2322
+ [...this.scopeSessions].map(([scope, sessions]) => [scope, Object.fromEntries(sessions)]),
2323
+ ),
2324
+ ghosts: Object.fromEntries(
2325
+ [...this.ghosts].map(([k, g]) => [k, { doc: g.doc, finalFlushSeq: g.finalFlushSeq, tables: [...g.tables] }]),
2326
+ ),
2327
+ },
2328
+ pins: Object.fromEntries(
2329
+ [...this.pins].map(([k, p]) => [
2330
+ k,
2331
+ {
2332
+ table: p.table,
2333
+ sourceKey: p.sourceKey,
2334
+ domain: p.domain,
2335
+ mid: p.mid,
2336
+ ...(p.daemonBoot !== undefined ? { daemonBoot: p.daemonBoot } : {}),
2337
+ ...(p.daemonCv !== undefined ? { daemonCv: p.daemonCv } : {}),
2338
+ tripwired: p.tripwired === true,
2339
+ },
2340
+ ]),
2341
+ ),
2342
+ daemonCarriedLmid: Object.fromEntries(this.daemonCarriedLmid),
2343
+ pending: this.pendingMutations.map((p) => ({
2344
+ mid: p.mid,
2345
+ // The client-global deal sequence — the REPLAY order (mids are per-domain, incomparable
2346
+ // across domains; see PendingMutation.seq). The harness asserts send order with this.
2347
+ seq: p.seq,
2348
+ name: p.name,
2349
+ domain: p.domain,
2350
+ // The write-source axes (§5.3): which physical source(s) this entry's writes routed onto —
2351
+ // the filter basis, and (per pk) the hold-back's cross-slice test. The harness asserts the
2352
+ // filter/routing against its independent model with these.
2353
+ touchedSources: [...p.touchedSources],
2354
+ writeSources: Object.fromEntries(p.writeSources),
2355
+ })),
2356
+ };
2357
+ }
2358
+
884
2359
  /** Recompute the pending axis for every query and fire `onPending` on transitions only. Called
885
2360
  * from the two points that move the pending set: invoke/invokeFolded (add) and the confirm-drop
886
2361
  * (remove) — exactly where `:359`/`:468` used to flip ResultType (§7.3). */
@@ -894,18 +2369,38 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
894
2369
  }
895
2370
  }
896
2371
 
897
- // --- the downstream stream (§8.5: buffer, then release coherently) ---------------
898
-
899
- private onNormalized(qid: QueryId, ev: NormalizedEvent): void {
900
- if (qid !== LMID_QID) this.emitServerDelta(qid, ev);
2372
+ // --- the downstream stream (§8.5: buffer, then release coherently — PER GATE, §5.1) ------
2373
+
2374
+ private onFrame(gate: SourceGate, qid: QueryId, ev: NormalizedEvent): void {
2375
+ // Ownership is fixed at RETAIN time (G-iii registration-time routing: {@link RemoteSub.channel},
2376
+ // the one source of truth) — a qid lives on the ONE channel its sub registered on. So a frame
2377
+ // arriving on any OTHER gate means the server routed a qid to the wrong channel: a wiring bug —
2378
+ // fail loudly rather than silently splitting one query's frames across two cv timelines. (This
2379
+ // used to be a lazy first-arrival CLAIM; it is now a pure assertion.) A qid with NO sub (a
2380
+ // harness-delivered raw feed) has no owner and buffers on the arriving gate; the per-channel
2381
+ // reserved LMID_QID is exempt — each gate owns its own.
2382
+ if (qid !== LMID_QID) {
2383
+ const owner = this.channelOf(qid);
2384
+ if (owner !== undefined && owner !== gate.key) {
2385
+ // I-iv retarget grace: a frame already in flight from the sub's PREVIOUS channel when
2386
+ // {@link retargetRemoteQuery} moved it (the unsubscribe races the server's last frames)
2387
+ // is stale, not a wiring bug — drop it. The grace window is exactly the deferred-GC
2388
+ // window: {@link flushRetargetGc} deletes the record, and the loud throw is restored.
2389
+ if (this.pendingRetargetGc.get(qid) === gate.key) return;
2390
+ throw new Error(`optimistic backend: qid ${qid} arrived on ${gate.key} but is owned by ${owner}`);
2391
+ }
2392
+ }
2393
+ // System-plane frames (I-iii) are bookkeeping, not view data: like the lmid stream they skip
2394
+ // the devtools server-delta tap (there is no local view to attribute them to).
2395
+ if (qid !== LMID_QID && !this.systemQids.has(qid)) this.emitServerDelta(qid, ev);
901
2396
  if (ev.type === "hello") {
902
2397
  // A hello is a (re)subscribe = a NEW epoch. The column map below is mutated eagerly, but
903
- // data frames are cv-buffered and drained later (onProgress). So any frame still buffered
904
- // for this qid is from a SUPERSEDED epoch and must NOT be scattered through this epoch's
905
- // (possibly changed) map — drop it. This epoch's snapshot, which always follows the hello,
906
- // re-hydrates the qid from scratch, so the dropped frames are redundant. Scoped to this
907
- // qid: other queries' frames (and the lmid system query's) keep their coherent release.
908
- this.buffer = this.buffer.filter((f) => f.qid !== qid);
2398
+ // data frames are cv-buffered and drained later (the gate's progress). So any frame still
2399
+ // buffered for this qid is from a SUPERSEDED epoch and must NOT be scattered through this
2400
+ // epoch's (possibly changed) map — drop it. This epoch's snapshot, which always follows the
2401
+ // hello, re-hydrates the qid from scratch, so the dropped frames are redundant. Scoped to
2402
+ // this qid: other queries' frames (and the lmid system query's) keep their coherent release.
2403
+ gate.buffer = gate.buffer.filter((f) => f.qid !== qid);
909
2404
  this.addServerDependencyTables(qid, ev.tables.map((t) => t.name));
910
2405
  // Learn this query's per-table column map (PROJECTION-SUPPORT-DESIGN.md §5.2): map each
911
2406
  // advertised column to its base ColId BY NAME. The hello may carry FEWER columns than the
@@ -923,15 +2418,15 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
923
2418
  // Register a non-trivial map; otherwise revert to '*' — and CLEAR any stale map a prior
924
2419
  // epoch left (a server that expanded then contracted back), so the now-exact rows don't
925
2420
  // scatter through a `-1`-bearing layout (silent cell corruption).
926
- if (cols.length !== full || cols.some((c, i) => c !== i)) this.sync.registerProjection(qid, t.name, cols);
927
- else this.sync.unregisterProjection(qid, t.name);
2421
+ if (cols.length !== full || cols.some((c, i) => c !== i)) gate.sync.registerProjection(qid, t.name, cols);
2422
+ else gate.sync.unregisterProjection(qid, t.name);
928
2423
  }
929
2424
  return; // envelope validation is the source's job
930
2425
  }
931
2426
  const cv = ev.cv ?? 0;
932
- if (cv <= this.appliedCv && ev.type === "batch") return; // stale redelivery
933
- this.buffer.push({ cv, qid, kind: ev.type, ops: ev.ops, seq: this.nextSeq++ });
934
- if (this.buffer.length > this.bufferCap) this.overflow();
2427
+ if (cv <= gate.appliedCv && ev.type === "batch") return; // stale redelivery ON THIS TIMELINE
2428
+ gate.buffer.push({ cv, qid, kind: ev.type, ops: ev.ops, seq: gate.nextSeq++ });
2429
+ if (gate.buffer.length > this.bufferCap) this.overflow(gate);
935
2430
  }
936
2431
 
937
2432
  private emitServerDelta(sourceQid: QueryId, ev: NormalizedEvent): void {
@@ -949,75 +2444,577 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
949
2444
  return localQids.length ? localQids : [sourceQid];
950
2445
  }
951
2446
 
952
- private onProgress(frame: ProgressFrame): void {
953
- // (1) Take every buffered frame at cv cvMin, in (cv, arrival) order, and fold it:
954
- // lmid system-query frames advance `confirmedLmid`; data frames fold through the
955
- // cross-query refcount into ONE net base delta the §1.3 `D`. Confirmation and
956
- // data of the same commit share a cv, so they release together by construction.
957
- const ready = this.buffer
2447
+ /** One gate's release (§5.1 release gate): compute the coherent delta from THIS gate's cv-buffer,
2448
+ * then apply it against the gate's source/domain. Split into {@link computeRelease} (buffer
2449
+ * delta, lmid watermark) and {@link applyRelease} (per-source confirm-drop + reconcile) —
2450
+ * N independent gates all feed the ONE apply half; {@link __testRelease} drives it directly. */
2451
+ private onGateProgress(gate: SourceGate, frame: ProgressFrame): void {
2452
+ // §301 direction B: record the frame's upstream-absorption advert BEFORE the release applies,
2453
+ // so the drop pass at applyRelease's tail evaluates exactly this release's advert — the room
2454
+ // emits data-then-progress on one socket, so by the time the advert says "absorbed through
2455
+ // cv U" the re-published echo data is in this release's fold (never ahead of it). Verbatim,
2456
+ // not max-folded: on a daemon restart the advertised cv space legitimately resets and the
2457
+ // §2.4 boot rule (not monotonicity) carries the ordering.
2458
+ if (frame.upstreamCv !== undefined) {
2459
+ gate.upstreamCv = frame.upstreamCv;
2460
+ gate.upstreamBoot = frame.upstreamBoot;
2461
+ }
2462
+ const { deltas, newlyHydrated, touchedScopes } = this.computeRelease(gate, frame);
2463
+ this.applyRelease(gate.key, deltas, undefined, newlyHydrated);
2464
+ // I-iv phase 2: a retargeted sub whose first ROOM snapshot released just now gets its old
2465
+ // channel's rows GC'd — AFTER the release fully applied, so the winner flip is value-equal
2466
+ // against the freshly-folded room rows (never a remove-before-the-refill).
2467
+ this.flushRetargetGc(gate);
2468
+ // I-iv doorbell events LAST — everything this release carried (data, confirms, the occupancy
2469
+ // fold itself, the retarget cutover) is already applied when the consumer's reaction (an
2470
+ // async re-lease) is kicked off. One event per touched scope, count evaluated at the fold
2471
+ // clock's now (deterministic under an injected clock).
2472
+ if (touchedScopes !== null) {
2473
+ for (const scope of touchedScopes) {
2474
+ this.scopeSessionsHandler({ scope, others: this.otherScopeSessions(scope) });
2475
+ }
2476
+ }
2477
+ }
2478
+
2479
+ /** Compute one coherent release from ONE gate's cv-buffer (§5.1) — gate-scoped: its buffer, its
2480
+ * cvMin timeline. Take every buffered frame at `cv ≤ cvMin`, in (cv, arrival) order, and fold
2481
+ * it: the lmid system-query frame advances `watermark[gate.key]` (via {@link foldLmidOps} — the
2482
+ * daemon stream folds "daemon", a room stream folds its own domain); data frames fold through
2483
+ * this SOURCE's cross-query refcount into ONE net base delta — the §1.3 `D`. Returns that delta
2484
+ * plus the set of local views this release JUST hydrated (so their reconcile batch phases as a
2485
+ * `snapshot`). Mutates the gate's buffer/`appliedCv`, hydration, and the gate's domain
2486
+ * watermark; the pending set and the reconcile are {@link applyRelease}'s job. */
2487
+ private computeRelease(
2488
+ gate: SourceGate,
2489
+ frame: ProgressFrame,
2490
+ ): { deltas: Mutation[]; newlyHydrated: Set<QueryId> | null; touchedScopes: Set<string> | null } {
2491
+ // Snapshot which local views are already hydrated BEFORE this release folds: any that cross into
2492
+ // hydrated below get their first result set as this cycle's batch, which must phase as a snapshot.
2493
+ const wasHydrated = new Set(this.hydrated);
2494
+ const ready = gate.buffer
958
2495
  .filter((f) => f.cv <= frame.cvMin)
959
2496
  .sort((a, b) => a.cv - b.cv || a.seq - b.seq);
960
- this.buffer = this.buffer.filter((f) => f.cv > frame.cvMin);
2497
+ gate.buffer = gate.buffer.filter((f) => f.cv > frame.cvMin);
2498
+ // The §4 lifecycle SYSTEM frames fold FIRST, in a FIXED structural category order (Slice
2499
+ // I-iii; see {@link foldSystemFrames} for why the order is load-bearing), then the ordinary
2500
+ // lmid + data frames fold exactly as before. With no system retain the partition is empty and
2501
+ // this release is byte-identical to pre-I-iii. The returned scope set feeds the I-iv doorbell
2502
+ // events `onGateProgress` fires once the WHOLE release has applied.
2503
+ const touchedScopes = this.foldSystemFrames(ready.filter((f) => this.systemQids.has(f.qid)));
961
2504
  const muts: Mutation[] = [];
962
2505
  for (const f of ready) {
2506
+ if (this.systemQids.has(f.qid)) {
2507
+ // Folded above; a system stream has no store view and MUST NOT enter the sync layer (its
2508
+ // tables are not in the schema) — but its first snapshot still marks the sub hydrated so
2509
+ // the overflow/introspection bookkeeping stays uniform.
2510
+ if (f.kind === "snapshot") this.markSubHydrated(f.qid);
2511
+ continue;
2512
+ }
963
2513
  if (f.qid === LMID_QID) {
964
- this.foldLmidOps(f.ops);
2514
+ // Confirmation and data of the same commit share a cv, so they release together — each
2515
+ // channel's lmid stream folds into ITS OWN domain's watermark (§7.1).
2516
+ this.foldLmidOps(f.ops, gate.key);
965
2517
  continue;
966
2518
  }
967
2519
  muts.push(
968
- ...(f.kind === "snapshot" ? this.sync.rehydrate(f.qid, f.ops) : this.sync.applyBatch(f.qid, f.ops)),
2520
+ ...(f.kind === "snapshot" ? gate.sync.rehydrate(f.qid, f.ops) : gate.sync.applyBatch(f.qid, f.ops)),
969
2521
  );
970
2522
  // A query's first released snapshot is its hydration point — even an empty one (0 rows is an
971
2523
  // authoritative answer): lift every local view this sub feeds out of `unknown` (loading).
972
2524
  if (f.kind === "snapshot") this.markSubHydrated(f.qid);
973
2525
  }
974
- this.appliedCv = Math.max(this.appliedCv, frame.cvMin);
2526
+ gate.appliedCv = Math.max(gate.appliedCv, frame.cvMin);
2527
+ let newlyHydrated: Set<QueryId> | null = null;
2528
+ for (const qid of this.hydrated) {
2529
+ if (!wasHydrated.has(qid)) (newlyHydrated ??= new Set()).add(qid);
2530
+ }
2531
+ return { deltas: muts, newlyHydrated, touchedScopes };
2532
+ }
975
2533
 
976
- // (2) Drop confirmed pending (§1.3 step 5's bookkeeping half): mid the lmid this
977
- // release itself delivered. A failed mutation drops the same way — the release just
978
- // carries no effects for it, so the rewind in (3) snaps the prediction back. An UNFLUSHED
979
- // folded entry (`mid == null`) is never confirmable it has not crossed the wire — so it is
980
- // always retained until its own flush stamps a real mid (FOLDED-MUTATIONS-DESIGN §4.1).
2534
+ /** Apply one released delta against `sourceKey`'s domain (§7.2 per-domain confirm-drop + the §1.3
2535
+ * reconcile cycle). `watermarkUpdate`, when given, advances `watermark[sourceKey]` first — the
2536
+ * hook a per-source lmid confirm rides on (the daemon path folds its watermark in
2537
+ * {@link computeRelease} and passes `undefined`). Then: drop every pending entry its OWN domain's
2538
+ * watermark now covers (a room confirm can never retire a daemon entry, and vice-versa — the §7.1
2539
+ * ledger-collision fix), and run the reconcile cycle against `sourceKey` when the base delta or the
2540
+ * pending set changed. `newlyHydrated` stamps the initial-hydration batch as a catch-up. */
2541
+ private applyRelease(
2542
+ sourceKey: string,
2543
+ deltas: Mutation[],
2544
+ watermarkUpdate?: number,
2545
+ newlyHydrated: Set<QueryId> | null = null,
2546
+ ): void {
2547
+ if (watermarkUpdate !== undefined) {
2548
+ this.watermark.set(sourceKey, Math.max(this.watermark.get(sourceKey) ?? 0, watermarkUpdate));
2549
+ }
2550
+ // Echo hold-back (§7.3), BEFORE the confirm-drop: an entry `p` this release CONFIRMS
2551
+ // (`p.mid <= watermark[p.domain]`) but which staged a write onto a DIFFERENT source `S` (a
2552
+ // cross-slice write — e.g. a room-domain mutation whose write routed to the daemon slice) is
2553
+ // about to leave `pendingMutations`, so it will NOT be re-invoked on `S`'s next rewind. Park each
2554
+ // such write inertly on `S` so that rewind (which un-applies the not-re-invoked write) does not
2555
+ // flash-revert it. Two variants, one per write shape (G-i unified them engine-side):
2556
+ // - ADD/EDIT (`rec.row` present): pinned PRESENCE — drops when `S`'s confirmed baseline
2557
+ // value-matches (its echo lands — the E-i-tested drop).
2558
+ // - REMOVE (`rec.row === undefined`, G-iii): pinned ABSENCE — the §7.3 tombstone, parked with
2559
+ // the captured pre-image (`rec.oldRow`) so `S`'s rewind cannot resurrect the deleted row;
2560
+ // drops when `S`'s baseline lacks the pk (the delete echoed).
2561
+ // Inert in single-domain: every write's source equals its domain.
2562
+ for (const p of this.pendingMutations) {
2563
+ if (p.mid === null || p.mid > (this.watermark.get(p.domain) ?? 0)) continue; // not confirmed now
2564
+ for (const [table, byPk] of p.writes) {
2565
+ // The Collapsed-park gate (G-iii): NEVER park on an un-promoted table. Inert-anyway today
2566
+ // (`rewind_collapsed` never consults `held_back`), so skipping is behaviorally identical —
2567
+ // but a stale Collapsed-era entry would become overlay-first-visible (a permanent pin) if
2568
+ // the table later promotes. Slice H's prove-or-slow-path routing makes the real scenario
2569
+ // (a room-confirmed mutation writing a Collapsed table) impossible; this covers until then.
2570
+ if (!this.promotedTables.has(table)) continue;
2571
+ for (const [pkKey, rec] of byPk) {
2572
+ const s = p.writeSources.get(writeSourceKey(table, pkKey));
2573
+ if (s === undefined || s === p.domain) continue; // same-slice: the rewind replays it
2574
+ if (rec.row !== undefined) this.local.holdBack(table, s, rec.row);
2575
+ else if (rec.oldRow !== undefined) this.local.holdBackAbsent(table, s, rec.oldRow);
2576
+ else continue;
2577
+ // §301 §2.1: register the pin's delivery-fence inputs in the same breath as the park.
2578
+ this.registerPin(table, s, pkKey, rec.row ?? (rec.oldRow as WireValue[]), p);
2579
+ }
2580
+ }
2581
+ }
2582
+ // Drop confirmed pending (§1.3 step 5's bookkeeping half), PER DOMAIN: an entry is retired only
2583
+ // when ITS domain's watermark reaches its mid — so two concurrent streams never alias one counter
2584
+ // (§7.1). A failed mutation drops the same way (the release carries no effects, so the rewind snaps
2585
+ // the prediction back). An UNFLUSHED fold (`mid == null`) is never confirmable — retained until its
2586
+ // flush stamps a real mid (FOLDED-MUTATIONS-DESIGN §4.1), regardless of any domain's watermark.
2587
+ // H-v NOTE — retiring here treats coverage as SUCCESS, which for a room domain is sound only
2588
+ // because outcome resolution ALWAYS precedes the coverage that retires: on the room socket
2589
+ // the outcome frames outrun the lmid acks (same-socket ordering + the resync re-send), and on
2590
+ // the daemon-carried path (I-iii) `foldSystemFrames` routes the co-committed outcome ROWS
2591
+ // through handleMutationOutcome BEFORE the ledger fold advances the watermark this filter
2592
+ // reads — a deopted entry has already flipped off the domain by the time its burnt mid is
2593
+ // covered, either way (the named invariant above handleMutationOutcome).
981
2594
  const before = this.pendingMutations.length;
982
- this.pendingMutations = this.pendingMutations.filter((p) => p.mid === null || p.mid > this.confirmedLmid);
2595
+ this.pendingMutations = this.pendingMutations.filter(
2596
+ (p) => p.mid === null || p.mid > (this.watermark.get(p.domain) ?? 0),
2597
+ );
983
2598
  const pendingChanged = this.pendingMutations.length !== before;
984
2599
 
985
- // (3) The reconcile cycle — only when something can have changed: a base delta to
986
- // fold in, or a pending set that shrank (its optimistic layer must rewind out).
987
- if (muts.length || pendingChanged) {
988
- this.runReconcileCycle(muts);
2600
+ // The reconcile cycle — only when something can have changed: a base delta to fold in, or a
2601
+ // pending set that shrank (its optimistic layer must rewind out). The batch it emits for any view
2602
+ // that JUST became hydrated is that view's initial result set, so mark those qids so the
2603
+ // local-event forwarder stamps their batch `catchUp` (→ Store phases it `snapshot`).
2604
+ if (deltas.length || pendingChanged) {
2605
+ const emitted = (this.catchUpEmitted = new Set<QueryId>());
2606
+ this.catchUpQids = newlyHydrated;
2607
+ try {
2608
+ this.runReconcileCycle(sourceKey, deltas);
2609
+ } finally {
2610
+ this.catchUpQids = null;
2611
+ this.catchUpEmitted = null;
2612
+ }
2613
+ // Drop the qids the reconcile actually delivered a batch for; the rest folded nothing.
2614
+ if (newlyHydrated) for (const qid of emitted) newlyHydrated.delete(qid);
989
2615
  }
990
2616
 
991
- // (4) ResultType is the SERVER CHANNEL's state only now (§7): `unknown` while not hydrated,
992
- // else `complete` — a pending mutation no longer moves it. The pending axis moves separately.
2617
+ // ResultType is the SERVER CHANNEL's state only now (§7): `unknown` while not hydrated, else
2618
+ // `complete` — a pending mutation no longer moves it. The pending axis moves separately.
993
2619
  for (const qid of this.queryTables.keys()) {
994
2620
  this.setResultType(qid, this.hydrated.has(qid) ? "complete" : "unknown");
995
2621
  }
2622
+ // A newly-hydrated query whose reconcile emitted NO batch (0 rows, its whole result already present
2623
+ // via a sibling → 0 net muts, or the reconcile was skipped) still needs a hydration signal, or its
2624
+ // SSR seed never retires and the view freezes. Send an explicit empty catch-up (now that it reads
2625
+ // `complete`, the Store retires the seed and reveals whatever is already in its tree).
2626
+ if (newlyHydrated) {
2627
+ for (const qid of newlyHydrated) this.handler(qid, { type: "batch", events: [], catchUp: true });
2628
+ }
996
2629
  this.refreshPending();
2630
+ // The §301 echo-fence drop pass — strictly AFTER the reconcile above folded this release's
2631
+ // data, so a fence that cleared in this release drops its pin only once the co-committed
2632
+ // echo data is actually in the baseline (dropping earlier would itself reopen the flash
2633
+ // window §7.3 closes). Structural no-op with no pins (every single-domain client).
2634
+ this.dropEchoFencePins();
2635
+ // The I-v ghost-drop watcher (§4.2), LAST: this release's watermark rows have folded
2636
+ // (computeRelease) and its confirm-drop has retired what it covers — exactly the two inputs
2637
+ // the drop condition reads. Structural no-op with no ghost.
2638
+ this.evaluateGhosts();
2639
+ }
2640
+
2641
+ /** Test-only per-source release seam (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §7.2/§8.5): drive
2642
+ * {@link applyRelease} for `sourceKey` directly — an explicit `watermarkUpdate` (a simulated lmid
2643
+ * confirm for that domain) and `deltas` (a coherent base delta), with no real gate. Lets a harness
2644
+ * exercise a room-domain confirm before the real second lmid stream / per-source gate is wired
2645
+ * (E-iii-b/c). The `__`-prefix marks it a test hook, alongside {@link __inspect}. */
2646
+ __testRelease(sourceKey: string, deltas: Mutation[], watermarkUpdate?: number): void {
2647
+ this.applyRelease(sourceKey, deltas, watermarkUpdate);
2648
+ }
2649
+
2650
+ // --- the §301 echo fences (301-ECHO-FENCE-DESIGN.md) -----------------------------------
2651
+
2652
+ /** Register (or overwrite — a later cross-slice confirm re-pinning the same slice-pk matches
2653
+ * the engine's `held_back.insert`) one {@link EchoFencePin}, in the same breath as the
2654
+ * engine park (§2.1). `p` is the confirmed entry whose write staged onto `sourceKey`; its
2655
+ * `mid` is non-null by the caller's confirm filter. */
2656
+ private registerPin(
2657
+ table: string,
2658
+ sourceKey: string,
2659
+ pkKey: string,
2660
+ probeRow: WireValue[],
2661
+ p: PendingMutation,
2662
+ ): void {
2663
+ const pin: EchoFencePin = {
2664
+ table,
2665
+ sourceKey,
2666
+ probeRow,
2667
+ domain: p.domain,
2668
+ mid: p.mid as number,
2669
+ };
2670
+ if (sourceKey !== "daemon" && p.domain === "daemon") {
2671
+ // Direction B stamps (§1.2): the parking daemon release's coherence position — the confirm
2672
+ // folded at-or-before it, so a room advertising absorption ≥ this cv has absorbed the
2673
+ // commit that carried the write — plus the client-observed boot it belongs to (§2.4).
2674
+ pin.daemonCv = this.daemonGate.appliedCv;
2675
+ if (this.daemonBootId !== undefined) pin.daemonBoot = this.daemonBootId;
2676
+ }
2677
+ // The tripwire reference (§2.5): the slice's confirmed-baseline row at park.
2678
+ pin.baselineAtPark = this.local.heldBackState(table, sourceKey, probeRow)?.baseline as
2679
+ | WireValue[]
2680
+ | undefined;
2681
+ this.pins.set(`${table}\0${sourceKey}\0${pkKey}`, pin);
997
2682
  }
998
2683
 
999
- /** Fold the lmid system query's released ops (lmid-as-data): the one row's
1000
- * `last_mutation_id` cell is this client's confirmed high-water mid. */
1001
- private foldLmidOps(ops: NormalizedOp[]): void {
2684
+ /** The §2.3 drop pass, run at the tail of every applied release: prune registry entries whose
2685
+ * engine pin is already gone (the rewind's state-match fallback or a whole-source removal beat
2686
+ * the fence — benign), drop every pin whose fence cleared (one engine drop each, delivered on
2687
+ * the ordinary event stream, bracketed as ONE notification commit), and tripwire the rest. */
2688
+ private dropEchoFencePins(): void {
2689
+ if (this.pins.size === 0) return; // every single-domain client: structural no-op
2690
+ let cleared: [string, EchoFencePin][] | null = null;
2691
+ for (const [key, pin] of this.pins) {
2692
+ const state = this.local.heldBackState(pin.table, pin.sourceKey, pin.probeRow);
2693
+ if (state === undefined) {
2694
+ this.pins.delete(key); // engine pin gone (state-match / source removal): lazy prune
2695
+ continue;
2696
+ }
2697
+ if (this.pinFenceCleared(pin)) (cleared ??= []).push([key, pin]);
2698
+ else this.maybeTripwirePin(key, pin, state.baseline as WireValue[] | undefined);
2699
+ }
2700
+ if (!cleared) return;
2701
+ this.inOneCommit(() => {
2702
+ for (const [key, pin] of cleared) {
2703
+ this.pins.delete(key);
2704
+ this.local.dropHeldBack(pin.table, pin.sourceKey, pin.probeRow);
2705
+ }
2706
+ });
2707
+ }
2708
+
2709
+ /** Has `pin`'s delivery fence provably closed its confirm→echo window? (§2.3/§2.4.) */
2710
+ private pinFenceCleared(pin: EchoFencePin): boolean {
2711
+ if (pin.sourceKey === "daemon") {
2712
+ // Direction A: the daemon-carried ledger for the confirming room domain covers the mid —
2713
+ // the I-ii co-commit means this release (or an earlier one) folded the flush data that
2714
+ // carried it into the daemon baseline (§1.1).
2715
+ return (this.daemonCarriedLmid.get(pin.domain) ?? 0) >= pin.mid;
2716
+ }
2717
+ // Direction B: only a daemon-confirmed write has the daemon-cv stamp; a room-staged pin
2718
+ // confirmed by ANOTHER room (outside today's two-tier topology) has no fence.
2719
+ if (pin.daemonCv === undefined) return false;
2720
+ const gate = this.gates.get(pin.sourceKey);
2721
+ if (!gate || gate.upstreamCv === undefined) return false; // no advert (old shell): §2.5 fallback
2722
+ if (gate.upstreamBoot === pin.daemonBoot) return gate.upstreamCv >= pin.daemonCv;
2723
+ // Boot mismatch (§2.4): a boot the client has OBSERVED as later proves absorption (the
2724
+ // room's post-restart re-snapshot came from daemon state that durably includes the
2725
+ // confirmed write); an unknown/older/unstamped boot holds — conservative, its next
2726
+ // re-snapshot advances it.
2727
+ if (pin.daemonBoot === undefined || gate.upstreamBoot === undefined) return false;
2728
+ const pinOrd = this.daemonBootOrdinals.get(pin.daemonBoot);
2729
+ const advOrd = this.daemonBootOrdinals.get(gate.upstreamBoot);
2730
+ return pinOrd !== undefined && advOrd !== undefined && advOrd > pinOrd;
2731
+ }
2732
+
2733
+ /** The §2.5 stuck-pin tripwire, in the scopesHash spirit: log ONCE per pin when its slice's
2734
+ * baseline row has CHANGED VALUE since park while the pin still holds — the suspicious state
2735
+ * that precedes every forever-pin (a fence-less pairing, a fence bug). Never drops anything. */
2736
+ private maybeTripwirePin(key: string, pin: EchoFencePin, baseline: WireValue[] | undefined): void {
2737
+ if (pin.tripwired) return;
2738
+ const same =
2739
+ pin.baselineAtPark === undefined || baseline === undefined
2740
+ ? pin.baselineAtPark === baseline
2741
+ : pin.baselineAtPark.length === baseline.length &&
2742
+ pin.baselineAtPark.every((c, i) => identicalCell(c, baseline[i]));
2743
+ if (same) return;
2744
+ pin.tripwired = true;
2745
+ const [table, sourceKey, pkKey] = key.split("\0");
2746
+ console.warn(
2747
+ `rindle: a §7.3 hold-back pin on table "${table}" slice "${sourceKey}" pk ${pkKey} (confirming domain ${pin.domain}, mid ${pin.mid}) is parked while its slice's baseline moved past it — if this client never converges on that row, this pin is why (301-ECHO-FENCE-DESIGN.md §2.5).`,
2748
+ );
2749
+ }
2750
+
2751
+ /** Record one observed daemon boot id (§2.4): first observation of an id assigns the next
2752
+ * ordinal (the client's own total order over opaque boot ids); every call refreshes the
2753
+ * current-boot stamp for direction-B parks. */
2754
+ private observeDaemonBoot(bootId: string): void {
2755
+ if (!this.daemonBootOrdinals.has(bootId)) {
2756
+ this.daemonBootOrdinals.set(bootId, this.daemonBootOrdinals.size);
2757
+ }
2758
+ this.daemonBootId = bootId;
2759
+ }
2760
+
2761
+ /** Promote `table` to a MERGED multi-source engine with room `sourceKey`'s per-row writable
2762
+ * scope (RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §5.2) — THE one promotion seam. The engine
2763
+ * attach and the {@link promotedTables} record move in the same breath: the §7.3 hold-back
2764
+ * trigger in {@link applyRelease} parks ONLY on promoted tables, so a promotion that bypassed
2765
+ * the record would silently disable the echo hold-back for that table (and a record without the
2766
+ * engine attach would park onto a slice that doesn't exist). Recorded AFTER the engine accepts —
2767
+ * a rejected descriptor/table must not leave a phantom promotion. Slice G-v's client drives this
2768
+ * from the lease's `realtime.tables` (`RoomTableSpec` → {@link WritableDescriptor}); idempotence
2769
+ * per `(sourceKey, table)` is the CALLER's job (the engine refuses a duplicate room).
2770
+ *
2771
+ * `spec` (H-iii) is the per-table ROUTING spec riding the same lease table block — recorded
2772
+ * into {@link roomRouting} (THE routing table the §3 router reads) in the same breath, with
2773
+ * `writable` derived from the descriptor's kind (`none` = a context table the room may not
2774
+ * write). Omitted (the E-iii-b harness alias below) ⇒ an empty spec: no join keys to guard, no
2775
+ * `footprintWhere` (that room's reads then fail closed to the daemon unless self/room-served). */
2776
+ promoteRoomTable(
2777
+ table: string,
2778
+ sourceKey: string,
2779
+ writable: WritableDescriptor,
2780
+ spec?: RoomTableRoutingSpec,
2781
+ ): void {
2782
+ this.local.addRoomSource(table, sourceKey, writable);
2783
+ this.promotedTables.add(table);
2784
+ let byTable = this.roomRouting.get(sourceKey);
2785
+ if (!byTable) this.roomRouting.set(sourceKey, (byTable = new Map()));
2786
+ byTable.set(table, {
2787
+ writable: writable.kind !== "none",
2788
+ joinKeyCols: [...(spec?.joinKeyCols ?? [])],
2789
+ ...(spec?.where !== undefined ? { where: spec.where } : {}),
2790
+ ...(spec?.footprintWhere !== undefined ? { footprintWhere: spec.footprintWhere } : {}),
2791
+ });
2792
+ }
2793
+
2794
+ /** The recorded routing specs for `sourceKey`'s promoted tables (table → its
2795
+ * {@link RoomTableRouting}) — read-only: the client's `__realtimeInspect`/idempotence
2796
+ * bookkeeping reads THIS record instead of keeping its own shadow copy (one source of truth).
2797
+ * Empty map when the room has promoted nothing. */
2798
+ roomTablesFor(sourceKey: string): ReadonlyMap<string, RoomTableRouting> {
2799
+ return this.roomRouting.get(sourceKey) ?? EMPTY_ROUTING;
2800
+ }
2801
+
2802
+ /** Test-named alias of {@link promoteRoomTable} (E-iii-b scaffolding — the multi-domain oracle
2803
+ * and per-source-gate suites drive it). Pure delegation: ONE body, one `promotedTables` record. */
2804
+ __addRoomSource(table: string, roomKey: string, writable: WritableDescriptor, spec?: RoomTableRoutingSpec): void {
2805
+ this.promoteRoomTable(table, roomKey, writable, spec);
2806
+ }
2807
+
2808
+ /** Fold `domain`'s lmid system query's released ops (lmid-as-data): the one row's
2809
+ * `last_mutation_id` cell is this client's confirmed high-water mid in that domain — it advances
2810
+ * `watermark[domain]` and, on a fresh session ahead of our issued mids, `nextMid[domain]`. The
2811
+ * daemon stream folds `"daemon"`; a room stream folds its own `"room:doc:X"`; the daemon-carried
2812
+ * §7.1 ledger rows fold through the same {@link foldConfirm} core (Slice I-iii). */
2813
+ private foldLmidOps(ops: NormalizedOp[], domain: string): void {
1002
2814
  for (const op of ops) {
1003
2815
  const row = op.op === "add" ? op.row : op.op === "edit" ? op.new : undefined;
1004
2816
  if (!row) continue; // a remove (client GC) confirms nothing
1005
- const lmid = Number(row[1]);
1006
- if (!Number.isFinite(lmid)) continue;
1007
- if (lmid > this.nextMid - 1) {
1008
- if (this.pendingMutations.length > 0) {
1009
- // The server confirmed a mid we never issued while we have mutations in
1010
- // flight two writers on one clientID or corrupted state. Unrecoverable.
1011
- throw new Error(
1012
- `optimistic backend: confirmed lmid ${lmid} is ahead of issued mids (${this.nextMid - 1})`,
1013
- );
2817
+ this.foldConfirm(domain, Number(row[1]));
2818
+ }
2819
+ }
2820
+
2821
+ /** THE one confirm fold (§7.1/§7.2): advance `watermark[domain]` to `lmid` (monotone max) and,
2822
+ * on a fresh session ahead of our issued mids, adopt `nextMid[domain]`. Shared verbatim by the
2823
+ * per-channel lmid system query ({@link foldLmidOps}) and the daemon-carried room-ledger rows
2824
+ * ({@link foldSystemFrames} one core so the two paths cannot drift). */
2825
+ private foldConfirm(domain: string, lmid: number): void {
2826
+ if (!Number.isFinite(lmid)) return;
2827
+ const highestIssued = (this.nextMid.get(domain) ?? 1) - 1;
2828
+ if (lmid > highestIssued) {
2829
+ // Only in-flight mutations of THIS domain can contradict its watermark (§7.1: the counters
2830
+ // are independent — a room-domain mutation pending while the daemon's historical lmid
2831
+ // snapshot arrives is a normal fresh-session interleaving, not a second writer). An
2832
+ // unflushed fold (`mid == null`) has issued nothing yet either way: it cannot explain a
2833
+ // confirmed-ahead lmid, and its eventual flush deals from the adopted counter below.
2834
+ if (this.pendingMutations.some((p) => p.domain === domain && p.mid !== null)) {
2835
+ // The server confirmed a mid we never issued while we have mutations in
2836
+ // flight on this domain — two writers on one clientID or corrupted state. Unrecoverable.
2837
+ throw new Error(
2838
+ `optimistic backend: confirmed lmid ${lmid} is ahead of issued mids (${highestIssued})`,
2839
+ );
2840
+ }
2841
+ // A fresh session over a clientID with history: adopt the server's high-water
2842
+ // mark so our next mid continues the sequence instead of colliding below it.
2843
+ this.nextMid.set(domain, lmid + 1);
2844
+ }
2845
+ this.watermark.set(domain, Math.max(this.watermark.get(domain) ?? 0, lmid));
2846
+ }
2847
+
2848
+ // --- the §4 lifecycle system-stream folds (Slice I-iii) --------------------------------
2849
+
2850
+ /** Fold one release's SYSTEM frames in a FIXED category order — the order is STRUCTURAL (one
2851
+ * function, categories in sequence), because it is the client half of THE NAMED INVARIANT
2852
+ * (§3.3's shipped note; documented above {@link handleMutationOutcome}): **never retire a
2853
+ * room-domain entry off a daemon-carried lmid without outcome resolution.**
2854
+ *
2855
+ * 1. **outcome rows** (`_rindle_room_mutation_outcomes`) — each row for OUR clientID is
2856
+ * synthesized into a {@link MutationOutcomeFrame} and routed through
2857
+ * {@link handleMutationOutcome}, the SAME H-v state machine the room socket's frames use
2858
+ * (one verdict path: frames and rows cannot drift). A deopt flips its pending entry to
2859
+ * the daemon IN PLACE (keep-seq, deal-and-send-now); a rejection surfaces + stays for the
2860
+ * ordinary burnt-mid retire; a duplicate (frame already seen, or the row re-delivered) is
2861
+ * absorbed by the processed set — which doubles as the resolved-verdict memory across
2862
+ * releases (per-domain FIFO, {@link MAX_PROCESSED_OUTCOMES_PER_DOMAIN}, mirroring the
2863
+ * shell's recorded-outcome cap).
2864
+ * 2. **room-ledger rows** (`_rindle_room_client_mutations`) — the FIRST daemon-carried
2865
+ * room-lmid path: OUR row's `last_mutation_id` folds into `watermark[room:<doc>]` via
2866
+ * {@link foldConfirm}. Because step 1 ALREADY resolved every non-applied verdict this
2867
+ * release carries (and earlier releases' verdicts were resolved at their own release),
2868
+ * the confirm-drop that follows in {@link applyRelease} retires only entries whose
2869
+ * outcome is resolution-by-absence — which I-ii's co-commit atomicity defines as APPLIED
2870
+ * (a room flush co-commits the ledger row and every non-applied mid's outcome row in ONE
2871
+ * daemon transaction, so a covering lmid without a row IS the applied verdict).
2872
+ * Processing this category before step 1 is the violation, in two proven directions
2873
+ * (each run break→fail→revert against `test/system_streams.test.ts`): (a) the ledger's
2874
+ * fresh-session `nextMid` ADOPTION must not run before historical outcome rows are
2875
+ * judged — adopted-first, a previous session's retained deopt row passes the
2876
+ * "never-issued" guard and spuriously re-invokes a mutation that session already handled
2877
+ * (a double-apply); (b) the RETIRE must not precede resolution — it does not BECAUSE the
2878
+ * confirm-drop runs in {@link applyRelease}, strictly after this whole function. That
2879
+ * deferral is load-bearing: an "optimization" retiring inline with the watermark fold
2880
+ * retires a deopted entry as a silent success (the exact lost-write H-v exists to
2881
+ * prevent) and mis-attributes a rejected row's reason.
2882
+ * 3. **watermark rows** (`_rindle_room_watermark`) — the §4.2 fence value, max-folded per
2883
+ * doc ({@link roomWatermarks}); I-v's ghost-drop consumer, no reaction here.
2884
+ * 4. **scope-session rows** (`_rindle_scope_sessions`) — the §4.1 occupancy map
2885
+ * ({@link scopeSessions}); I-iv's doorbell consumer, no reaction here.
2886
+ *
2887
+ * Ordinary data ops fold AFTER all of these (the caller's main loop) — outcome/ledger state
2888
+ * must be in place before {@link applyRelease}'s confirm-drop + reconcile consume the release.
2889
+ * Every row is filtered against the retain's {@link SystemStreamSpec} scope/doc AND (for the
2890
+ * client-keyed tables) our own `clientID` — defense in depth: the server predicate may have
2891
+ * been minted doc-only (no `clientId` at lease time), so other clients' rows are expected and
2892
+ * must be ignored, and a row for a doc this retain was not minted for is never folded.
2893
+ *
2894
+ * Returns the scopes category 4 touched (snapshot or ops) — the I-iv doorbell events' input;
2895
+ * `null` when none (every non-lifecycle release). The events themselves fire from
2896
+ * `onGateProgress` AFTER the release applies, never from inside the fold. */
2897
+ private foldSystemFrames(frames: BufferedFrame[]): Set<string> | null {
2898
+ if (frames.length === 0) return null;
2899
+ const byTable = (table: string): { spec: SystemStreamSpec; frame: BufferedFrame }[] =>
2900
+ frames.flatMap((frame) => {
2901
+ const spec = this.systemQids.get(frame.qid);
2902
+ return spec !== undefined && spec.table === table ? [{ spec, frame }] : [];
2903
+ });
2904
+ // (1) outcome rows → the H-v machine, BEFORE any ledger fold (the named invariant).
2905
+ for (const { spec, frame } of byTable(ROOM_MUTATION_OUTCOMES_TABLE)) {
2906
+ for (const op of frame.ops) {
2907
+ const cells = op.op === "add" ? op.row : op.op === "edit" ? op.new : undefined;
2908
+ if (!cells) continue; // a remove is retention pruning (mid ≤ lmid − 512), never a verdict
2909
+ const row = decodeOutcomeRow(cells);
2910
+ if (!row || row.clientId !== this.clientID) continue;
2911
+ if (spec.doc !== undefined && row.doc !== spec.doc) continue;
2912
+ const frameShape: MutationOutcomeFrame = {
2913
+ mid: row.mid,
2914
+ kind: row.kind,
2915
+ ...(row.reason !== undefined ? { reason: row.reason } : {}),
2916
+ ...(row.name !== undefined ? { name: row.name } : {}),
2917
+ ...(row.args !== undefined ? { args: row.args } : {}),
2918
+ };
2919
+ // Release-time invocation is sound here where out-of-band was REQUIRED for the socket
2920
+ // frames (`attachGate`): the socket frame races a buffered lmid ack it must beat, so it
2921
+ // may not wait behind the gate — a ROW cannot race its own release (it and the covering
2922
+ // ledger row co-committed at one cv and fold in THIS function's fixed order). The
2923
+ // machine's steps need nothing from an open release: the flip/reject only move pending
2924
+ // bookkeeping + ship an envelope, and the not-found re-invoke arm runs a fresh prediction
2925
+ // — legal before `applyRelease` opens the reconcile cycle, identical to an app invoke
2926
+ // racing the release.
2927
+ this.handleMutationOutcome(roomDomainKey(row.doc), frameShape);
2928
+ }
2929
+ }
2930
+ // (2) room-ledger rows → the daemon-carried per-domain confirm (outcomes above resolved first).
2931
+ for (const { spec, frame } of byTable(ROOM_CLIENT_MUTATIONS_TABLE)) {
2932
+ for (const op of frame.ops) {
2933
+ const cells = op.op === "add" ? op.row : op.op === "edit" ? op.new : undefined;
2934
+ if (!cells) continue; // a ledger remove confirms nothing (mirrors foldLmidOps)
2935
+ const [doc, clientId, lmid] = cells;
2936
+ if (typeof doc !== "string" || clientId !== this.clientID) continue;
2937
+ if (spec.doc !== undefined && doc !== spec.doc) continue;
2938
+ const covered = Number(lmid);
2939
+ this.foldConfirm(roomDomainKey(doc), covered);
2940
+ // The §301 direction-A fence input: the DAEMON-CARRIED ledger fold ALONE (the room
2941
+ // socket's own lmid stream confirms long before the flush reaches the daemon, so it
2942
+ // folds the shared watermark above but never this map — 301 §1.1). Max-fold: ledger
2943
+ // rows re-deliver across re-hydrates.
2944
+ if (Number.isFinite(covered)) {
2945
+ const dom = roomDomainKey(doc);
2946
+ this.daemonCarriedLmid.set(dom, Math.max(this.daemonCarriedLmid.get(dom) ?? 0, covered));
1014
2947
  }
1015
- // A fresh session over a clientID with history: adopt the server's high-water
1016
- // mark so our next mid continues the sequence instead of colliding below it.
1017
- this.nextMid = lmid + 1;
1018
2948
  }
1019
- this.confirmedLmid = Math.max(this.confirmedLmid, lmid);
1020
2949
  }
2950
+ // (3) watermark rows → the monotone §4.2 fence value per doc.
2951
+ for (const { spec, frame } of byTable(ROOM_WATERMARK_TABLE)) {
2952
+ for (const op of frame.ops) {
2953
+ const cells = op.op === "add" ? op.row : op.op === "edit" ? op.new : undefined;
2954
+ if (!cells) continue; // the fence is monotone — a remove never regresses it
2955
+ const [doc, flushSeq] = cells;
2956
+ const seq = Number(flushSeq);
2957
+ if (typeof doc !== "string" || !Number.isFinite(seq)) continue;
2958
+ if (spec.doc !== undefined && doc !== spec.doc) continue;
2959
+ this.roomWatermarks.set(doc, Math.max(this.roomWatermarks.get(doc) ?? 0, seq));
2960
+ }
2961
+ }
2962
+ // (4) scope-session rows → the §4.1 occupancy map (a snapshot REPLACES the scope's map — an
2963
+ // authoritative re-hydrate must drop sessions that aged out while the stream was down; a
2964
+ // batch folds add/edit/remove incrementally). Touched scopes are collected for the I-iv
2965
+ // doorbell events (a snapshot touches its minted scope even with zero ops — an emptied-out
2966
+ // scope is a legitimate 1→0 observation for the transition tracker).
2967
+ let touchedScopes: Set<string> | null = null;
2968
+ const touch = (scope: string): void => {
2969
+ (touchedScopes ??= new Set()).add(scope);
2970
+ };
2971
+ for (const { spec, frame } of byTable(SCOPE_SESSIONS_TABLE)) {
2972
+ if (frame.kind === "snapshot" && spec.scope !== undefined) {
2973
+ this.scopeSessions.set(spec.scope, new Map());
2974
+ touch(spec.scope);
2975
+ }
2976
+ for (const op of frame.ops) {
2977
+ // A remove's identity rides its (full) removed row; add/edit carry the post-image.
2978
+ const cells = op.op === "edit" ? op.new : op.row;
2979
+ const [scope, clientId, expiresAt] = cells;
2980
+ if (typeof scope !== "string" || typeof clientId !== "string") continue;
2981
+ if (spec.scope !== undefined && scope !== spec.scope) continue;
2982
+ let sessions = this.scopeSessions.get(scope);
2983
+ if (!sessions) this.scopeSessions.set(scope, (sessions = new Map()));
2984
+ if (op.op === "remove") {
2985
+ sessions.delete(clientId);
2986
+ } else {
2987
+ const exp = Number(expiresAt);
2988
+ if (Number.isFinite(exp)) sessions.set(clientId, exp);
2989
+ }
2990
+ touch(scope);
2991
+ }
2992
+ }
2993
+ return touchedScopes;
2994
+ }
2995
+
2996
+ /** The I-iv occupancy count — THE one rule (§4.1/D7): unexpired (`expires_at >` the fold
2997
+ * clock's now) sessions under `scope` from OTHER clientIDs. Shared by the doorbell events
2998
+ * ({@link onGateProgress}) and the client's registration-time check (a doorbell that folded
2999
+ * BEFORE a candidate registered must still be able to trigger it) so the two can never
3000
+ * disagree. Own-clientID rows never count — a solo client cannot ring its own bell — and
3001
+ * expiry is judged on the injectable {@link FoldClock} (deterministic in a virtual-clock
3002
+ * harness, the folded-oracle discipline). */
3003
+ otherScopeSessions(scope: string): number {
3004
+ const sessions = this.scopeSessions.get(scope);
3005
+ if (!sessions) return 0;
3006
+ const now = this.clock.now();
3007
+ let n = 0;
3008
+ for (const [clientId, expiresAt] of sessions) {
3009
+ if (clientId !== this.clientID && expiresAt > now) n++;
3010
+ }
3011
+ return n;
3012
+ }
3013
+
3014
+ /** Register the I-iv doorbell event sink — see {@link ScopeSessionsEvent}. One handler (a later
3015
+ * registration replaces it, the {@link onLocalWrite} convention); client.ts is the consumer. */
3016
+ onScopeSessions(handler: (event: ScopeSessionsEvent) => void): void {
3017
+ this.scopeSessionsHandler = handler;
1021
3018
  }
1022
3019
 
1023
3020
  /** One §1.3 reconcile cycle: rewind the optimistic layer and fold the coherent SERVER
@@ -1026,34 +3023,58 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
1026
3023
  * deliver the coalesced result (`serverBatchEnd`). This is the engine's only sync-moving
1027
3024
  * boundary — `onProgress` releases and `unregisterQuery`'s GC both go through here so head
1028
3025
  * and sync never diverge (the §1.2 invariant; CRIT#2). */
1029
- private runReconcileCycle(serverDeltas: Mutation[]): void {
1030
- this.local.serverBatchBegin(serverDeltas.map(toServerOp));
1031
- // The rewind cleared every optimistic write incl. prior `__agg` edits so head is now
1032
- // the server baseline. Rebuild the optimistic agg delta from scratch off the re-invoked
1033
- // (confirm-filtered) pending set, so a just-confirmed mutation's delta vanishes exactly as
1034
- // its server count is absorbed (§5 watermark — no double count).
1035
- this.overlay.reset();
1036
- // Re-invoke in WIRE order: assigned mids ascending, then unflushed folds (`mid == null`) last
1037
- // by creation order the deterministic slot of FOLDED-MUTATIONS-DESIGN §4.1. A read-dependent
1038
- // mutator must replay against the same base the server computed from, which is mid order; with
1039
- // deferred fold mids, mid order ≠ creation order, so we sort rather than trust array order. The
1040
- // comparator is explicit (NOT `(mid ?? ∞) - (mid ?? ∞)`, which is `∞ - ∞ = NaN` for two unflushed
1041
- // foldsa NaN comparator silently corrupts V8's sort): assigned-before-unflushed, mids
1042
- // ascending, and STABLE for two unflushed folds so they keep creation order across cycles.
1043
- const order = [...this.pendingMutations].sort((a, b) => {
1044
- if (a.mid === null && b.mid === null) return 0; // both unflushed → stable creation order
1045
- if (a.mid === null) return 1; // an unflushed fold sorts after every assigned mid
1046
- if (b.mid === null) return -1;
1047
- return a.mid - b.mid;
1048
- });
3026
+ private runReconcileCycle(sourceKey: string, serverDeltas: Mutation[]): void {
3027
+ // `sourceKey` names the authority these `deltas` confirm — `"daemon"` on the live daemon path (and
3028
+ // the GC path), a `room:doc:X` string on a per-source release (E-iii). It selects which physical
3029
+ // source the wasm engine's rewind folds the delta into (`Db::serverBatchBegin`). The E-iii-b/c
3030
+ // per-source rewind + intersecting-pending re-invoke filter ride the same seam later.
3031
+ this.local.serverBatchBegin(sourceKey, serverDeltas.map(toServerOp));
3032
+ // The aggregate overlay (§4) is DAEMON-tracked: its `__agg_*` tables are server-authoritative
3033
+ // synthetic tables that only the daemon feed drives. A room cycle must not wipe or rebuild it
3034
+ // (that would double-count / lose daemon agg state), so gate every overlay interaction — reset,
3035
+ // observe, and reconcileAggHead on the daemon cycle. Single-domain: always daemon unchanged.
3036
+ const daemonCycle = sourceKey === "daemon";
3037
+ if (daemonCycle) {
3038
+ // The rewind cleared every optimistic write incl. prior `__agg` edits so head is now the
3039
+ // server baseline. Rebuild the optimistic agg delta from scratch off the re-invoked
3040
+ // (confirm-filtered) pending set, so a just-confirmed mutation's delta vanishes exactly as its
3041
+ // server count is absorbed (§5 watermark no double count).
3042
+ this.overlay.reset();
3043
+ }
3044
+ // Sort ALL pending into SEND order (the client-global `seq` ascending, then unflushed folds
3045
+ // last by creation order — the deterministic §4.1 slot; the comparator is explicit, NOT
3046
+ // `(seq ?? ∞) - (seq ?? ∞)` which is `∞ - ∞ = NaN` and corrupts V8's sort), THEN FILTER to the
3047
+ // writers of THIS source (§5.3): a per-source rewind un-applied ONLY that source's staged
3048
+ // writes, so ONLY those must re-invoke — the others' predictions are intact on their un-rewound
3049
+ // trees. The key MUST be `seq`, never `mid`: mids are per-domain (§7.1) so mids from different
3050
+ // domains are incomparable — a mid-sort would replay a room mid 1 before a daemon mid 5 that
3051
+ // was sent FIRST, letting a read-dependent mutator re-predict from a base it never saw
3052
+ // (confirmation order is per-domain; replay order is client-global). FILTERING the
3053
+ // globally-ordered array preserves the relative send order among the subset (the 200 §4.1
3054
+ // invariant); NEVER re-sort the subset. Single-domain: every entry has
3055
+ // `touchedSources = {"daemon"}` and seq order == mid order (except H-v deopt re-enqueues,
3056
+ // which keep their ORIGINAL seq under a later daemon mid — deliberately, so this very sort
3057
+ // replays them at their original overlay position) ⇒ byte-identical to before.
3058
+ const order = [...this.pendingMutations]
3059
+ .sort((a, b) => {
3060
+ if (a.seq === null && b.seq === null) return 0; // both unflushed → stable creation order
3061
+ if (a.seq === null) return 1; // an unflushed fold sorts after every dealt seq
3062
+ if (b.seq === null) return -1;
3063
+ return a.seq - b.seq;
3064
+ })
3065
+ .filter((p) => p.touchedSources.has(sourceKey));
1049
3066
  const dropped = new Set<PendingMutation>();
1050
3067
  try {
1051
3068
  for (const p of order) {
1052
- const touched = new Set<string>();
3069
+ // NO `readLog` here — recording is armed only on the initial `invoke` (§3.2 #2 note on
3070
+ // `PendingMutation.reads`); a re-invocation's write-set still needs fresh capture (below).
3071
+ const writes: WriteSet = new Map();
1053
3072
  const ops: ChildOp[] = [];
3073
+ const stagedKeys: string[] = [];
3074
+ let sources: string[] = [];
1054
3075
  try {
1055
- this.local.writeWith((tx) => {
1056
- this.runMutator(this.registry[p.name], trackingTx(tx, touched, this.specs, this.localTables, this.opCollector(ops)), p.args);
3076
+ sources = this.local.writeWith((tx) => {
3077
+ this.runMutator(this.registry[p.name], trackingTx(tx, writes, this.specs, this.localTables, this.opCollector(ops), false, undefined, stagedKeys), p.args);
1057
3078
  });
1058
3079
  } catch {
1059
3080
  // A re-invocation threw — e.g. a read-dependent mutator whose base row the server
@@ -1065,50 +3086,65 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
1065
3086
  dropped.add(p);
1066
3087
  continue;
1067
3088
  }
1068
- // The re-invocation stuck — fold its child ops into the rebuilt optimistic agg delta.
1069
- for (const op of ops) this.overlay.observe(op);
3089
+ // The re-invocation stuck — fold its child ops into the rebuilt optimistic agg delta (daemon
3090
+ // cycle only a room cycle leaves the daemon-tracked overlay untouched).
3091
+ if (daemonCycle) for (const op of ops) this.overlay.observe(op);
1070
3092
  // The pending footprint is the UNION across invocations: a re-run that no-ops (touched =
1071
3093
  // {}) must NOT shrink it, else a still-pending mutation reports not-pending and its
1072
- // pending-axis clear fires early (§7.2).
1073
- for (const t of touched) p.touched.add(t);
3094
+ // pending-axis clear fires early (§7.2). `writes`/the source axes mirror this: merge, never
3095
+ // replace. (A re-invocation may route differently, e.g. onto a source the room feed now
3096
+ // holds — union so the filter still catches it on either source's next cycle.)
3097
+ for (const t of writes.keys()) p.touched.add(t);
3098
+ mergeWriteSet(p.writes, writes);
3099
+ for (const s of sources) p.touchedSources.add(s);
3100
+ mergeWriteSources(p.writeSources, stagedKeys, sources);
1074
3101
  }
1075
3102
  // Preserve creation order in the live array (the unflushed-fold sort tiebreak depends on it).
1076
3103
  if (dropped.size) this.pendingMutations = this.pendingMutations.filter((p) => !dropped.has(p));
1077
3104
  // Re-apply the optimistic agg delta onto the (rewound) `__agg` head rows — INSIDE the open
1078
3105
  // cycle, so the writes buffer and coalesce into the one per-query delivery `serverBatchEnd`
1079
- // makes (and never escape as a separate batch).
1080
- this.reconcileAggHead();
3106
+ // makes (and never escape as a separate batch). Daemon cycle only (see above).
3107
+ if (daemonCycle) this.reconcileAggHead();
1081
3108
  } finally {
1082
3109
  this.local.serverBatchEnd(); // ALWAYS close the cycle — ONE delivery per affected query.
1083
3110
  }
1084
3111
  }
1085
3112
 
1086
- /** The daemon restarted (a new boot id): it lost all materialization + `cv` state and its `cv`
1087
- * sequence reset, so previously-released `cv`s no longer bound the new stream. The source has
1088
- * already re-subscribed every query (reconnect → resync); drop the buffer and the `cv`
1089
- * watermark so the fresh, low-`cv` snapshots are RELEASED instead of dropped as stale
1090
- * (`onNormalized`/`onProgress` gate on `appliedCv`). Pending optimistic mutations stay put
1091
- * they re-apply on the next reconcile, and the lmid system query's fresh snapshot restores the
1092
- * confirmation watermark. */
1093
- private resetForRestart(): void {
1094
- this.buffer = [];
1095
- this.appliedCv = 0;
3113
+ /** ONE channel's authority restarted (a new boot id): it lost all materialization + `cv` state
3114
+ * and its `cv` sequence reset, so previously-released `cv`s no longer bound the new stream. The
3115
+ * source has already re-subscribed every query (reconnect → resync); drop THIS gate's buffer
3116
+ * and `cv` watermark so the fresh, low-`cv` snapshots are RELEASED instead of dropped as stale
3117
+ * (`onFrame`/`computeRelease` gate on `appliedCv`). The OTHER gates are untouched an
3118
+ * authority restart is per-channel (§5.1). Pending optimistic mutations stay put they
3119
+ * re-apply on the next reconcile, and the channel's lmid system query's fresh snapshot restores
3120
+ * its domain's confirmation watermark. */
3121
+ private resetGate(gate: SourceGate): void {
3122
+ gate.buffer = [];
3123
+ gate.appliedCv = 0;
3124
+ // The §301 upstream-absorption advert dies with the incarnation that made it — the fresh
3125
+ // one re-advertises (direction-B pins conservatively hold until it does).
3126
+ delete gate.upstreamBoot;
3127
+ delete gate.upstreamCv;
1096
3128
  }
1097
3129
 
1098
- /** The §8.5 escape: the buffer outgrew its cap (a pinned `cvMin` under churn). Drop
1099
- * everything buffered and re-register every query on the source — the fresh
1100
- * snapshots arrive as ordinary frames and the next release re-hydrates via the
1101
- * footprint diff (the §5.3 path); still-pending optimism re-applies in that cycle. */
1102
- private overflow(): void {
1103
- this.buffer = [];
3130
+ /** The §8.5 escape: ONE gate's buffer outgrew its cap (a pinned `cvMin` under churn on that
3131
+ * channel). Drop everything it buffered and re-register every query on that source — the fresh
3132
+ * snapshots arrive as ordinary frames and the next release re-hydrates via the footprint diff
3133
+ * (the §5.3 path); still-pending optimism re-applies in that cycle. The other gates' buffers
3134
+ * and subscriptions are untouched. */
3135
+ private overflow(gate: SourceGate): void {
3136
+ gate.buffer = [];
1104
3137
  for (const sub of this.remoteSubs.values()) {
1105
- this.source.unregisterQuery(sub.sourceQid);
1106
- this.source.registerQuery(sub.sourceQid, sub.remote);
3138
+ // Re-register only the subs THIS channel owns ({@link RemoteSub.channel} — the one source
3139
+ // of truth, G-iii): resubscribing another channel's sub here would fork its stream.
3140
+ if (sub.channel !== gate.key) continue;
3141
+ gate.source.unregisterQuery(sub.sourceQid);
3142
+ gate.source.registerQuery(sub.sourceQid, sub.remote);
1107
3143
  }
1108
3144
  // The lmid system query's buffered frames were dropped too — re-subscribe it so a
1109
3145
  // fresh snapshot restores the confirmation watermark.
1110
- this.source.unregisterQuery(LMID_QID);
1111
- this.source.registerQuery(LMID_QID, { name: LMID_QUERY_NAME, args: {} });
3146
+ gate.source.unregisterQuery(LMID_QID);
3147
+ gate.source.registerQuery(LMID_QID, { name: LMID_QUERY_NAME, args: {} });
1112
3148
  }
1113
3149
 
1114
3150
  private setResultType(qid: QueryId, rt: ResultType): void {
@@ -1138,15 +3174,30 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
1138
3174
  }
1139
3175
  }
1140
3176
 
1141
- private retainRemote(retainQid: QueryId, remote: RemoteQuery, localQueryId: QueryId | undefined = retainQid): void {
3177
+ /** `channel` (G-iii registration-time routing): the gate the sub registers on the qid's
3178
+ * ownership is fixed HERE, at retain time (no lazy claim; `onFrame` only asserts it). Default
3179
+ * `"daemon"`, so every channel-less caller is byte-identical to before. */
3180
+ private retainRemote(
3181
+ retainQid: QueryId,
3182
+ remote: RemoteQuery,
3183
+ localQueryId: QueryId | undefined = retainQid,
3184
+ channel = "daemon",
3185
+ ): void {
3186
+ const gate = this.requireGate(channel); // throw loudly BEFORE any sub state moves
1142
3187
  const key = remoteKey(remote);
1143
3188
  let sub = this.remoteSubs.get(key);
1144
3189
  let isNew = false;
1145
3190
  if (!sub) {
1146
- sub = { sourceQid: retainQid, remote, refCount: 0, localQids: new Map(), hydrated: false };
3191
+ sub = { sourceQid: retainQid, remote, refCount: 0, localQids: new Map(), hydrated: false, channel };
1147
3192
  this.remoteSubs.set(key, sub);
1148
3193
  this.sourceToRemote.set(sub.sourceQid, key);
1149
3194
  isNew = true;
3195
+ } else if (sub.channel !== channel) {
3196
+ // A (name,args) sub lives on ONE channel — a second retain naming another is a wiring bug
3197
+ // (it would split the query's frames across two cv timelines). Fail loudly.
3198
+ throw new Error(
3199
+ `optimistic backend: query "${remote.name}" is already retained on channel ${JSON.stringify(sub.channel)} — cannot retain it on ${JSON.stringify(channel)}`,
3200
+ );
1150
3201
  }
1151
3202
  sub.refCount++;
1152
3203
  if (localQueryId !== undefined) {
@@ -1160,7 +3211,9 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
1160
3211
  }
1161
3212
  this.localToRemote.set(retainQid, key);
1162
3213
  this.remoteRetainToLocal.set(retainQid, localQueryId);
1163
- if (isNew) this.source.registerQuery(sub.sourceQid, remote);
3214
+ // Register on the CHANNEL's source (G-iii): the qid's frames will arrive — and buffer, release,
3215
+ // and overflow — on that channel's own §5.1 gate.
3216
+ if (isNew) gate.source.registerQuery(sub.sourceQid, remote);
1164
3217
  }
1165
3218
 
1166
3219
  private releaseRemote(retainQid: QueryId): QueryId | undefined {
@@ -1178,7 +3231,10 @@ export class OptimisticBackend<S extends ColsMap> implements Backend {
1178
3231
  else sub.localQids.delete(localQueryId);
1179
3232
  }
1180
3233
  if (sub.refCount > 0) return undefined;
1181
- this.source.unregisterQuery(sub.sourceQid);
3234
+ // Unregister from the SAME gate's source the retain registered on. Since I-v a gate CAN be
3235
+ // removed ({@link disconnectSource}) — but never with a live sub on it ({@link
3236
+ // demoteRoomSource} validates loudly), so the daemon fallback is purely defensive.
3237
+ (this.gates.get(sub.channel) ?? this.daemonGate).source.unregisterQuery(sub.sourceQid);
1182
3238
  this.sourceToRemote.delete(sub.sourceQid);
1183
3239
  this.remoteSubs.delete(key);
1184
3240
  return sub.sourceQid;
@@ -1213,6 +3269,11 @@ interface RemoteSub {
1213
3269
  localQids: Map<QueryId, number>;
1214
3270
  /** Whether this sub's first server snapshot has been released (drives hydration of its views). */
1215
3271
  hydrated: boolean;
3272
+ /** The authority channel (gate/source key) this sub registered on — fixed at retain time
3273
+ * (G-iii registration-time routing), `"daemon"` unless a channel-keyed retain named another.
3274
+ * The ONE source of truth for qid→channel ownership: `onFrame`'s wrong-channel assertion and
3275
+ * the per-gate `overflow` both read it (via {@link OptimisticBackend.channelOf} / directly). */
3276
+ channel: string;
1216
3277
  }
1217
3278
 
1218
3279
  function remoteKey(remote: RemoteQuery): string {
@@ -1272,18 +3333,140 @@ function colIndexFromSchema<S extends ColsMap>(schema: Schema<S>): Record<string
1272
3333
  return out;
1273
3334
  }
1274
3335
 
1275
- /** Wrap the raw wasm txn as the client `MutationTx`, recording the touched tables (the client
1276
- * knows its own footprint what the pending axis derives from, §7.2). The keyed methods validate
1277
- * column names eagerly: a typo'd table or column throws with the valid names listed, at the moment
1278
- * the mutator runs.
3336
+ /** Merge a fresh invocation's write-set into a `PendingMutation`'s accumulated one (§3.2 #1, rebase
3337
+ * re-invocation): each pk's value is OVERWRITTEN with the newest image (a later invocation ran
3338
+ * against the base the rebase just replaced, so its view supersedes the earlier one), but a key
3339
+ * present only in `dest` is left alone — the same union-never-shrink rule `touched` already
3340
+ * follows (§7.2: "a re-run that no-ops must NOT shrink it"). */
3341
+ function mergeWriteSet(dest: WriteSet, src: WriteSet): void {
3342
+ for (const [table, byPk] of src) {
3343
+ let d = dest.get(table);
3344
+ if (!d) dest.set(table, (d = new Map()));
3345
+ for (const [pkKey, rec] of byPk) d.set(pkKey, rec);
3346
+ }
3347
+ }
3348
+
3349
+ /** The key a per-pk write source is stored under in {@link PendingMutation.writeSources}: `table`
3350
+ * + NUL + the pk-key (`stableJson(pk)`). NUL appears in neither a table name nor `stableJson`
3351
+ * output, so it is an unambiguous separator — the same key the confirm-drop hold-back reconstructs
3352
+ * from `(table, pkKey)` while iterating `writes`. */
3353
+ function writeSourceKey(table: string, pkKey: string): string {
3354
+ return `${table}\u0000${pkKey}`;
3355
+ }
3356
+
3357
+ /** Zip the staged-order per-write source keys (`stagedKeys`, from `trackingTx`) with the staged-order
3358
+ * `sources[]` (`commitTracked`) into `dest`, last-write-per-key winning (§5.3). Never shrinks `dest`
3359
+ * — a rebase re-invocation MERGES its fresh routing in, mirroring {@link mergeWriteSet}. The two
3360
+ * arrays align 1:1: `trackingTx` pushes exactly one key per staged change, and `commitTracked`
3361
+ * returns one source per staged change, both in staged order. */
3362
+ function mergeWriteSources(dest: Map<string, string>, stagedKeys: string[], sources: string[]): void {
3363
+ const n = Math.min(stagedKeys.length, sources.length);
3364
+ for (let i = 0; i < n; i++) dest.set(stagedKeys[i], sources[i]);
3365
+ }
3366
+
3367
+ // --- the §3 router's pure helpers (H-iii) --------------------------------------------
3368
+
3369
+ /** STRICT cell identity for the join-key no-change rule (§3 write rule #3): `===`, nothing more —
3370
+ * no coercion, no deep equality (a non-primitive join-key cell compares by reference and thus
3371
+ * conservatively fails), `NaN !== NaN` conservatively fails. Failing closed here only costs a
3372
+ * daemon route. */
3373
+ function identicalCell(a: WireValue, b: WireValue): boolean {
3374
+ return a === b;
3375
+ }
3376
+
3377
+ /** Whether `cond` is decidable from the KEY ALONE: every column it references — collected from
3378
+ * BOTH operand positions (under-counting could over-claim decidability; over-counting only fails
3379
+ * closed — the H-iv-a discipline) — is a pk column. A `correlatedSubquery` is never key-local. */
3380
+ // Exported for test/predicate_agreement.test.ts — the TS corner of the cross-evaluator
3381
+ // agreement corpus (rust/rindle-room-core/tests/fixtures/predicate-agreement.json). NOT part of
3382
+ // the package surface (index.ts curates exports); the corpus is the drift tripwire that keeps
3383
+ // this evaluator honest against the engine + room-gate pair it shares no code with.
3384
+ export function keyDecidable(cond: Condition, pkCols: ReadonlySet<string>): boolean {
3385
+ switch (cond.type) {
3386
+ case "simple": {
3387
+ if (cond.left.type === "column" && !pkCols.has(cond.left.name)) return false;
3388
+ if (cond.right.type === "column" && !pkCols.has(cond.right.name)) return false;
3389
+ return true;
3390
+ }
3391
+ case "and":
3392
+ case "or":
3393
+ return cond.conditions.every((c) => keyDecidable(c, pkCols));
3394
+ case "correlatedSubquery":
3395
+ return false;
3396
+ }
3397
+ }
3398
+
3399
+ /** Evaluate a key-decidable `footprintWhere` on a read's pk cells — H-iii's ONE client-side
3400
+ * non-engine evaluator, DELIBERATELY MINIMAL (the TS evaluation caveat on the router block
3401
+ * comment): `simple` `=`/`!=` between a pk COLUMN and a same-primitive-type NON-NULL literal,
3402
+ * composed under and/or. The empty AND — the compiler's vacuous-true emission for an exact
3403
+ * unconstrained footprint root — evaluates `true` (load-bearing: whole-table footprints keep
3404
+ * provable reads); the empty OR is vacuous-false. Returns `undefined` = NOT EVALUABLE for
3405
+ * anything else (other ops, null on either side, cross-type comparisons, column-vs-column,
3406
+ * literal-vs-literal, non-primitive cells, correlated subqueries) — the caller fails to the
3407
+ * daemon. A non-evaluable node anywhere poisons the whole tree (no short-circuit past it):
3408
+ * partial evaluation could otherwise claim a verdict the engine's semantics might contradict. */
3409
+ // Exported for test/predicate_agreement.test.ts (see keyDecidable's note above).
3410
+ export function evalFootprintOnPk(cond: Condition, pkCells: ReadonlyMap<string, WireValue>): boolean | undefined {
3411
+ switch (cond.type) {
3412
+ case "simple": {
3413
+ if (cond.op !== "=" && cond.op !== "!=") return undefined;
3414
+ const col = cond.left.type === "column" ? cond.left : cond.right.type === "column" ? cond.right : undefined;
3415
+ const lit = cond.left.type === "literal" ? cond.left : cond.right.type === "literal" ? cond.right : undefined;
3416
+ if (col === undefined || lit === undefined) return undefined; // column-vs-column / literal-vs-literal
3417
+ if (!pkCells.has(col.name)) return undefined; // not a pk column (keyDecidable pre-screens)
3418
+ const cell = pkCells.get(col.name);
3419
+ const value = lit.value;
3420
+ if (cell === null || value === null) return undefined; // no null semantics client-side
3421
+ if (typeof cell !== typeof value) return undefined; // no cross-type comparison semantics
3422
+ if (typeof cell !== "string" && typeof cell !== "number" && typeof cell !== "boolean") return undefined;
3423
+ return cond.op === "=" ? cell === value : cell !== value;
3424
+ }
3425
+ case "and": {
3426
+ let out = true;
3427
+ for (const c of cond.conditions) {
3428
+ const v = evalFootprintOnPk(c, pkCells);
3429
+ if (v === undefined) return undefined;
3430
+ out = out && v;
3431
+ }
3432
+ return out;
3433
+ }
3434
+ case "or": {
3435
+ let out = false;
3436
+ for (const c of cond.conditions) {
3437
+ const v = evalFootprintOnPk(c, pkCells);
3438
+ if (v === undefined) return undefined;
3439
+ out = out || v;
3440
+ }
3441
+ return out;
3442
+ }
3443
+ case "correlatedSubquery":
3444
+ return undefined;
3445
+ }
3446
+ }
3447
+
3448
+ /** Wrap the raw wasm txn as the client `MutationTx`, capturing a pk-granular write-set as it
3449
+ * applies (`writes`, a {@link WriteSet} — table → pk-key → last-write-wins image, §3.2 #1);
3450
+ * `touched` (the pending axis's table-granular Set, §7.2) is derived by the CALLER as
3451
+ * `new Set(writes.keys())`, never populated here. The keyed methods validate column names eagerly:
3452
+ * a typo'd table or column throws with the valid names listed, at the moment the mutator runs.
3453
+ *
3454
+ * With `trapReads` (the FOLDED path, §5), the PUBLIC reads `tx.get`/`tx.row`/`tx.query` throw
3455
+ * `FoldReadError` — a folded mutator that reads state to compute its write is non-absorbing and
3456
+ * refused. The keyed writers (`update`/`upsert`/`insertIgnore`/`delete`) still read internally to
3457
+ * preserve unspecified columns / check pre-existence; that is fold-legal (the trap wraps only the
3458
+ * returned object's `get`/`row`/`query` surface, never the writers' internal probe) — unchanged
3459
+ * by H-ii, which records those probes but arms recording only where the trap never is.
1279
3460
  *
1280
- * With `trapReads` (the FOLDED path, §5), the PUBLIC reads `tx.get`/`tx.row` throw `FoldReadError`
1281
- * a folded mutator that reads state to compute its write is non-absorbing and refused. The keyed
1282
- * writers (`update`/`upsert`/`delete`) still read internally to preserve unspecified columns; that
1283
- * is column-preservation, not value-derivation, so it stays allowed. */
3461
+ * With `readLog` (recording mode, RINDLE-REALTIME-QUERY-ENABLEMENT-DESIGN.md §3.2 #2) a SIBLING
3462
+ * of `trapReads`, the two never armed together by any call site the PUBLIC `tx.get`/`tx.row`
3463
+ * push a `(table, pk, outcome, source?)` {@link ReadRecord} (per-read provenance via the
3464
+ * `provenance` probe, H-ii §3.2 #3), `tx.query` pushes its resolved AST, and (H-ii) the keyed
3465
+ * writers' pre-existence probes record through the same path. Pure capture: it changes no return
3466
+ * value, throws nothing, and is a no-op when `readLog` is omitted. */
1284
3467
  /** Apply one logical {@link MutationOp} (yielded by a shared generator mutator) onto the client's
1285
3468
  * keyed {@link MutationTx} — the same methods a plain client mutator calls directly. Column
1286
- * validation, touched-table tracking, and op collection all happen inside those methods. */
3469
+ * validation, write-set capture, and op collection all happen inside those methods. */
1287
3470
  function applyOpToTx(tx: MutationTx, op: MutationOp): void {
1288
3471
  switch (op.kind) {
1289
3472
  case "insert":
@@ -1301,11 +3484,17 @@ function applyOpToTx(tx: MutationTx, op: MutationOp): void {
1301
3484
 
1302
3485
  function trackingTx(
1303
3486
  tx: WasmWriteTxn,
1304
- touched: Set<string>,
3487
+ writes: WriteSet,
1305
3488
  specs: TableSpecs,
1306
3489
  localTables: Set<string>,
1307
3490
  onOp?: (op: ChildOp) => void,
1308
3491
  trapReads = false,
3492
+ readLog?: ReadLog,
3493
+ stagedKeys?: string[],
3494
+ /** Per-read provenance probe (H-ii §3.2 #3): called once per RECORDED present read with the
3495
+ * full-width row the read observed; meaningful only alongside `readLog` (never consulted when
3496
+ * recording is off — the fold and replay paths pass neither). */
3497
+ provenance?: (table: string, row: WireValue[]) => string | undefined,
1309
3498
  ): MutationTx {
1310
3499
  const spec = (table: string) => {
1311
3500
  const s = specs[table];
@@ -1348,6 +3537,28 @@ function trackingTx(
1348
3537
  const pkCells = (table: string, obj: KeyedRow): WireValue[] =>
1349
3538
  spec(table).primaryKey.map((i) => obj[spec(table).columns[i]]);
1350
3539
 
3540
+ // pk from the POSITIONAL wire shape (raw cells in schema column order) — the counterpart of
3541
+ // `pkCells` (which reads a KeyedRow) for the raw `add`/`remove`/`edit` writers below (§3.2 #1).
3542
+ const pkFromCells = (table: string, cells: WireValue[]): WireValue[] =>
3543
+ spec(table).primaryKey.map((i) => cells[i]);
3544
+
3545
+ // Record (or overwrite) this pk's write for the invocation (§3.2 #1): last-write-wins WITHIN
3546
+ // this invocation — an add-then-edit (or edit-then-edit) of the same pk collapses to its final
3547
+ // image, matching the engine head's own semantics for that pk. The record is replaced with
3548
+ // exactly the arguments given: the CALLERS (`edit`/`remove` below, consulting `prior`) decide
3549
+ // the pre-image per the H-ii coalescing matrix on {@link WriteRecord}.
3550
+ const recordWrite = (table: string, pk: WireValue[], row: WireValue[] | undefined, oldRow?: WireValue[]): void => {
3551
+ let byPk = writes.get(table);
3552
+ if (!byPk) writes.set(table, (byPk = new Map()));
3553
+ const pkKey = stableJson(pk);
3554
+ // Defensive copies: the wasm binding's returned arrays are not contractually immutable/unique,
3555
+ // so a captured record must not alias a cell array the engine could later reuse or mutate.
3556
+ byPk.set(pkKey, { table, pk: [...pk], row: row ? [...row] : undefined, ...(oldRow ? { oldRow: [...oldRow] } : {}) });
3557
+ // Staged-order key trace (§5.3): pushed once per staged change, so it zips 1:1 with the staged
3558
+ // order of `commitTracked`'s `sources[]` — letting the caller learn each write's routed source.
3559
+ stagedKeys?.push(writeSourceKey(table, pkKey));
3560
+ };
3561
+
1351
3562
  // A full insert row: each cell is `obj[c]`, or `null` for an omitted nullable column (design 206
1352
3563
  // §6.2); a `json` object is stringified for the engine (`toCell`). Non-nullable columns are
1353
3564
  // guaranteed present by `checkColumns(full)`.
@@ -1362,27 +3573,102 @@ function trackingTx(
1362
3573
  return out;
1363
3574
  };
1364
3575
 
1365
- // Internal read used by the keyed writers below and the keyed `row` reader; never trapped (for
1366
- // the FOLD trap) but ALWAYS guarded against local reads (M1).
3576
+ // The raw, UN-recorded read primitive behind `getImpl`/`rowImpl` the M1 local guard + the txn
3577
+ // read, nothing else. Slice B deliberately kept the keyed writers (`update`/`upsert`/
3578
+ // `insertIgnore`/`delete`) on this, un-recorded ("recording is about the PUBLIC read entry
3579
+ // points"). H-ii deliberately REVERSES that: the pre-existence probe each keyed writer BRANCHES
3580
+ // on is genuine value-dependence the §3 routing proof must see — the concrete silent-drop shape
3581
+ // is `update("cards", {id:5,…})` where the client's daemon slice has the row but the room's
3582
+ // footprint lacks it: the room-side update no-ops, the commit "succeeds" with zero effects, the
3583
+ // confirm retires the entry, and the user's edit silently vanishes. The proof can only refuse
3584
+ // that route if the probe is on the record. So the keyed writers now probe through `getImpl`
3585
+ // (recorded like any public read, §3.2 #3); the FOLD trap is unaffected — it wraps only the
3586
+ // returned object's `get`/`row`/`query` surface, so keyed writers stay fold-legal and the
3587
+ // trapped path (where `readLog` is never armed) records nothing, exactly as before.
1367
3588
  const rawGet = (table: string, pk: WireValue[]) => {
1368
3589
  assertNotLocal(table, "read");
1369
3590
  return tx.get(table, pk) as WireValue[] | undefined;
1370
3591
  };
3592
+
3593
+ // Push one {@link ReadRecord} when recording is armed (§3.2 #2/#3): outcome from `row`'s
3594
+ // presence, per-read provenance probed with the observed FULL-WIDTH row — present reads only
3595
+ // (an absent read has no row to probe with, and records no source; see {@link ReadRecord}).
3596
+ // The `source` key is OMITTED (not set to undefined) when there is no answer, so a
3597
+ // single-domain record is byte-identical to Slice B's.
3598
+ const recordRead = (table: string, pk: WireValue[], row: WireValue[] | undefined): void => {
3599
+ if (!readLog) return;
3600
+ const source = row === undefined ? undefined : provenance?.(table, row);
3601
+ readLog.reads.push({
3602
+ table,
3603
+ pk: [...pk],
3604
+ outcome: row === undefined ? "absent" : "present",
3605
+ ...(source !== undefined ? { source } : {}),
3606
+ });
3607
+ };
3608
+
3609
+ // The PUBLIC positional read (§3.2 #2) — and, since H-ii, the keyed writers' pre-existence
3610
+ // probe (§3.2 #3, see the `rawGet` note): `rawGet` plus a `readLog` record when recording is
3611
+ // armed. A no-op record when `readLog` is omitted — exactly `rawGet`'s behavior then.
3612
+ const getImpl = (table: string, pk: WireValue[]): WireValue[] | undefined => {
3613
+ const result = rawGet(table, pk);
3614
+ recordRead(table, pk, result);
3615
+ return result;
3616
+ };
3617
+
3618
+ // The PUBLIC keyed read (§3.2 #2), the `row` counterpart of `getImpl`.
3619
+ const rowImpl = (table: string, pk: KeyedRow): KeyedRow | undefined => {
3620
+ checkColumns(table, pk, false);
3621
+ const pkc = pkCells(table, pk);
3622
+ const cells = rawGet(table, pkc);
3623
+ recordRead(table, pkc, cells);
3624
+ return cells ? toKeyed(table, cells) : undefined;
3625
+ };
3626
+
3627
+ // The pk's existing record from THIS invocation, if any — the coalescing-matrix input for
3628
+ // `edit`/`remove` below (see {@link WriteRecord}).
3629
+ const prior = (table: string, pk: WireValue[]): WriteRecord | undefined =>
3630
+ writes.get(table)?.get(stableJson(pk));
1371
3631
  const add = (table: string, row: WireValue[]) => {
1372
3632
  assertNotLocal(table, "write");
1373
- touched.add(table);
3633
+ recordWrite(table, pkFromCells(table, row), row);
1374
3634
  onOp?.({ table, kind: "add", row });
1375
3635
  tx.add(table, row);
1376
3636
  };
1377
3637
  const remove = (table: string, row: WireValue[]) => {
1378
3638
  assertNotLocal(table, "write");
1379
- touched.add(table);
3639
+ const pk = pkFromCells(table, row);
3640
+ // The remove PRE-IMAGE (G-iii, §7.3 tombstone; the H-ii matrix on {@link WriteRecord}):
3641
+ // remove-after-edit/-remove keeps the ORIGINAL captured pre-image (the txn-entry base — the
3642
+ // net effect is a remove of the row the external world last knew, never the edited transient).
3643
+ // Otherwise (first touch, or remove-after-add) the truthful full-width row is the txn-visible
3644
+ // one — `tx.get` read BEFORE the remove stages (read-your-writes: an add of this pk earlier in
3645
+ // the SAME invocation shows through). Captured NOW so Slice H's writable-predicate evaluation
3646
+ // over it needs no migration. Falls back to the caller's asserted `row` when the pk is not
3647
+ // resident (a raw remove of an absent row) — a captured remove thus always carries a
3648
+ // full-width pre-image (the engine width-checks `holdBackAbsent`).
3649
+ const oldRow = prior(table, pk)?.oldRow ?? (tx.get(table, pk) as WireValue[] | undefined) ?? row;
3650
+ recordWrite(table, pk, undefined, oldRow);
1380
3651
  onOp?.({ table, kind: "remove", row });
1381
3652
  tx.remove(table, row);
1382
3653
  };
1383
3654
  const edit = (table: string, oldRow: WireValue[], newRow: WireValue[]) => {
1384
3655
  assertNotLocal(table, "write");
1385
- touched.add(table);
3656
+ const pk = pkFromCells(table, newRow);
3657
+ // The edit PRE-IMAGE (H-ii; the matrix on {@link WriteRecord}) — H-iii's join-key no-change
3658
+ // input, and the row the engine routes the Edit by (H-i). First touch: the txn-visible row
3659
+ // read BEFORE staging, falling back to the caller's asserted `oldRow` when the pk is not
3660
+ // resident (covers the pk-MOVING raw edit — the record is keyed by the NEW pk; the pre-image
3661
+ // carries the OLD row). Edit-after-edit: keep the FIRST pre-image (the txn-entry base).
3662
+ // Edit-after-add / edit-after-remove: the record collapses to a (re-)insert — NO pre-image
3663
+ // (the pk did not pre-exist this invocation's base).
3664
+ const p = prior(table, pk);
3665
+ const pre =
3666
+ p === undefined
3667
+ ? ((tx.get(table, pk) as WireValue[] | undefined) ?? oldRow)
3668
+ : p.row !== undefined && p.oldRow !== undefined
3669
+ ? p.oldRow
3670
+ : undefined;
3671
+ recordWrite(table, pk, newRow, pre);
1386
3672
  onOp?.({ table, kind: "edit", row: newRow, old: oldRow });
1387
3673
  tx.edit(table, oldRow, newRow);
1388
3674
  };
@@ -1401,29 +3687,28 @@ function trackingTx(
1401
3687
  const runQuery = (q: QueryArg): QueryResultRow[] => {
1402
3688
  const ast = q.ast();
1403
3689
  for (const t of collectTables(ast)) assertNotLocal(t, "read");
3690
+ readLog?.queries.push(ast);
1404
3691
  return tx.query(ast) as QueryResultRow[];
1405
3692
  };
1406
3693
 
1407
3694
  return {
1408
- get: trapReads ? trapped : rawGet,
3695
+ get: trapReads ? trapped : getImpl,
1409
3696
  query: trapReads ? trapped : runQuery,
1410
3697
  add,
1411
3698
  remove,
1412
3699
  edit,
1413
- row: trapReads
1414
- ? trapped
1415
- : (table, pk) => {
1416
- checkColumns(table, pk, false);
1417
- const cells = rawGet(table, pkCells(table, pk));
1418
- return cells ? toKeyed(table, cells) : undefined;
1419
- },
3700
+ row: trapReads ? trapped : rowImpl,
1420
3701
  insert: (table, row) => {
1421
3702
  checkColumns(table, row, true);
1422
3703
  add(table, toCells(table, row));
1423
3704
  },
3705
+ // The keyed writers' pre-existence probes go through `getImpl` — RECORDED reads since H-ii
3706
+ // (§3.2 #3): each writer BRANCHES on the probe, a value-dependence the routing proof must see
3707
+ // (the silent-drop rationale on `rawGet` above). Fold-legal exactly as before (the trap wraps
3708
+ // the public surface above, never these), and byte-identical when recording is off.
1424
3709
  update: (table, row) => {
1425
3710
  checkColumns(table, row, false);
1426
- const current = rawGet(table, pkCells(table, row));
3711
+ const current = getImpl(table, pkCells(table, row));
1427
3712
  if (!current) return; // rebase-friendly: the row may have vanished upstream
1428
3713
  const s = spec(table);
1429
3714
  // Named columns overwrite (a `json` object stringified via `toCell`); unnamed keep `current`,
@@ -1433,17 +3718,17 @@ function trackingTx(
1433
3718
  },
1434
3719
  upsert: (table, row) => {
1435
3720
  checkColumns(table, row, true);
1436
- const current = rawGet(table, pkCells(table, row));
3721
+ const current = getImpl(table, pkCells(table, row));
1437
3722
  if (current) edit(table, current, toCells(table, row));
1438
3723
  else add(table, toCells(table, row));
1439
3724
  },
1440
3725
  insertIgnore: (table, row) => {
1441
3726
  checkColumns(table, row, true);
1442
- if (!rawGet(table, pkCells(table, row))) add(table, toCells(table, row));
3727
+ if (!getImpl(table, pkCells(table, row))) add(table, toCells(table, row));
1443
3728
  },
1444
3729
  delete: (table, pk) => {
1445
3730
  checkColumns(table, pk, false);
1446
- const current = rawGet(table, pkCells(table, pk));
3731
+ const current = getImpl(table, pkCells(table, pk));
1447
3732
  if (!current) return; // rebase-friendly no-op
1448
3733
  remove(table, current);
1449
3734
  },
@@ -1478,10 +3763,3 @@ function intersects(a: Set<string>, b: Set<string>): boolean {
1478
3763
  return false;
1479
3764
  }
1480
3765
 
1481
- /** Elementwise equality over positional bare-cell rows (the synthetic `__agg` rows are all
1482
- * scalar cells — group key + count — so identity comparison suffices). */
1483
- function rowsEqual(a: WireValue[], b: WireValue[]): boolean {
1484
- if (a.length !== b.length) return false;
1485
- for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
1486
- return true;
1487
- }