@ultimat3/realtime 23.0.0 → 25.0.0

package/src/live-definition.ts

@@ -9,8 +9,14 @@
 // every time. Collapsing the second onto the first is privilege escalation with a cache hit rate.
 
 import type { Ctx } from '@ultimat3/core';
-import { finiteOption, runWithContext, serviceActor, withChildContext } from '@ultimat3/core';
-import { type AnyQuery, queryHash, queryName } from '@ultimat3/query';
+import {
+  anonymousActor,
+  finiteOption,
+  runWithContext,
+  serviceActor,
+  withChildContext,
+} from '@ultimat3/core';
+import { type AnyQuery, queryHash, queryName, spendQueryLimit } from '@ultimat3/query';
 import { LiveRowUnidentifiedError } from './errors';
 import { isRow, type JsonValue, type Row } from './json';
 import type { LiveQueryDefinition, SnapshotResult } from './live-contract';
@@ -27,8 +33,10 @@ export interface LiveDefinitionOptions {
    */
   readonly ctx: Ctx;
   /**
-   * Where the shared window sits in the change stream, asked at snapshot time. Without a feed
-   * position a reconnect can only re-snapshot, so a node with a replicator should pass its lsn.
+   * Where the shared window sits in the change stream, asked at snapshot time — the node's newest
+   * received change (`registry.lastLsn`). Omitted, or answering `''` before the first change, the
+   * snapshot claims no position and the retained ring is floored by its first patch instead: a
+   * cursor minted before that patch re-snapshots, one that received it resumes as a delta.
    */
   readonly lsn?: () => string;
   /** Pins the reconnect epoch in tests; the server derives it from the build. */
@@ -165,6 +173,9 @@ export function liveQueryDefinition(
       // The position is taken BEFORE the rows: the rows are then at least that new, so the claim
       // is true. Taken after, a commit landing mid-read was claimed and missing — and every change
       // up to it is dropped downstream as already folded.
+      // `''` when the node holds no position — no `lsn` option, or no change received yet. Never
+      // a made-up one: a position the node never held becomes a ring floor every foreign cursor
+      // is "inside", and the client that resumed there kept rows this node never patched.
       const lsn = options.lsn?.() ?? '';
       return { rows: await readFor(options.ctx, tenant, () => window.read()), lsn };
     },
@@ -177,6 +188,10 @@ export function liveQueryDefinition(
     // memoised anywhere: `authorize` runs on every subscribe, `visible` on every row of every
     // delivery, and there is no key here an actor could share with another actor.
     authorize: authorizeWithPolicy(target.policy, { query: name, ctx: options.ctx }),
+    // The read's own `rateLimit:` bucket, per subscriber — the one HTTP and MCP spend. A socket
+    // with no actor is the anonymous reader, keyed by its address as an unauthenticated HTTP read is.
+    spend: ({ actor, clientAddress, nowMs }) =>
+      spendQueryLimit(target, { actor: actor ?? anonymousActor(), clientAddress, nowMs }),
     visible: visibleWithPolicy(target.policy, { query: name, ctx: options.ctx }),
   };
 }
@@ -191,15 +206,25 @@ function evictOldest(windows: Map<string, SharedWindow>, max: number): void {
 }
 
 /**
- * Rows crossing the wire are addressed by `id` — patches, cursors and the local store all key on
- * it. A projection without one is refused here rather than delivered as a row nobody can patch:
- * the alternative is a subscription that appears to work until the first update.
+ * Rows crossing the wire are addressed by a TEXT `id` — patches, cursors and the local store all
+ * key on it. A safe-integer id is stringified, the one normalisation the change feed applies to the
+ * same row (`pg-replication.ts`'s `toRow`), so a snapshot row and its later patch name one identity.
+ * Anything else is refused here rather than delivered as a row nobody can patch.
  */
 function rowsOf(query: string, rows: readonly object[]): readonly Row[] {
   const out: Row[] = [];
   for (const row of rows) {
-    if (!isRow(row)) throw new LiveRowUnidentifiedError({ query, keys: Object.keys(row) });
-    out.push(row);
+    const id: unknown = (row as { id?: unknown }).id;
+    const candidate =
+      typeof id === 'number' && Number.isSafeInteger(id) ? { ...row, id: String(id) } : row;
+    if (!isRow(candidate)) {
+      throw new LiveRowUnidentifiedError({
+        query,
+        keys: Object.keys(row),
+        ...(id === undefined || id === null ? {} : { idType: typeof id }),
+      });
+    }
+    out.push(candidate);
   }
   return out;
 }

package/src/live-deliver.ts

@@ -0,0 +1,55 @@
+// One change, fanned out over every live window on the node. Split from `live-query.ts` at its
+// size ceiling: the registry owns who holds which window, this owns the ordering of one delivery.
+
+import type { ChangeEvent } from './changefeed';
+import { type FanoutDeps, fanoutChange } from './live-fanout';
+import type { QueryEntry } from './query-window';
+
+/**
+ * Fan one change out. Matched once per query id, authorized once per subscriber. Returns the
+ * number of frames sent — the metric the reconnect benchmark watches.
+ *
+ * Each entry's turn is taken in that entry's lane. Nothing upstream orders this: `sync` fires
+ * `void registry.deliver(change)` straight off the bus subscription, so two changes arriving back
+ * to back would otherwise interleave inside one query id — lsn 2 delivered before lsn 1, the
+ * subscriber's cursor rewound to 1, and every gate deciding against whichever window won.
+ *
+ * Every lane is *entered* before any of them is awaited, and nothing inside a fanout takes a
+ * second lane, so holding all of them at once cannot be a cycle. That is what makes the ordering
+ * claim true: two deliveries queue onto each query id in call order, serialized per query id and
+ * never per node — awaiting one entry before entering the next made one slow policy pass the
+ * whole node's pace, and let a lane that threw skip every entry behind it with nobody desynced.
+ */
+export async function deliverChange(
+  entries: Iterable<QueryEntry>,
+  fanout: FanoutDeps,
+  change: ChangeEvent,
+  onStale: (count: number) => void,
+): Promise<number> {
+  const lanes = [...entries].map(async (entry) => {
+    try {
+      const result = await entry.lock.run(() => fanoutChange(fanout, entry, change));
+      onStale(result.stale);
+      return result.sent;
+    } catch (error) {
+      // The window advanced under a fanout that did not finish, so every subscriber of this one
+      // query id now holds a cursor below the change and no later flush would correct them:
+      // desynced here, re-snapshotted on the next one. Silent divergence is the whole reason
+      // `markDesynced` exists, and skipping this is how a failure became one.
+      for (const subscription of entry.subscribers.values()) {
+        subscription.socket.markDesynced(subscription.sid);
+      }
+      throw error;
+    }
+  });
+  // `allSettled`, so one lane's rejection neither cancels the others nor goes unhandled. The
+  // first failure still reaches the caller — `sync` logs it — but it costs one query id.
+  let sent = 0;
+  let failure: { readonly error: unknown } | null = null;
+  for (const lane of await Promise.allSettled(lanes)) {
+    if (lane.status === 'fulfilled') sent += lane.value;
+    else failure ??= { error: lane.reason };
+  }
+  if (failure !== null) throw failure.error;
+  return sent;
+}

package/src/live-fanout.ts

@@ -3,15 +3,16 @@
 // registry owns the lanes and the entry table while this owns what happens inside one of them —
 // and because one file runs one job. The lane itself is never taken here: the caller is holding it.
 
-import type { Clock } from '@ultimat3/core';
+import { type Clock, isWriteDigest } from '@ultimat3/core';
 import type { ChangeEvent } from './changefeed';
 import { advance, type LiveCursor, makeCursor, type ResumeSource } from './cursor';
 import type { Row, RowPatch } from './json';
 import type { LiveSubscription } from './live-contract';
-import { applyToWindow, bridgeChange } from './matcher-bridge';
+import { applyToWindow, bridgeChange, type Projection, projectionOf } from './matcher-bridge';
 import { type QueryEntry, refillWindowInLane } from './query-window';
 import { type Subscriber, type SubscriberGate, windowIndex } from './subscriber-gate';
 import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
+import { FRAME_LIMITS } from './wire-version';
 
 export interface FanoutDeps {
   readonly gate: SubscriberGate;
@@ -47,9 +48,17 @@ export async function fanoutChange(
     return { sent: 0, stale: 0 };
   }
   if (change.op === 'truncate') return await truncated(deps, entry, change);
+  // Screened, not trusted: a frame carrying anything but a digest is refused WHOLE by the decoder,
+  // so a malformed label would cost every subscriber this change rather than one page its echo.
+  const write = isWriteDigest(change.write) ? change.write : null;
+  const writes = write === null ? [] : [write];
+  // What a re-snapshot out of rows that hold this change may name: the write, by the row it made.
+  const named = namedBy(entry, change, write);
   const reread = entry.stale;
   if (reread) {
     await refillWindowInLane(entry);
+    // What was retained before this read has a hole exactly where the staleness came from.
+    deps.source.floorAt?.(entry.qid, entry.lsn, { exclusive: true });
     // Every subscriber was fed from the rows this read just replaced, so each is owed a snapshot —
     // not only the ones `invalidate()` marked: a read that failed or timed out stales the window
     // and marks nobody.
@@ -60,7 +69,7 @@ export async function fanoutChange(
     // there is nothing to patch. Returning before the loop below was the defect: the guard after
     // this refused the change as stale and every desynced subscriber went without the change AND
     // without its snapshot — a run console frozen on its first event after one `updateWhere`.
-    if (change.lsn <= entry.lsn) return { sent: await resnapshotAll(deps, entry), stale: 0 };
+    if (change.lsn <= entry.lsn) return { sent: await resnapshotAll(deps, entry, named), stale: 0 };
   }
   // The consume-side twin of the replicator's own duplicate guard, which had none. `entry.lsn =
   // change.lsn` was unconditional, so a change the window already holds — a redelivery, or one
@@ -68,23 +77,33 @@ export async function fanoutChange(
   // to it and asked them to fold state they had already folded over newer rows.
   if (entry.lsn !== '' && change.lsn <= entry.lsn) return { sent: 0, stale: 1 };
   const bridged = bridgeChange(entry.shape, entry.matcher, change, entry.rows);
-  // Matching nothing ends the fanout, but never before a re-read's marks are answered.
+  // Matching nothing ends the fanout, but never before a re-read's marks are answered. Named
+  // nothing: that read began before this change, so it is not truth that holds its write.
   if (!bridged) return { sent: reread ? await resnapshotAll(deps, entry) : 0, stale: 0 };
   // Keyed ONCE, here, before the retained window stores them: a resume replays the same key.
   const result = { ...bridged, patches: keyPatches(entry, change, bridged.patches) };
+  // Asked of the window as it stood BEFORE this change: the partial row must not teach it a shape.
+  const partial = lacksOmitted(change, result.patches, projectionOf(entry.rows));
   entry.lsn = change.lsn;
   entry.rows = applyToWindow(entry.rows, result.patches);
-  // The window lost its tail, so what it holds is a guess — the next delivery re-reads it rather
-  // than patching a guess, and every subscriber below is re-snapshotted out of what that returns.
-  if (result.refill) entry.stale = true;
-  // The retained window holds the pre-policy patch; resume re-filters it per subscriber.
-  for (const patch of result.patches) deps.source.append(entry.qid, patch);
+  // The window lost its tail — or adopted a row the change could not carry whole — so what it
+  // holds is a guess: the next delivery re-reads it rather than patching a guess, and every
+  // subscriber below is re-snapshotted out of what that returns.
+  const guessed = result.refill || partial;
+  if (guessed) entry.stale = true;
+  // The retained window holds the pre-policy patch; resume re-filters it per subscriber. A partial
+  // row is never retained: replayed, it is the same missing column on a resuming client. Each
+  // retained patch keeps the write that made it, so a delta resume can name it too.
+  if (!partial)
+    for (const patch of result.patches) deps.source.append(entry.qid, retain(patch, write));
+  // And the ring says so now, not at the re-read: a resume in between got a delta lacking it.
+  else deps.source.floorAt?.(entry.qid, change.lsn, { exclusive: true });
 
   let sent = 0;
   // Indexed once for every subscriber below, never searched per subscriber per patch.
   const index = windowIndex(entry.rows);
   for (const subscription of entry.subscribers.values()) {
-    if (result.refill) {
+    if (guessed) {
       // The window lost its tail: guessing is how a sync engine silently diverges. Checked BEFORE
       // the mark, because a repair reads `entry.rows` — which this fanout has just declared a
       // guess — and then CLEARS the mark. A subscriber already diverged would be recorded as
@@ -100,7 +119,8 @@ export async function fanoutChange(
     // healthy socket. A marked subscriber is re-snapshotted out of the shared window instead, at
     // the cost of one frame and no DB read, and only then is the mark cleared.
     if (subscription.socket.desynced.has(subscription.sid)) {
-      if (await resnapshot(deps, entry, subscription)) sent += 1;
+      // Out of the window this change was just folded into, so it holds the change's write.
+      if (await resnapshot(deps, entry, subscription, named)) sent += 1;
       continue;
     }
     const who: Subscriber = { sid: subscription.sid, actor: subscription.socket.actor };
@@ -122,13 +142,7 @@ export async function fanoutChange(
       continue;
     }
     if (allowed.length === 0) continue;
-    const frame: Frame = {
-      type: 'patch',
-      v: PROTOCOL_VERSION,
-      sid: subscription.sid,
-      patches: allowed,
-      lsn: change.lsn,
-    };
+    const frame = patchFrame(subscription.sid, allowed, change.lsn, writes);
     if (subscription.socket.send(frame)) {
       subscription.cursor = advance(subscription.cursor, allowed, change.lsn, change.at);
       sent += 1;
@@ -139,6 +153,32 @@ export async function fanoutChange(
   return { sent, stale: 0 };
 }
 
+/**
+ * Whether a row ENTERS the window without a column the change did not carry
+ * (`ChangeEvent.omitted`: an unchanged out-of-line value Postgres logged no bytes for). An update
+ * patch is safe — it merges onto a window row that already holds the column — but an add adopts
+ * the row as the whole row, into the window, every later snapshot and the retained ring. An
+ * unread projection (an empty window) cannot rule a column out, so every omitted one counts.
+ */
+function lacksOmitted(
+  change: ChangeEvent,
+  patches: readonly RowPatch[],
+  projection: Projection,
+): boolean {
+  const omitted = change.omitted;
+  if (omitted === undefined || omitted.length === 0) return false;
+  return patches.some(
+    (patch) =>
+      patch.op === 'insert' &&
+      patch.row !== null &&
+      omitted.some(
+        (column) =>
+          (projection === undefined || projection.has(column)) &&
+          !Object.hasOwn(patch.row ?? {}, column),
+      ),
+  );
+}
+
 /**
  * Every row of a relation this window reads is gone. There is nothing to patch from — a truncate
  * names no row — so the window is re-read now, in the lane, and every subscriber is re-snapshotted
@@ -150,9 +190,10 @@ async function truncated(
   entry: QueryEntry,
   change: ChangeEvent,
 ): Promise<FanoutResult> {
-  if (!entry.shape.entities.includes(change.entity)) return { sent: 0, stale: 0 };
+  if (!entry.shape.entities.includes(change.table)) return { sent: 0, stale: 0 };
   await refillWindowInLane(entry);
   if (change.lsn > entry.lsn) entry.lsn = change.lsn;
+  deps.source.floorAt?.(entry.qid, entry.lsn, { exclusive: true });
   for (const subscription of entry.subscribers.values()) {
     subscription.socket.markDesynced(subscription.sid);
   }
@@ -160,11 +201,15 @@ async function truncated(
 }
 
 /** Each marked subscriber, re-snapshotted out of the window just read. Frames that left. */
-async function resnapshotAll(deps: FanoutDeps, entry: QueryEntry): Promise<number> {
+async function resnapshotAll(
+  deps: FanoutDeps,
+  entry: QueryEntry,
+  named: readonly NamedWrite[] = [],
+): Promise<number> {
   let sent = 0;
   for (const subscription of entry.subscribers.values()) {
     if (!subscription.socket.desynced.has(subscription.sid)) continue;
-    if (await resnapshot(deps, entry, subscription)) sent += 1;
+    if (await resnapshot(deps, entry, subscription, named)) sent += 1;
   }
   return sent;
 }
@@ -195,6 +240,7 @@ async function resnapshot(
   deps: FanoutDeps,
   entry: QueryEntry,
   subscription: LiveSubscription,
+  named: readonly NamedWrite[],
 ): Promise<boolean> {
   const who: Subscriber = { sid: subscription.sid, actor: subscription.socket.actor };
   let rows: readonly Row[];
@@ -206,31 +252,58 @@ async function resnapshot(
     return false;
   }
   const cursor = makeCursor(entry.qid, entry.lsn, rows, deps.clock.now().getTime());
-  if (!subscription.socket.send(snapshotFrame(entry, subscription.sid, rows, cursor))) return false;
+  const frame = snapshotFrame(entry, subscription.sid, rows, cursor, named);
+  if (!subscription.socket.send(frame)) return false;
   subscription.cursor = cursor;
   subscription.socket.clearDesynced(subscription.sid);
   return true;
 }
 
+/** A write this truth may hold, and the RECORD key of the row it made — what it is shown by. */
+export interface NamedWrite {
+  readonly write: string;
+  readonly key: string;
+}
+
 /**
- * The one place a snapshot frame is built, so the identity scope and the record keys cannot be
- * told to one caller only. `keys` rides only when some key differs from its row's `id` — an entity
- * keyed by `id` sends the frame it always sent.
+ * The one place a snapshot frame is built, so the identity scope, the record keys and the writes
+ * cannot be told to one caller only. `keys` rides only when some key differs from its row's `id`
+ * — an entity keyed by `id` sends the frame it always sent. A write in `named` is sent only when
+ * the row it made is among `rows`, which are already filtered for THIS subscriber: a write behind
+ * a row it may not see is another actor's, never named to it.
  */
 export function snapshotFrame(
   entry: QueryEntry,
   sid: string,
   rows: readonly Row[],
   cursor: LiveCursor,
+  named: readonly NamedWrite[] = [],
 ): Frame {
-  const base = { type: 'snapshot', v: PROTOCOL_VERSION, sid, rows, cursor } as const;
-  const scoped = entry.rowEntity === null ? base : { ...base, entity: entry.rowEntity };
   const keyOf = entry.rowKey;
-  if (keyOf === null) return scoped;
-  const keys = rows.map((row) => keyOf(row));
+  const keys = rows.map((row) => (keyOf === null ? row.id : keyOf(row)));
+  const shown = new Set(keys);
+  const writes = new Set<string>();
+  for (const { write, key } of named) if (shown.has(key)) writes.add(write);
+  const base = { type: 'snapshot', v: PROTOCOL_VERSION, sid, rows, cursor } as const;
+  // Capped where the decoder caps it: a frame over the ceiling is refused whole, and a write left
+  // unnamed only settles on its own HTTP answer.
+  const listed = [...writes].slice(0, FRAME_LIMITS.patches);
+  const withWrites = listed.length === 0 ? base : { ...base, writes: listed };
+  const scoped = entry.rowEntity === null ? withWrites : { ...withWrites, entity: entry.rowEntity };
   return keys.every((key, index) => key === rows[index]?.id) ? scoped : { ...scoped, keys };
 }
 
+/** The change's write, by the record key of the row it made; nothing for an unkeyed change. */
+function namedBy(
+  entry: QueryEntry,
+  change: ChangeEvent,
+  write: string | null,
+): readonly NamedWrite[] {
+  const whole = change.after ?? change.before;
+  if (write === null || whole === null) return [];
+  return [{ write, key: entry.rowKey === null ? whole.id : entry.rowKey(whole) }];
+}
+
 /**
  * A patch's record key, from the change's WHOLE row — an update patch carries only the changed
  * columns, and a key needs every primary-key column. Only stamped where it differs from `id`.
@@ -246,3 +319,23 @@ function keyPatches(
   const key = keyOf(whole);
   return patches.map((patch) => (key === patch.id ? patch : { ...patch, key }));
 }
+
+/** The patch the ring retains: the pre-policy patch plus the write that made its change. */
+function retain(patch: RowPatch, write: string | null): RowPatch {
+  return write === null ? patch : { ...patch, write };
+}
+
+/**
+ * The one place a patch frame is built, so a live fanout and a delta resume cannot spell it two
+ * ways. `writes` rides only when some keyed write made a patch here: an unkeyed change sends the
+ * frame it always sent.
+ */
+export function patchFrame(
+  sid: string,
+  patches: readonly RowPatch[],
+  lsn: string,
+  writes: readonly string[],
+): Frame {
+  const base = { type: 'patch', v: PROTOCOL_VERSION, sid, patches, lsn } as const;
+  return writes.length === 0 ? base : { ...base, writes };
+}

package/src/live-floor.ts

@@ -0,0 +1,35 @@
+// What a window read that just returned owes the retained ring, and the position the subscriber it
+// serves is told. The rule itself is `change-buffer.ts`'s (`Ring`); this is its one caller for a
+// read, split from `live-query.ts` at that file's size ceiling.
+
+import type { ResumeSource } from './cursor';
+import type { FilledWindow, QueryEntry } from './query-window';
+
+export interface NodePosition {
+  /** The newest change this node has received; `''` before the first. */
+  readonly lastLsn: string;
+  /** This node's own mark for a read with no position behind it. Sorts below every real lsn. */
+  readonly origin: string;
+}
+
+/**
+ * Synchronous on purpose: the caller awaits a policy pass next, and the position has to be the
+ * one these ROWS were read at — read after that await, a change folded in meanwhile would be
+ * claimed by a snapshot that does not hold it.
+ */
+export function floorAfterRead(
+  source: ResumeSource,
+  entry: QueryEntry,
+  window: FilledWindow,
+  node: NodePosition,
+): string {
+  // Served out of a window that was already filled: its patch history is still true.
+  if (window.landed === null) return window.lsn;
+  // A read with no position behind it is marked with this node's own origin, so the cursor it
+  // mints is one no other node could have issued.
+  const lsn = window.lsn === '' ? node.origin : window.lsn;
+  if (entry.lsn === '') entry.lsn = lsn;
+  if (window.landed === 'forced') source.floorAt?.(entry.qid, lsn, { exclusive: true });
+  else source.floorAt?.(entry.qid, lsn, { sole: node.lastLsn === '' || window.lsn === '' });
+  return lsn;
+}

package/src/live-query-options.ts

@@ -0,0 +1,44 @@
+// What a `LiveQueryRegistry` is built from, and the one ceiling it defaults. Split from
+// `live-query.ts` at that file's size ceiling: the registry owns behaviour, this owns its knobs.
+
+import type { Actor, Clock } from '@ultimat3/core';
+import type { ReconnectBudget, ResumeSource } from './cursor';
+import type { SubscriberGateOptions } from './subscriber-gate';
+import type { Scheduler } from './thundering-herd';
+
+export interface LiveQueryRegistryOptions extends SubscriberGateOptions {
+  readonly source: ResumeSource;
+  readonly budget?: ReconnectBudget;
+  readonly clock?: Clock;
+  readonly maxPerSocket?: number;
+  readonly maxPerTenant?: number;
+  /**
+   * Live subscriptions one actor — or one anonymous client address — may hold on this node across
+   * all its sockets. Defaults to `DEFAULT_MAX_PER_ACTOR` (1,000); the boot passes the app's
+   * `realtime.maxSubscriptionsPerActor`.
+   */
+  readonly maxPerActor?: number;
+  readonly tenantOf?: (actor: Actor | null) => string | null;
+  /**
+   * Distinct `(query, input)` pairs this node will hold at once. A `qid` derives from
+   * client-chosen input, so without a ceiling one socket mints entries — a matcher, a row window,
+   * a `WindowLock` and a fanout target each — until the process dies.
+   */
+  readonly maxEntries?: number;
+  /**
+   * How long one entry's SHARED snapshot read may hold its slot. Defaults to
+   * `DEFAULT_READ_DEADLINE_MS`. Without it a `definition.snapshot` that never settles pinned the
+   * slot for the life of the process and every later cold subscriber joined a promise nothing
+   * would resolve — one wedged read taking every future subscriber of that query id with it.
+   */
+  readonly readDeadlineMs?: number;
+  /** Injected so that deadline is provable without waiting for one. Production uses `setTimeout`. */
+  readonly schedule?: Scheduler;
+}
+
+/**
+ * Live `(query, input)` pairs one node holds. Reached, the next NEW pair is refused with
+ * `X_SUBSCRIPTION_LIMIT`; subscribing to a pair that already exists keeps working, because the
+ * cost this bounds is the entry, not the subscriber.
+ */
+export const DEFAULT_MAX_ENTRIES = 10_000;