@ultimat3/realtime 23.0.0 → 24.0.0

package/src/query-window.ts

@@ -131,9 +131,8 @@ export function createEntry(
  * had already been fanned out would rewind every later subscriber to rows the window has moved
  * past, so a stale read is discarded and its caller is served from the newer window instead.
  */
-export async function fillWindow(
-  entry: QueryEntry,
-): Promise<{ rows: readonly Row[]; lsn: string }> {
+export async function fillWindow(entry: QueryEntry): Promise<FilledWindow> {
+  let landed: FilledWindow['landed'] = null;
   for (let attempt = 0; ; attempt += 1) {
     // Read before `startRead` clears it: a second caller arriving during the read joins it and is
     // not the one that forced it, which is what keeps one forced read from becoming N.
@@ -155,15 +154,28 @@ export async function fillWindow(
       const first = entry.applied === 0;
       if (isNewestRead(entry, pending) && (forced || first || result.lsn >= entry.lsn)) {
         applyRead(entry, pending, result);
+        // A forced read outranks a first one: what it replaced was known to be wrong.
+        landed = forced || landed === 'forced' ? 'forced' : first ? 'first' : landed;
       }
       // A change reached this window while its first read was in flight and could not be folded,
       // so what just landed may predate it: read once more before serving anyone a partial window.
       return first && entry.stale && attempt < COLD_REREADS;
     });
-    if (!again) return { rows: entry.rows, lsn: entry.lsn };
+    if (!again) return { rows: entry.rows, lsn: entry.lsn, landed };
   }
 }
 
+/**
+ * The window a caller is served from, and whether THIS call put a read into it. `landed` is what
+ * decides the retained ring's floor (`change-buffer.ts`): `null` is a window that was already
+ * filled — its patch history is still true, and flooring it would evict every other subscriber's.
+ */
+export interface FilledWindow {
+  readonly rows: readonly Row[];
+  readonly lsn: string;
+  readonly landed: 'first' | 'forced' | null;
+}
+
 /**
  * How many times a cold window re-reads because writes kept landing during its read. Bounded: a
  * table written faster than it can be read would otherwise never serve a subscriber, and after the

package/src/record-persister.ts

@@ -88,10 +88,15 @@ export function recordPersister(options: RecordPersisterOptions): RecordPersiste
   });
 
   const restore = async (): Promise<number> => {
-    const scope = scopeKey(principal());
+    const owner = principal();
+    const scope = scopeKey(owner);
     if (scope === undefined) return 0;
+    const stored = await options.local.rows(scope);
+    // The read is async and a principal change can land inside it: its rescope has already cleared
+    // the store, so restoring now would hand the previous principal's rows to the next one.
+    if (principal() !== owner) return 0;
     let restored = 0;
-    for (const [type, rows] of await options.local.rows(scope)) {
+    for (const [type, rows] of stored) {
       if (!options.types.has(type)) continue;
       options.store.restore(type, rows);
       restored += Object.keys(rows).length;

package/src/replication-errors.ts

@@ -17,10 +17,14 @@ export class ReplicationProtocolError extends RealtimeError {
   constructor(args: { stage: string; detail: string; fix?: string }) {
     super({
       code: 'X_REPLICATION_PROTOCOL',
-      cause: `postgres replication ${args.stage}: ${args.detail}`,
-      fix:
-        args.fix ??
-        'x doctor db — the server must be postgres >= 14 with a pgoutput publication and wal_level=logical',
+      // The requirement rides the cause when the caller names no fix of its own: a `fix:` is one
+      // command, and `x doctor` has no `db` subcommand for the old line to have been one.
+      cause:
+        `postgres replication ${args.stage}: ${args.detail}` +
+        (args.fix === undefined
+          ? ' — the server must be postgres >= 14 with a pgoutput publication and wal_level=logical'
+          : ''),
+      fix: args.fix ?? 'x doctor --json',
     });
   }
 }

package/src/replicator-envelope.ts

@@ -0,0 +1,110 @@
+// The bus envelope, read side: one published change decoded back into a `ChangeEvent` (rows revived
+// through their entity's columns), with the producer and sequence a lost message is visible in —
+// and the detector a `sync` node runs over them. The publisher is `replicator.ts`.
+
+import { isWriteDigest } from '@ultimat3/core';
+import type { ChangeEvent } from './changefeed';
+import { reviveBusRow } from './replicator-row';
+
+/**
+ * What the bus actually carries: one change, plus who published it and where in that publisher's
+ * stream it sits.
+ *
+ * Fanout is core NATS — `publish`, no ack, at most once — so a consumer that reads only the change
+ * cannot tell "nothing happened" from "eleven changes went past while I was reconnecting". An lsn
+ * cannot answer it either: a WAL position is a byte offset, so a legitimate next change is an
+ * arbitrary jump forwards. A per-publisher counter is the one number a gap is visible in.
+ *
+ * Both fields are optional on the wire, so a node reading a publisher that predates them simply
+ * detects nothing — the same rule the `snapshot.entity` field follows, on a subject no client sees.
+ */
+export interface ChangeEnvelope {
+  readonly change: ChangeEvent;
+  /** Monotonic within `producer`, from 1. `null` from a publisher that does not sequence. */
+  readonly seq: number | null;
+  /** Identifies one replicator *run*. A new one restarts `seq` — and follows a stream that died. */
+  readonly producer: string | null;
+}
+
+/** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
+export function parseChange(payload: string): ChangeEvent | null {
+  return parseEnvelope(payload)?.change ?? null;
+}
+
+/** The same decode, keeping the two fields a gap is visible in. `parseChange` is this, narrowed. */
+export function parseEnvelope(payload: string): ChangeEnvelope | null {
+  try {
+    const parsed: unknown = JSON.parse(payload);
+    if (typeof parsed !== 'object' || parsed === null) return null;
+    const shape = parsed as Partial<ChangeEvent> & { seq?: unknown; producer?: unknown };
+    if (typeof shape.entity !== 'string' || typeof shape.lsn !== 'string') return null;
+    if (
+      shape.op !== 'insert' &&
+      shape.op !== 'update' &&
+      shape.op !== 'delete' &&
+      shape.op !== 'truncate'
+    ) {
+      return null;
+    }
+    return {
+      change: {
+        entity: shape.entity,
+        op: shape.op,
+        before: reviveBusRow(shape.entity, shape.before ?? null),
+        after: reviveBusRow(shape.entity, shape.after ?? null),
+        lsn: shape.lsn,
+        txid: typeof shape.txid === 'string' ? shape.txid : '',
+        orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
+        at: typeof shape.at === 'number' ? shape.at : 0,
+        // Only the minted shape crosses: a frame decoder refuses anything else, so a malformed
+        // label here would cost every member the whole frame rather than one page its echo.
+        ...(isWriteDigest(shape.write) ? { write: shape.write } : {}),
+        ...(isNames(shape.omitted) ? { omitted: shape.omitted } : {}),
+      },
+      seq: typeof shape.seq === 'number' && Number.isFinite(shape.seq) ? shape.seq : null,
+      producer: typeof shape.producer === 'string' ? shape.producer : null,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** A non-empty list of property names — the only shape `ChangeEvent.omitted` crosses the bus in. */
+const isNames = (value: unknown): value is readonly string[] =>
+  Array.isArray(value) && value.length > 0 && value.every((name) => typeof name === 'string');
+
+/**
+ * The consume-side twin of the publisher's counter, and the only thing on a `sync` node that can
+ * say "this node missed changes". Publish-side duplicate and out-of-order guards already existed;
+ * there was no equivalent here, so a NATS blip during a rolling restart was eleven changes that
+ * simply never happened as far as every subscriber on this node could tell.
+ *
+ * Per producer, because a replicator restart legitimately rewinds the counter. A repeat or a
+ * reordering is *not* reported as a gap — the window's own lsn guard refuses those — and neither is
+ * the first message this node ever reads: a node that joined late has missed everything by
+ * definition, and every subscription it holds started after it did.
+ *
+ * A NEW producer after a known one IS a gap. The run before it died — that is why there is a new
+ * one — and nothing says its last messages reached this node: the tail of a dead stream has no
+ * later sequence number to be missed against. One re-read per replicator restart is the price.
+ */
+export class SeqGapDetector {
+  readonly #next = new Map<string, number>();
+
+  /** `true` when at least one message between the last one and this one was never delivered. */
+  observe(envelope: ChangeEnvelope): boolean {
+    const { producer, seq } = envelope;
+    if (producer === null || seq === null) return false;
+    const expected = this.#next.get(producer);
+    const succeeds = expected === undefined && this.#next.size > 0;
+    // Never backwards: a redelivery must not lower the bar and turn the next legitimate message
+    // into a gap of its own.
+    this.#next.set(producer, Math.max(expected ?? 0, seq + 1));
+    return succeeds || (expected !== undefined && seq > expected);
+  }
+
+  /** Producers this node has read. Bounded by replicator restarts, so it is swept on a drain. */
+  forget(): void {
+    this.#next.clear();
+  }
+}

package/src/replicator-row.ts

@@ -0,0 +1,75 @@
+// Single responsibility: a change's rows across the replicator bus. The bus carries text, a row
+// holds a `Date` and a `Uint8Array`, and `JSON.parse` returns neither — so the publisher writes a
+// form JSON can carry and the sync node reads it back through the ENTITY's own columns.
+
+import { entityForTable } from '@ultimat3/entity';
+import type { Row } from './json';
+
+const CHUNK = 0x8000;
+
+/** Base64 of raw bytes. Chunked: one `String.fromCharCode(...bytes)` overflows the stack at ~100k. */
+function toBase64(bytes: Uint8Array): string {
+  let binary = '';
+  for (let at = 0; at < bytes.length; at += CHUNK) {
+    binary += String.fromCharCode(...bytes.subarray(at, at + CHUNK));
+  }
+  return btoa(binary);
+}
+
+function fromBase64(text: string): Uint8Array {
+  const binary = atob(text);
+  const bytes = new Uint8Array(binary.length);
+  for (let at = 0; at < binary.length; at += 1) bytes[at] = binary.charCodeAt(at);
+  return bytes;
+}
+
+/**
+ * Publish side. A `Date` already serialises to the ISO text its column reads back; a `Uint8Array`
+ * serialises to `{"0":1,"1":2}`, which nothing reads back, so it crosses as base64. Only a
+ * top-level value can be one: `arrayOf()` refuses `bytes()` at declaration.
+ */
+export function busRow(row: Row | null): Row | null {
+  if (row === null) return null;
+  let out: Record<string, unknown> | undefined;
+  for (const [property, value] of Object.entries(row as Record<string, unknown>)) {
+    if (!(value instanceof Uint8Array)) continue;
+    out ??= { ...row };
+    out[property] = toBase64(value);
+  }
+  return out === undefined ? row : (out as unknown as Row);
+}
+
+/**
+ * Consume side: the row as the replicator held it before it was text. Without this every `Date`
+ * reached the matcher as an ISO string, `compareValues(Date, string)` fell to comparing their
+ * printed forms, and one edit to one column moved its row to the top of every window ordered by
+ * a timestamp — on every deployment with a bus, and in no in-process test, which hands the object
+ * over directly.
+ *
+ * Decided by the column that DECLARED the value, never by what the value looks like: a `text()`
+ * holding `2026-08-09T12:00:00.000Z` stays text. `table` is `ChangeEvent.entity`, which on the WAL
+ * path is the relation name. A table with no entity here, a property with no column, and a value
+ * its column refuses are all handed back as they arrived — a change that crosses unrevived is the
+ * old behaviour; a change dropped is a window that silently diverges.
+ */
+export function reviveBusRow(table: string, row: Row | null): Row | null {
+  if (row === null) return null;
+  const entity = entityForTable(table);
+  if (entity === undefined) return row;
+  const out: Record<string, unknown> = {};
+  for (const [property, value] of Object.entries(row as Record<string, unknown>)) {
+    out[property] = value;
+    if (value === null || !Object.hasOwn(entity.$columns, property)) continue;
+    const column = entity.$columns[property];
+    if (column === undefined || column.$meta.sealed !== undefined) continue;
+    try {
+      out[property] =
+        column.$meta.kind === 'bytea' && typeof value === 'string'
+          ? fromBase64(value)
+          : column.$parse(value);
+    } catch {
+      // Kept as it arrived — see the function comment.
+    }
+  }
+  return out as unknown as Row;
+}