@ultimat3/realtime 1.2.0 → 3.0.0

package/src/query-window.ts

@@ -0,0 +1,187 @@
+// The shared, pre-policy result window one query id is served from: how it is built, how it is
+// read once for N subscribers, and how one that is known to be wrong is replaced. The authz
+// decision is never here — `live-query.ts` owns that, once per subscriber, over what this returns.
+
+import type { JsonValue, Row } from './json';
+import type { LiveQueryDefinition, LiveSubscription, SnapshotResult } from './live-contract';
+import type { IncrementalMatcher, SubscriptionShape } from './matcher-bridge';
+import { WindowLock } from './window-lock';
+
+export interface QueryEntry {
+  readonly qid: string;
+  readonly definition: LiveQueryDefinition;
+  readonly input: JsonValue;
+  /** Told to the client on every snapshot: the identity scope its rows belong under. */
+  readonly rowEntity: string | null;
+  readonly shape: SubscriptionShape;
+  readonly matcher: IncrementalMatcher;
+  readonly subscribers: Map<string, LiveSubscription>;
+  /**
+   * The shared, *pre-policy* result window. One per query id, bounded by the query's `limit`, and
+   * the reason the matcher can run once for N subscribers: the read is shared, the authz is not.
+   */
+  rows: readonly Row[];
+  lsn: string;
+  /**
+   * This window is known to have missed at least one change, so patching it would compound the
+   * error. Set when the change stream skipped a sequence and when a matcher reports it lost the
+   * window's tail; cleared by the read that replaces the rows. It is *not* a subscriber's desync —
+   * that is per socket, and this is the window every one of them shares.
+   */
+  stale: boolean;
+  /** Serial lane over `rows`/`lsn`. Every fanout and every window assignment takes its turn here. */
+  readonly lock: WindowLock;
+  /** The read in flight, shared by every subscriber that arrives during it. `null` between reads. */
+  reading: PendingRead | null;
+  /**
+   * Reads issued against this entry, ever. It is the ORDER of two reads, which nothing else here
+   * can answer: a definition with no lsn provider returns `''` from every snapshot.
+   */
+  generation: number;
+  /** The generation of the newest read whose rows are in `rows`. `0` before the first one lands. */
+  applied: number;
+}
+
+/**
+ * One read and which read it is. They are one fact — a joiner needs the promise AND the generation
+ * it will have to compare against when it lands — and two fields on the entry is two writes a later
+ * edit can separate.
+ */
+export interface PendingRead {
+  readonly generation: number;
+  readonly result: Promise<SnapshotResult>;
+}
+
+export function createEntry(
+  qid: string,
+  definition: LiveQueryDefinition,
+  input: JsonValue,
+  matcher: IncrementalMatcher,
+): QueryEntry {
+  return {
+    qid,
+    definition,
+    input,
+    // Resolved with the matcher, from the same build: `prepare` has already run, so a definition
+    // that compiles its shape per input can answer.
+    rowEntity: definition.rowEntity?.(input) ?? null,
+    shape: {
+      qid,
+      // The matcher knows the dependency set this *input* produced; `definition.entities` is the
+      // static declaration and can only be a superset of it. Preferring the matcher is what lets a
+      // definition built from a real query carry no static list at all.
+      entities: matcher.entities.length > 0 ? matcher.entities : definition.entities,
+      orgId: orgIdOf(input),
+      ...(definition.columns ? { columns: definition.columns } : {}),
+    },
+    matcher,
+    subscribers: new Map(),
+    rows: [],
+    lsn: '',
+    stale: false,
+    lock: new WindowLock(),
+    reading: null,
+    generation: 0,
+    applied: 0,
+  };
+}
+
+/**
+ * The window this subscriber is served from, read once per entry. A subscriber arriving while
+ * another's read is in flight joins that read rather than issuing its own — N cold subscribers on
+ * one query id being N reads is the shared window not existing.
+ *
+ * The result lands in the lane, and never backwards. A snapshot that resolved after a newer change
+ * had already been fanned out would rewind every later subscriber to rows the window has moved
+ * past, so a stale read is discarded and its caller is served from the newer window instead.
+ */
+export async function fillWindow(
+  entry: QueryEntry,
+): Promise<{ rows: readonly Row[]; lsn: string }> {
+  // Read before `startRead` clears it: a second caller arriving during the read joins it and is
+  // not the one that forced it, which is what keeps one forced read from becoming N.
+  const forced = entry.stale;
+  const pending = forced || entry.reading === null ? startRead(entry) : entry.reading;
+  const result = await pending.result;
+  return await entry.lock.run(async () => {
+    // Two rules, and neither can stand in for the other. Against another READ it is identity —
+    // the same check `startRead` makes on `entry.reading` one function down, and the one
+    // `packages/cache/src/single-flight.ts` makes for the same reason — because an lsn cannot
+    // order two reads at all: a definition with no lsn provider answers `''` for both, and
+    // `'' >= ''` let the older one overwrite the gap repair the newer one had just landed, with
+    // `stale` already cleared by its issue and therefore nothing left to re-read. Against a
+    // CHANGE it is still the lsn, because a fanout moved `entry.lsn` forwards while this read was
+    // in flight and rewinding to what the read saw hands that subscriber rows the fanout has
+    // moved past — except for a forced read, which was issued *because* what is under it is
+    // wrong.
+    if (isNewestRead(entry, pending) && (forced || result.lsn >= entry.lsn)) {
+      applyRead(entry, pending, result);
+    }
+    return { rows: entry.rows, lsn: entry.lsn };
+  });
+}
+
+/**
+ * The same replacement, for a caller that is already holding the lane. A fanout cannot call
+ * `fillWindow` — that takes the entry's own lane, and a lane is not reentrant — so the one path
+ * that repairs a stale window mid-fanout is spelled here rather than deadlocking on the other.
+ */
+export async function refillWindowInLane(entry: QueryEntry): Promise<void> {
+  const pending = startRead(entry);
+  const result = await pending.result;
+  // Same identity rule as `fillWindow`: a read issued before this one may still be in flight, and
+  // whichever was issued LAST is the one the window keeps.
+  if (isNewestRead(entry, pending)) applyRead(entry, pending, result);
+}
+
+/** Is this the newest read to have landed? An older one's rows are behind the window, not on it. */
+function isNewestRead(entry: QueryEntry, pending: PendingRead): boolean {
+  return pending.generation > entry.applied;
+}
+
+function applyRead(entry: QueryEntry, pending: PendingRead, result: SnapshotResult): void {
+  entry.applied = pending.generation;
+  entry.rows = result.rows;
+  if (result.lsn > entry.lsn) entry.lsn = result.lsn;
+}
+
+/** Publishes the in-flight read, and clears it as it settles — the share is per read, not a cache. */
+function startRead(entry: QueryEntry): PendingRead {
+  // Cleared here rather than when the read lands: the read about to be issued is the one that
+  // answers the staleness, so a second caller must join it instead of forcing another.
+  entry.stale = false;
+  entry.generation += 1;
+  const reading: PendingRead = { generation: entry.generation, result: readSnapshot(entry) };
+  entry.reading = reading;
+  const done = (): void => {
+    if (entry.reading === reading) entry.reading = null;
+  };
+  void reading.result.then(done, done);
+  return reading;
+}
+
+/**
+ * The definition's read, with the staleness put back when it does not answer.
+ *
+ * Clearing the mark on the way in and never restoring it was the gap repair happening once and
+ * never again: the snapshot that was going to replace an invalidated window rejects — the pool is
+ * exhausted by the same incident that caused the gap — and the entry is left unmarked over rows it
+ * is known to have missed a change on. Nothing re-reads, `#resnapshot` serves every desynced
+ * subscriber out of that divergent window and clears their marks, and the divergence `stale` exists
+ * to prevent is now permanent and silent. `async` so a definition that throws synchronously takes
+ * the same path as one that rejects.
+ */
+async function readSnapshot(entry: QueryEntry): Promise<SnapshotResult> {
+  try {
+    return await entry.definition.snapshot({ input: entry.input });
+  } catch (error) {
+    entry.stale = true;
+    throw error;
+  }
+}
+
+export function orgIdOf(input: JsonValue): string | null {
+  if (typeof input !== 'object' || input === null || Array.isArray(input)) return null;
+  const value = input['orgId'];
+  return typeof value === 'string' ? value : null;
+}

package/src/rebase.ts

@@ -108,26 +108,86 @@ export function reconcile<T extends TableMap = TableMap>(
   const table = store.table(ack.entity);
   const local = table.get(ack.id);
 
-  // Everything at or after the acked sequence is optimistic and must be undone newest-first.
-  const affected = log.pending().filter((candidate) => candidate.seq >= (entry?.seq ?? 0));
+  const { affected, rolledBack } = undoFrom(store, log, entry?.seq ?? 0);
+
+  const base = store.table(ack.entity).get(ack.id);
+  const winner = land(store, ack, strategy, { local, base }, options);
+  log.drop(ack.key);
+
+  return {
+    strategy: strategyName(strategy),
+    rolledBack,
+    reapplied: replayExcept(store, affected, ack.key),
+    winner,
+  };
+}
+
+/**
+ * Everything at or after `from` is optimistic, so it is undone newest-first — and `reconcile` and
+ * `rollbackMutation` are one rule with different middles, not two. Spelled twice, the next change
+ * to the replay order has to be made twice, and the half that is missed diverges silently.
+ */
+function undoFrom<T extends TableMap>(
+  store: LocalStore<T>,
+  log: RebaseLog<T>,
+  from: number,
+): { affected: readonly RebaseEntry<T>[]; rolledBack: string[] } {
+  const affected = log.pending().filter((candidate) => candidate.seq >= from);
   const rolledBack: string[] = [];
   for (const candidate of [...affected].reverse()) {
     store.rollback(candidate.key);
     rolledBack.push(candidate.key);
   }
+  return { affected, rolledBack };
+}
 
-  const base = store.table(ack.entity).get(ack.id);
-  const winner = land(store, ack, strategy, { local, base }, options);
-  log.drop(ack.key);
-
+/**
+ * The other half: replay in sequence order, skipping the one the server has now settled. `local` is
+ * pure, which is what makes replaying it deterministic and therefore safe to do at all.
+ */
+function replayExcept<T extends TableMap>(
+  store: LocalStore<T>,
+  affected: readonly RebaseEntry<T>[],
+  settled: string,
+): string[] {
   const reapplied: string[] = [];
   for (const candidate of affected) {
-    if (candidate.key === ack.key) continue;
+    if (candidate.key === settled) continue;
     store.apply(candidate.key, (tx) => candidate.apply(tx));
     reapplied.push(candidate.key);
   }
+  return reapplied;
+}
+
+export interface RollbackResult {
+  readonly rolledBack: readonly string[];
+  readonly reapplied: readonly string[];
+}
+
+/**
+ * The other half of `reconcile`: the server **refused** a mutation, so there is no server truth to
+ * land — only an optimistic write to take back. Same shape as a reconcile, and for the same reason:
+ * the writes made after it may depend on it, so everything from its sequence onward is undone
+ * newest-first and then replayed without it. Replay is deterministic because `local` is pure.
+ *
+ * Idempotent for a key the log does not hold: a denial can arrive twice, and tier 2 records nothing
+ * to undo in the first place.
+ */
+export function rollbackMutation<T extends TableMap = TableMap>(args: {
+  store: LocalStore<T>;
+  log: RebaseLog<T>;
+  key: string;
+}): RollbackResult {
+  const { store, log, key } = args;
+  const entry = log.get(key);
+  if (!entry) return { rolledBack: [], reapplied: [] };
 
-  return { strategy: strategyName(strategy), rolledBack, reapplied, winner };
+  const { affected, rolledBack } = undoFrom(store, log, entry.seq);
+  // The one thing that differs from `reconcile`'s middle: there is no server truth to land. Dropped
+  // and never retried — a denial is a decision about this intent, so replaying it on the next
+  // reconcile would put the write the server refused back on the screen.
+  log.drop(key);
+  return { rolledBack, reapplied: replayExcept(store, affected, key) };
 }
 
 function land<T extends TableMap>(

package/src/replicator.ts

@@ -11,7 +11,7 @@
 // its feed and reports `/readyz` false; it retries with jittered backoff and takes over the moment
 // the holder dies. Scaling the replicator is therefore always vertical, and that is by design.
 
-import { logger, withSpan } from '@ultimat3/core';
+import { logger, uuid, withSpan } from '@ultimat3/core';
 import type { ChangeEvent, ChangeFeed } from './changefeed';
 import type { Transport } from './fanout';
 import { type BackoffPolicy, backoffDelay, defaultBackoff, type Rng } from './thundering-herd';
@@ -72,6 +72,26 @@ export interface ReplicatorStats {
   readonly outOfOrder: number;
 }
 
+/**
+ * What the bus actually carries: one change, plus who published it and where in that publisher's
+ * stream it sits.
+ *
+ * Fanout is core NATS — `publish`, no ack, at most once — so a consumer that reads only the change
+ * cannot tell "nothing happened" from "eleven changes went past while I was reconnecting". An lsn
+ * cannot answer it either: a WAL position is a byte offset, so a legitimate next change is an
+ * arbitrary jump forwards. A per-publisher counter is the one number a gap is visible in.
+ *
+ * Both fields are optional on the wire, so a node reading a publisher that predates them simply
+ * detects nothing — the same rule the `snapshot.entity` field follows, on a subject no client sees.
+ */
+export interface ChangeEnvelope {
+  readonly change: ChangeEvent;
+  /** Monotonic within `producer`, from 1. `null` from a publisher that does not sequence. */
+  readonly seq: number | null;
+  /** Identifies one replicator *run*. A new one restarts `seq`, and that is not a gap. */
+  readonly producer: string | null;
+}
+
 export interface Replicator {
   /** `false` = another replicator holds the lock; this process must stay `/readyz` false. */
   start(): Promise<boolean>;
@@ -91,6 +111,11 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
   let published = 0;
   let skipped = 0;
   let outOfOrder = 0;
+  // One id per run, not per process: a replicator that took the lock back after a crash publishes
+  // from its persisted lsn, and a consumer must read that as a new stream rather than as a gap in
+  // the old one.
+  let producer = uuid();
+  let seq = 0;
 
   const onChange = async (raw: ChangeEvent): Promise<void> => {
     const change = normalize(raw);
@@ -104,7 +129,13 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
       return;
     }
     await withSpan('realtime.replicate', async () => {
-      await options.transport.publish(subjectOf(change), JSON.stringify(change));
+      seq += 1;
+      // The envelope is the change plus two fields, flat, so a consumer that only knows about the
+      // change reads it unchanged — `parseChange` still answers on the same payload.
+      await options.transport.publish(
+        subjectOf(change),
+        JSON.stringify({ ...change, seq, producer }),
+      );
       lastLsn = change.lsn;
       published += 1;
     });
@@ -118,6 +149,8 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
         return false;
       }
       running = true;
+      producer = uuid();
+      seq = 0;
       await options.feed.start(
         options.from === undefined ? { onChange } : { from: options.from, onChange },
       );
@@ -163,23 +196,63 @@ export function normalize(change: ChangeEvent): ChangeEvent | null {
 
 /** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
 export function parseChange(payload: string): ChangeEvent | null {
+  return parseEnvelope(payload)?.change ?? null;
+}
+
+/** The same decode, keeping the two fields a gap is visible in. `parseChange` is this, narrowed. */
+export function parseEnvelope(payload: string): ChangeEnvelope | null {
   try {
     const parsed: unknown = JSON.parse(payload);
     if (typeof parsed !== 'object' || parsed === null) return null;
-    const shape = parsed as Partial<ChangeEvent>;
+    const shape = parsed as Partial<ChangeEvent> & { seq?: unknown; producer?: unknown };
     if (typeof shape.entity !== 'string' || typeof shape.lsn !== 'string') return null;
     if (shape.op !== 'insert' && shape.op !== 'update' && shape.op !== 'delete') return null;
     return {
-      entity: shape.entity,
-      op: shape.op,
-      before: shape.before ?? null,
-      after: shape.after ?? null,
-      lsn: shape.lsn,
-      txid: typeof shape.txid === 'string' ? shape.txid : '',
-      orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
-      at: typeof shape.at === 'number' ? shape.at : 0,
+      change: {
+        entity: shape.entity,
+        op: shape.op,
+        before: shape.before ?? null,
+        after: shape.after ?? null,
+        lsn: shape.lsn,
+        txid: typeof shape.txid === 'string' ? shape.txid : '',
+        orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
+        at: typeof shape.at === 'number' ? shape.at : 0,
+      },
+      seq: typeof shape.seq === 'number' && Number.isFinite(shape.seq) ? shape.seq : null,
+      producer: typeof shape.producer === 'string' ? shape.producer : null,
     };
   } catch {
     return null;
   }
 }
+
+/**
+ * The consume-side twin of the publisher's counter, and the only thing on a `sync` node that can
+ * say "this node missed changes". Publish-side duplicate and out-of-order guards already existed;
+ * there was no equivalent here, so a NATS blip during a rolling restart was eleven changes that
+ * simply never happened as far as every subscriber on this node could tell.
+ *
+ * Per producer, because a replicator restart legitimately rewinds the counter. A repeat or a
+ * reordering is *not* reported as a gap — the window's own lsn guard refuses those — and neither is
+ * the first message of a stream: a node that joined late has missed everything by definition, and
+ * every subscription it holds started after it did.
+ */
+export class SeqGapDetector {
+  readonly #next = new Map<string, number>();
+
+  /** `true` when at least one message between the last one and this one was never delivered. */
+  observe(envelope: ChangeEnvelope): boolean {
+    const { producer, seq } = envelope;
+    if (producer === null || seq === null) return false;
+    const expected = this.#next.get(producer);
+    // Never backwards: a redelivery must not lower the bar and turn the next legitimate message
+    // into a gap of its own.
+    this.#next.set(producer, Math.max(expected ?? 0, seq + 1));
+    return expected !== undefined && seq > expected;
+  }
+
+  /** Producers this node has read. Bounded by replicator restarts, so it is swept on a drain. */
+  forget(): void {
+    this.#next.clear();
+  }
+}