@ultimat3/realtime 23.0.0 → 25.0.0

package/src/live-resume.ts

@@ -11,11 +11,12 @@ import {
   type ResumeSource,
   resumeFrom,
 } from './cursor';
+import type { RowPatch } from './json';
 import type { SnapshotResult } from './live-contract';
-import { snapshotFrame } from './live-fanout';
+import { type NamedWrite, patchFrame, snapshotFrame } from './live-fanout';
 import { fillWindow, type QueryEntry } from './query-window';
 import type { Subscriber, SubscriberGate } from './subscriber-gate';
-import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
+import type { Frame } from './sync-protocol';
 
 export interface ResumeOntoDeps {
   readonly source: ResumeSource;
@@ -24,6 +25,8 @@ export interface ResumeOntoDeps {
   readonly gate: SubscriberGate;
   /** The window, read once for the entry and filtered for THIS subscriber. */
   read(): Promise<SnapshotResult>;
+  /** Called before the delta path fills an unread entry — a database read the caller pays for. */
+  beforeFill?(): Promise<void>;
 }
 
 /**
@@ -45,16 +48,23 @@ export async function resumeOnto(
     snapshot: () => deps.read(),
   });
   if (resumed.kind !== 'delta') {
+    // Read after every retained patch, so it holds their writes; `snapshotFrame` names only those
+    // behind a row this subscriber was given. Out of the ring's window there is nothing to name,
+    // and a write left unnamed settles on its own HTTP answer.
+    const named = retainedWrites(deps.source.since(entry.qid, cursor.lsn) ?? []);
     return {
       cursor: resumed.cursor,
-      frame: snapshotFrame(entry, who.sid, resumed.rows, resumed.cursor),
+      frame: snapshotFrame(entry, who.sid, resumed.rows, resumed.cursor, named),
     };
   }
   // The gate decides about whole rows out of the shared window, and an entry nothing has read
   // yet has none — every patch would meet an empty window and be withheld. Filling is
   // conditional on purpose: a restart storm resumes onto entries that already hold a live
   // window, and re-reading per resuming subscriber is the cost a delta resume exists to skip.
-  if (entry.lsn === '') await fillWindow(entry);
+  if (entry.lsn === '') {
+    await deps.beforeFill?.();
+    await fillWindow(entry);
+  }
   // The live entry on purpose: a resume runs outside the lane, so the window under it may
   // have moved on — always forwards, and a row whose grant was revoked in the meantime is
   // one this pass must refuse rather than replay from the state it had at the cursor's lsn.
@@ -67,8 +77,34 @@ export async function resumeOnto(
   // another tenant's row id and the instant it went. The leak that branch closes, re-opened
   // one layer up. `live-fanout.ts` advances over `allowed` for exactly this reason.
   const seated = advance(cursor, patches, resumed.cursor.lsn, now);
-  return {
-    cursor: seated,
-    frame: { type: 'patch', v: PROTOCOL_VERSION, sid: who.sid, patches, lsn: seated.lsn },
-  };
+  const { bare, writes } = namedWrites(patches);
+  return { cursor: seated, frame: patchFrame(who.sid, bare, seated.lsn, writes) };
+}
+
+/**
+ * The writes behind the patches this subscriber is sent, each named once, and the patches as the
+ * wire spells them. Read off the FILTERED list: a write behind a row the gate withheld is another
+ * actor's, and naming it here would tell this page that write happened.
+ */
+function namedWrites(patches: readonly RowPatch[]): {
+  readonly bare: readonly RowPatch[];
+  readonly writes: readonly string[];
+} {
+  const writes = new Set<string>();
+  const bare = patches.map((patch) => {
+    if (patch.write === undefined) return patch;
+    writes.add(patch.write);
+    const { write: _named, ...rest } = patch;
+    return rest;
+  });
+  return { bare, writes: [...writes] };
+}
+
+/** Each retained write, by the record key of the row its patch made. */
+function retainedWrites(patches: readonly RowPatch[]): readonly NamedWrite[] {
+  const named: NamedWrite[] = [];
+  for (const patch of patches) {
+    if (patch.write !== undefined) named.push({ write: patch.write, key: patch.key ?? patch.id });
+  }
+  return named;
 }

package/src/live-rows.ts

@@ -70,6 +70,7 @@ export class RowWindows {
     type: string | null,
     rows: readonly Row[],
     keys?: readonly string[],
+    writes: readonly string[] = [],
   ): void {
     const next = type ?? registration.type;
     // The server's record key where it sent one; a row's `id` IS its key everywhere else.
@@ -80,12 +81,27 @@ export class RowWindows {
       keyed.map(([id]) => id),
       () => {
         for (const [id, row] of keyed) this.#store.merge(next, id, row);
+        // Settled in this batch, as `patch` settles: truth that holds the write, never under it.
+        settle(
+          this.#store,
+          writes,
+          keyed.map(([id]) => recordKey(next, id)),
+        );
       },
     );
   }
 
-  /** A patch list: values merged into the store, membership and order folded over the ids. */
-  patch(registration: Registration, sent: readonly RowPatch[]): void {
+  /**
+   * A patch list: values merged into the store, membership and order folded over the ids. `writes`
+   * are the writes the frame names (`PatchFrame.writes`): each this page still holds is settled
+   * against the rows merged here, INSIDE the same batch — so no holder ever renders the server's
+   * row with that write's twin replayed over it (truth plus overlay: one like counted twice).
+   */
+  patch(
+    registration: Registration,
+    sent: readonly RowPatch[],
+    writes: readonly string[] = [],
+  ): void {
     const type = registration.type;
     // A window holds RECORD keys: a patch the server keyed is folded under its key, not its id.
     const patches = sent.map((patch) =>
@@ -93,10 +109,16 @@ export class RowWindows {
     );
     // A `delete` is this window losing the row, never the store losing it: another holder keeps it.
     this.#reseat(registration, type, orderAfterPatches(registration.ids, patches), (held) => {
+      // Only a row merged here is truth this frame carried. A `delete` is membership, not the row:
+      // a write it names keeps its overlay on that row until the server's next word on it.
+      const carried = new Set<RecordKey>();
       for (const patch of patches) {
         if (patch.op === 'delete' || patch.row === null) continue;
-        if (held.has(patch.id)) this.#store.merge(type, patch.id, patch.row);
+        if (!held.has(patch.id)) continue;
+        this.#store.merge(type, patch.id, patch.row);
+        carried.add(recordKey(type, patch.id));
       }
+      settle(this.#store, writes, carried);
     });
   }
 
@@ -144,3 +166,10 @@ function holds(registration: Registration, changed: ReadonlySet<RecordKey>): boo
   }
   return false;
 }
+
+/** Each named write this page still holds, settled against the rows the frame merged. */
+function settle(store: RecordStore, writes: readonly string[], merged: Iterable<RecordKey>): void {
+  if (writes.length === 0) return;
+  const carried = new Set(merged);
+  for (const write of writes) store.settleWrite(write, carried);
+}

package/src/live-spend.ts

@@ -0,0 +1,50 @@
+// When a live subscribe spends the read's declared `rateLimit:`, and against whom. Split from
+// `live-query.ts` at its size ceiling: the registry decides WHAT is served, this decides what it
+// costs — once per subscribe that reads, on the node's clock, keyed by the socket's own address.
+
+import type { Clock } from '@ultimat3/core';
+import type { LiveCursor } from './cursor';
+import type { JsonValue } from './json';
+import type { LiveQueryDefinition } from './live-contract';
+import type { SyncSocket } from './socket';
+
+export interface SubscribeArgs {
+  readonly socket: SyncSocket;
+  readonly name: string;
+  readonly input: JsonValue;
+  readonly sid?: string;
+  readonly cursor?: LiveCursor | null;
+}
+
+/**
+ * One subscribe's bill, shared by every pass of it. `subscribe` serves AGAIN when the socket's
+ * actor is replaced under it, and that second pass is the same subscription: spending there
+ * charged one subscribe twice.
+ */
+export interface Charge {
+  due: boolean;
+}
+
+/**
+ * Spend once, then never again for this subscribe — called by the registry immediately before
+ * any DATABASE READ this subscriber causes: a cold snapshot, a resume that fell back to one, or a
+ * delta resume onto an entry nothing has read yet. Only a real in-window delta served out of the
+ * ring is free. Never decided from the cursor: its `qid` is computable from the query, the input
+ * and the org, so a cursor is a client's claim to a window, not proof it held one — trusting it
+ * made every subscribe+unsubscribe cycle a free read. `due` drops before the await, so a pass
+ * racing this one (the re-auth retry) cannot pay as well.
+ */
+export async function spendOnce(
+  definition: LiveQueryDefinition,
+  socket: SyncSocket,
+  charge: Charge,
+  clock: Clock,
+): Promise<void> {
+  if (!charge.due) return;
+  charge.due = false;
+  await definition.spend?.({
+    actor: socket.actor,
+    clientAddress: socket.clientAddress,
+    nowMs: clock.now().getTime(),
+  });
+}

package/src/local-store-idb.ts

@@ -115,6 +115,11 @@ export class MemoryLocalStore implements LocalStore {
   }
 }
 
+/** The one way to build the in-process store (tests, a browser with no IndexedDB); the class is a type in the barrel only (`X_FACTORY_NAME_SPELLING`). */
+export function memoryLocalStore(): MemoryLocalStore {
+  return new MemoryLocalStore();
+}
+
 class IdbLocalStore implements LocalStore {
   readonly kind = 'indexeddb';
   constructor(private readonly db: IdbDatabaseLike) {}

package/src/matcher-bridge.ts

@@ -6,7 +6,13 @@
 
 import { type LiveQuery, match, type Patch } from '@ultimat3/query';
 import type { ChangeEvent } from './changefeed';
-import { changedColumns, isJsonObject, type JsonObject, type Row, type RowPatch } from './json';
+import {
+  changedColumns,
+  isParsedJsonObject,
+  type JsonObject,
+  type Row,
+  type RowPatch,
+} from './json';
 
 export interface SubscriptionShape {
   readonly qid: string;
@@ -37,7 +43,7 @@ export interface IncrementalMatcher {
 
 /** The cheap pre-filter. Pure, and the only thing that runs for a change nobody subscribed to. */
 export function canAffect(shape: SubscriptionShape, change: ChangeEvent): boolean {
-  if (!shape.entities.includes(change.entity)) return false;
+  if (!shape.entities.includes(change.table)) return false;
   if (shape.orgId !== null && change.orgId !== null && shape.orgId !== change.orgId) return false;
   if (shape.columns && change.op === 'update' && change.after !== null) {
     const touched = Object.keys(changedColumns(change.before, change.after));
@@ -89,7 +95,7 @@ export function matcherFor(live: LiveQuery, projection?: () => Projection): Incr
       const row = change.after ?? change.before;
       if (!row) return NO_CHANGE;
       const patches = match<Row>(live.name, live.shape, rows, {
-        entity: change.entity,
+        entity: change.table,
         op: change.op,
         row,
         ...(change.before === null ? {} : { before: change.before }),
@@ -200,7 +206,7 @@ export function applyToWindow(rows: readonly Row[], patches: readonly RowPatch[]
 export function normalizePatch(candidate: unknown, change: ChangeEvent): RowPatch | null {
   if (candidate === null || candidate === undefined || candidate === false) return null;
   if (candidate === true) return patchFromChange(change);
-  if (!isJsonObject(candidate)) return null;
+  if (!isParsedJsonObject(candidate)) return null;
   const op = candidate['op'];
   if (op !== 'insert' && op !== 'update' && op !== 'delete') return patchFromChange(change);
   const id = candidate['id'];
@@ -209,7 +215,7 @@ export function normalizePatch(candidate: unknown, change: ChangeEvent): RowPatc
   const base: RowPatch = {
     op,
     id: typeof id === 'string' ? id : (change.after?.id ?? change.before?.id ?? ''),
-    row: op === 'delete' ? null : isJsonObject(rowValue) ? rowValue : (change.after ?? null),
+    row: op === 'delete' ? null : isParsedJsonObject(rowValue) ? rowValue : (change.after ?? null),
     lsn: typeof candidate['lsn'] === 'string' ? candidate['lsn'] : change.lsn,
   };
   if (base.id === '') return null;

package/src/nats-client.ts

@@ -33,8 +33,8 @@ export interface NatsRequestManyOptions {
 
 /**
  * What the bus needs from a NATS connection, and nothing more. A subscription outlives a reconnect —
- * the client re-establishes it underneath the caller — which is why nothing above this port keeps
- * subscription bookkeeping of its own.
+ * the client re-establishes it underneath the caller. It does not outlive the client: once the
+ * reconnect budget is spent (`onClosed`), the caller dials a new client and binds again.
  */
 export interface NatsClient {
   /** The connected server's version, as `assertServerVersion` reads it. */
@@ -75,14 +75,38 @@ export interface NatsClientOptions {
   readonly onError?: ((error: unknown) => void) | undefined;
   /** The library re-established the connection. The cluster behind it may be a different one. */
   readonly onReconnect?: (() => void) | undefined;
+  /**
+   * The library gave up: its reconnect budget is spent and this connection is closed for good,
+   * subscriptions included. Never called for the caller's own `close()`. The caller re-dials.
+   */
+  readonly onClosed?: (() => void) | undefined;
 }
 
 /** The one seam a test replaces: a client that never touches a socket. */
 export type NatsConnect = (options: NatsClientOptions) => Promise<NatsClient>;
 
 export const DEFAULT_NATS_PORT = 4222;
+
+const NATS_URL_FIX = 'export NATS_URL=nats://user:password@host:4222';
 export const DEFAULT_REQUEST_TIMEOUT_MS = 5_000;
 
+/**
+ * One URL credential, percent-decoded. `decodeURIComponent` throws a bare `URIError` on a stray
+ * `%` (`50%off`), so a secret pasted unencoded failed boot with no code and no fix. Refused under
+ * the part's NAME and the host — never the value: it is the credential, and a cause reaches a log.
+ */
+function decodeCredential(raw: string, part: 'user' | 'password', host: string): string {
+  try {
+    return decodeURIComponent(raw);
+  } catch {
+    throw new TransportUnavailableError({
+      transport: 'nats',
+      reason: `the connection URL for ${host} has a malformed percent-escape in its ${part}`,
+      fix: `bun -e 'console.log(encodeURIComponent(process.argv[1]))' '<the raw value>'   # then put the result in the ${part} of NATS_URL`,
+    });
+  }
+}
+
 /**
  * `nats://user:pass@host:4222`. The one place a bus URL is read — the library takes a bare
  * `host:port` plus credentials as options, and never looks at a URL's userinfo.
@@ -92,9 +116,12 @@ export function parseNatsUrl(url: string): NatsTarget {
   try {
     parsed = new URL(url);
   } catch {
+    // The variable, never its value: a bus URL carries the credential, and an error is the one
+    // thing that reaches a log, `--json`, an agent transcript and a ticket. `parsePgUrl`'s rule.
     throw new TransportUnavailableError({
       transport: 'nats',
-      reason: `"${url}" is not a connection URL`,
+      reason: 'NATS_URL is not a connection URL',
+      fix: NATS_URL_FIX,
     });
   }
   if (parsed.protocol !== 'nats:' && parsed.protocol !== 'tls:') {
@@ -104,7 +131,11 @@ export function parseNatsUrl(url: string): NatsTarget {
     });
   }
   if (parsed.hostname === '') {
-    throw new TransportUnavailableError({ transport: 'nats', reason: `"${url}" has no host` });
+    throw new TransportUnavailableError({
+      transport: 'nats',
+      reason: 'NATS_URL has no host',
+      fix: NATS_URL_FIX,
+    });
   }
   const hasUser = parsed.username !== '';
   const hasPass = parsed.password !== '';
@@ -118,8 +149,8 @@ export function parseNatsUrl(url: string): NatsTarget {
       fix: 'set the URL to nats://<user>:<pass>@host:4222, or the bare-token form nats://<token>@host:4222',
     });
   }
-  const user = hasUser ? decodeURIComponent(parsed.username) : undefined;
-  const pass = hasPass ? decodeURIComponent(parsed.password) : undefined;
+  const user = hasUser ? decodeCredential(parsed.username, 'user', parsed.hostname) : undefined;
+  const pass = hasPass ? decodeCredential(parsed.password, 'password', parsed.hostname) : undefined;
   return {
     host: parsed.hostname,
     port: parsed.port === '' ? DEFAULT_NATS_PORT : Number.parseInt(parsed.port, 10),

package/src/nats-lib-client.ts

@@ -4,7 +4,7 @@
 //
 // WHERE FAILURES ARE TRANSLATED, and it is deliberately not all here. This file coded the calls
 // that have no synchronous caller frame to catch them: `request`, `requestMany`, the dial, and the
-// background `#watch` that is the only place a lost connection is announced. `publish` and
+// background `#watch`/`#watchClosed`, the only places a lost connection is announced. `publish` and
 // `subscribe` are synchronous and stay raw — `NatsTransport.#translating` codes them, because
 // `NatsTransportOptions.connect` is a PUBLIC injection seam: translating in this class would cover
 // the one client the repo ships and leave every app-supplied one uncovered, and translating in both
@@ -72,6 +72,8 @@ class LibNatsClient implements NatsClient {
   readonly #timeoutMs: number;
   readonly #report: (error: unknown) => void;
   #connected = true;
+  /** Our own `close()` ends the status stream too, and that end is not the library giving up. */
+  #closing = false;
 
   constructor(connection: NatsConnection, target: NatsTarget, options: NatsClientOptions) {
     this.#connection = connection;
@@ -155,19 +157,22 @@ class LibNatsClient implements NatsClient {
   }
 
   async close(): Promise<void> {
+    this.#closing = true;
     this.#connected = false;
     await this.#connection.close();
   }
 
   /**
-   * The library's own status stream is the only place a background loss is announced. Nothing
+   * The library's own status stream is the only place a drop or a recovery is announced. Nothing
    * awaits it, so it can neither throw nor end the process: a drop reports and flips `connected`,
    * a reconnect flips it back and tells the transport its cluster may be a new one.
    */
   async #watch(options: NatsClientOptions): Promise<void> {
     const report = options.onError ?? ((): void => undefined);
+    const stream = this.#connection.status();
+    void this.#watchClosed(options, stream);
     try {
-      for await (const status of this.#connection.status()) {
+      for await (const status of stream) {
         if (status.type === Events.Disconnect) {
           this.#connected = false;
           report(unavailable(this.#target, 'the connection dropped'));
@@ -178,17 +183,41 @@ class LibNatsClient implements NatsClient {
           report(unavailable(this.#target, `the server reported ${String(status.data)}`));
         }
       }
-      // The iterator ends when the connection is done: either `close()` or a reconnect budget spent.
-      this.#connected = false;
-      const failure = await this.#connection.closed();
-      if (failure !== undefined) report(unavailable(this.#target, describe(failure)));
     } catch (error) {
       this.#connected = false;
       report(unavailable(this.#target, describe(error)));
     }
   }
+
+  /**
+   * The end of the connection, read off `closed()` — never off the status stream, which in
+   * nats@2.29.3 does NOT end when the connection closes (the library stops its protocol's
+   * listeners, not the connection's). A close we did not ask for is the library giving up on its
+   * reconnect budget: `onClosed`, so the transport replaces this client rather than handing a dead
+   * one out forever. The stream is stopped here, or every replaced client leaks a pending loop.
+   */
+  async #watchClosed(options: NatsClientOptions, stream: unknown): Promise<void> {
+    let failure: unknown;
+    try {
+      failure = await this.#connection.closed();
+    } catch (error) {
+      failure = error;
+    }
+    this.#connected = false;
+    stopStream(stream);
+    if (this.#closing) return;
+    if (failure !== undefined) options.onError?.(unavailable(this.#target, describe(failure)));
+    options.onClosed?.();
+  }
 }
 
+/** `QueuedIterator.stop()` is the library's, and missing from the `AsyncIterable` it is typed as. */
+const stopStream = (stream: unknown): void => {
+  if (typeof stream !== 'object' || stream === null || !('stop' in stream)) return;
+  const stop: unknown = stream.stop;
+  if (typeof stop === 'function') stop.call(stream);
+};
+
 /**
  * The production `NatsConnect`. The first dial retries on the same budget as a later loss
  * (`waitOnFirstConnect`), so a `sync` container that raced the bus into readiness recovers on its

package/src/nats-subscriptions.ts

@@ -0,0 +1,63 @@
+// Single responsibility: the transport's kept list of live subscriptions, so a client the library
+// closed for good can be replaced and every subscription bound again on the new one. The library's
+// own reconnect never reaches this list — it re-establishes the SAME client's subscriptions — so a
+// subscription is bound once per client and a re-bind can never double a delivery.
+
+import type { NatsClient, NatsMessageHandler, NatsSubscription } from './nats-client';
+
+interface Kept {
+  readonly subject: string;
+  readonly deliver: NatsMessageHandler;
+  /** The binding on the current client; `undefined` once that client is gone or the bind failed. */
+  live: NatsSubscription | undefined;
+}
+
+/** A refused bind, handed back rather than thrown: one bad subject must not strand the others. */
+export interface BindFailure {
+  readonly subject: string;
+  readonly error: unknown;
+}
+
+export class NatsSubscriptions {
+  readonly #kept = new Set<Kept>();
+
+  /**
+   * Binds on `client` first and keeps it only once bound: a subscribe the client refused is the
+   * caller's error, and keeping it would re-try a refusal on every re-dial. Returns the release.
+   */
+  add(
+    client: NatsClient,
+    subject: string,
+    deliver: NatsMessageHandler,
+    bind: (call: () => NatsSubscription) => NatsSubscription,
+  ): () => void {
+    const kept: Kept = { subject, deliver, live: undefined };
+    kept.live = bind(() => client.subscribe(subject, deliver));
+    this.#kept.add(kept);
+    return () => {
+      if (!this.#kept.delete(kept)) return;
+      const live = kept.live;
+      kept.live = undefined;
+      live?.unsubscribe();
+    };
+  }
+
+  /** The client is gone, and every binding with it. Nothing is unsubscribed: there is no wire. */
+  orphan(): void {
+    for (const kept of this.#kept) kept.live = undefined;
+  }
+
+  /** Every kept subscription, bound on a NEW client. Returns the ones it refused. */
+  bindAll(client: NatsClient): readonly BindFailure[] {
+    const failures: BindFailure[] = [];
+    for (const kept of this.#kept) {
+      if (kept.live !== undefined) continue;
+      try {
+        kept.live = client.subscribe(kept.subject, kept.deliver);
+      } catch (error) {
+        failures.push({ subject: kept.subject, error });
+      }
+    }
+    return failures;
+  }
+}