@ultimat3/realtime 20.2.1 → 22.0.0

package/src/query-window.ts

@@ -32,6 +32,8 @@ export interface QueryEntry {
   readonly input: JsonValue;
   /** Told to the client on every snapshot: the identity scope its rows belong under. */
   readonly rowEntity: string | null;
+  /** The record key a row travels under; `null` = its `id` (a plain table). */
+  readonly rowKey: ((row: Row) => string) | null;
   readonly shape: SubscriptionShape;
   readonly matcher: IncrementalMatcher;
   readonly subscribers: Map<string, LiveSubscription>;
@@ -87,6 +89,7 @@ export function createEntry(
     // Resolved with the matcher, from the same build: `prepare` has already run, so a definition
     // that compiles its shape per input can answer.
     rowEntity: definition.rowEntity?.(input) ?? null,
+    rowKey: definition.rowKey?.(input) ?? null,
     shape: {
       qid,
       // The matcher knows the dependency set this *input* produced; `definition.entities` is the
@@ -122,29 +125,43 @@ export function createEntry(
 export async function fillWindow(
   entry: QueryEntry,
 ): Promise<{ rows: readonly Row[]; lsn: string }> {
-  // Read before `startRead` clears it: a second caller arriving during the read joins it and is
-  // not the one that forced it, which is what keeps one forced read from becoming N.
-  const forced = entry.stale;
-  const pending = forced || entry.reading === null ? startRead(entry) : entry.reading;
-  const result = await pending.result;
-  return await entry.lock.run(async () => {
-    // Two rules, and neither can stand in for the other. Against another READ it is identity —
-    // the same check `startRead` makes on `entry.reading` one function down, and the one
-    // `packages/cache/src/single-flight.ts` makes for the same reason — because an lsn cannot
-    // order two reads at all: a definition with no lsn provider answers `''` for both, and
-    // `'' >= ''` let the older one overwrite the gap repair the newer one had just landed, with
-    // `stale` already cleared by its issue and therefore nothing left to re-read. Against a
-    // CHANGE it is still the lsn, because a fanout moved `entry.lsn` forwards while this read was
-    // in flight and rewinding to what the read saw hands that subscriber rows the fanout has
-    // moved past — except for a forced read, which was issued *because* what is under it is
-    // wrong.
-    if (isNewestRead(entry, pending) && (forced || result.lsn >= entry.lsn)) {
-      applyRead(entry, pending, result);
-    }
-    return { rows: entry.rows, lsn: entry.lsn };
-  });
+  for (let attempt = 0; ; attempt += 1) {
+    // Read before `startRead` clears it: a second caller arriving during the read joins it and is
+    // not the one that forced it, which is what keeps one forced read from becoming N.
+    const forced = entry.stale;
+    const pending = forced || entry.reading === null ? startRead(entry) : entry.reading;
+    const result = await pending.result;
+    const again = await entry.lock.run(async () => {
+      // Two rules, and neither can stand in for the other. Against another READ it is identity —
+      // the same check `startRead` makes on `entry.reading` one function down, and the one
+      // `packages/cache/src/single-flight.ts` makes for the same reason — because an lsn cannot
+      // order two reads at all: a definition with no lsn provider answers `''` for both, and
+      // `'' >= ''` let the older one overwrite the gap repair the newer one had just landed, with
+      // `stale` already cleared by its issue and therefore nothing left to re-read. Against a
+      // CHANGE it is still the lsn, because a fanout moved `entry.lsn` forwards while this read was
+      // in flight and rewinding to what the read saw hands that subscriber rows the fanout has
+      // moved past — except for a forced read, which was issued *because* what is under it is
+      // wrong, and for the FIRST read, which has no window under it to rewind: a fanout never
+      // patches a window no read has landed in (`live-fanout.ts`), it marks it stale instead.
+      const first = entry.applied === 0;
+      if (isNewestRead(entry, pending) && (forced || first || result.lsn >= entry.lsn)) {
+        applyRead(entry, pending, result);
+      }
+      // A change reached this window while its first read was in flight and could not be folded,
+      // so what just landed may predate it: read once more before serving anyone a partial window.
+      return first && entry.stale && attempt < COLD_REREADS;
+    });
+    if (!again) return { rows: entry.rows, lsn: entry.lsn };
+  }
 }
 
+/**
+ * How many times a cold window re-reads because writes kept landing during its read. Bounded: a
+ * table written faster than it can be read would otherwise never serve a subscriber, and after the
+ * bound the window stays `stale`, so the next change re-reads it anyway.
+ */
+const COLD_REREADS = 3;
+
 /**
  * The same replacement, for a caller that is already holding the lane. A fanout cannot call
  * `fillWindow` — that takes the entry's own lane, and a lane is not reentrant — so the one path

package/src/reactivity.ts

@@ -0,0 +1,70 @@
+// What a hook renders through: THIS island bundle's signal factory. Module scope on purpose —
+// every island carries its own solid-js, so a signal must come from the bundle whose effects read
+// it, while the store behind it is the page's (`page-store.ts`). The island bootstrap installs it.
+
+import type { SignalFactory } from './client-contract';
+import { RealtimeUninstalledError } from './page-errors';
+import { pageRealtime, type SyncTarget } from './page-store';
+
+export interface RealtimeInstall {
+  /** `createSignal` from this island's solid-js, narrowed to two functions. */
+  readonly signal: SignalFactory;
+  /** Where the page's socket dials. Omitted by an island that holds no live hook. */
+  readonly sync?: SyncTarget;
+}
+
+let installed: SignalFactory | undefined;
+
+/**
+ * Called by the island bootstrap `x build` prepends — never by an island's own code. Per bundle
+ * for the signal, per page for the sync target (the first one given wins; every island on a page
+ * was rendered by one server and names one node).
+ */
+export function installRealtime(install: RealtimeInstall): void {
+  installed = install.signal;
+  const page = pageRealtime();
+  if (install.sync !== undefined && page.sync === undefined) page.sync = install.sync;
+}
+
+/**
+ * A DOM is the whole question, the same probe and the same words as `@ultimat3/ui`'s `solid()`:
+ * with one, a hook that finds nothing installed is a real bug; without one it is a server render.
+ */
+export function hasDom(): boolean {
+  return typeof document !== 'undefined' && typeof window !== 'undefined';
+}
+
+/**
+ * A signal that never changes, because nothing on the server can change it: one render, one pass.
+ * The setter is kept so a caller reads its own write back — a signal that swallowed writes would
+ * be a different lie.
+ */
+const inertSignal: SignalFactory = <T>(initial: T): [() => T, (next: T) => void] => {
+  let held = initial;
+  return [
+    (): T => held,
+    (next: T): void => {
+      held = next;
+    },
+  ];
+};
+
+/** The factory `hook` renders through: this bundle's, or the inert one on a server render. */
+export function signalFor(hook: string): SignalFactory {
+  if (installed !== undefined) return installed;
+  if (hasDom()) throw new RealtimeUninstalledError({ hook });
+  return inertSignal;
+}
+
+/**
+ * A server render: nothing installed and no DOM. A hook answers the honest server state and never
+ * touches the page state — on a server that would be ONE store shared by every request.
+ */
+export function isServerRender(): boolean {
+  return installed === undefined && !hasDom();
+}
+
+/** For tests: forget this bundle's install so cases stay independent. */
+export function uninstallRealtime(): void {
+  installed = undefined;
+}

package/src/realtime-error.ts

@@ -6,7 +6,7 @@
 // that would be a cycle: `extends` runs at module evaluation, imports hoist above it, and the base
 // would be in its temporal dead zone by the time the subclass module was evaluated.
 
-import { UltimateError } from '@ultimat3/core';
+import { UltimateError } from '@ultimat3/core/page';
 import type { RealtimeErrorCode } from './errors';
 
 /**

package/src/record-await.ts

@@ -0,0 +1,102 @@
+// The overlays waiting on server truth: a write whose answer did not carry every row it touched
+// keeps its overlay until the server next reaches one of those rows, bounded by a timer. Also what
+// the server reached while a write was still in flight — the frame routinely beats the answer.
+
+import { finiteCount } from '@ultimat3/core/page';
+import type { RecordKey } from './record-key';
+import type { Scheduler } from './thundering-herd';
+
+/** An overlay whose answer did not carry every row it wrote waits this long for one, no longer. */
+export const DEFAULT_AWAIT_SERVER_MS = 10_000;
+
+export class ServerWait {
+  /** Each waiting overlay: the rows it waits for, and the timer bounding the wait. */
+  readonly #awaiting = new Map<string, { readonly rows: ReadonlySet<RecordKey>; cancel(): void }>();
+  /**
+   * Rows server truth reached while each overlay's write was still in flight. The node fans a
+   * commit out before the response is written, so the frame routinely beats the answer — and a
+   * settle that then waited for ANOTHER server write replayed the twin over a row that already
+   * held it, counting the write twice until the bound.
+   */
+  readonly #heard = new Map<string, Set<RecordKey>>();
+  readonly #schedule: Scheduler;
+  readonly #ms: number;
+
+  constructor(schedule: Scheduler | undefined, awaitMs: number | undefined) {
+    // Inline rather than `thundering-herd`'s `timeoutScheduler`: that module carries the backoff,
+    // and a `useRecord`-only island would pay for it to arm one timer.
+    this.#schedule =
+      schedule ??
+      ((fn, ms) => {
+        const timer = setTimeout(fn, ms);
+        return () => clearTimeout(timer);
+      });
+    this.#ms = finiteCount('RecordStore', 'awaitMs', awaitMs ?? DEFAULT_AWAIT_SERVER_MS, 1);
+  }
+
+  /** Wait for any of `rows`; `expire` runs if none is reached in time. */
+  start(key: string, rows: ReadonlySet<RecordKey>, expire: () => void): void {
+    this.#awaiting.get(key)?.cancel();
+    const cancel = this.#schedule(() => {
+      // Nothing answered in time: the caller drops the overlay and server truth stands.
+      if (this.#awaiting.delete(key)) expire();
+    }, this.#ms);
+    this.#awaiting.set(key, { rows, cancel });
+  }
+
+  /** The overlay went some other way: its wait and what it heard go with it. */
+  forget(key: string): void {
+    this.#awaiting.get(key)?.cancel();
+    this.#awaiting.delete(key);
+    this.#heard.delete(key);
+  }
+
+  /** A replay dropped the overlay: what it heard goes; a wait ends when `resolve` sees it gone. */
+  unhear(key: string): void {
+    this.#heard.delete(key);
+  }
+
+  /** What the server reached while `key` was in flight — taken, so a second settle starts over. */
+  takeHeard(key: string): ReadonlySet<RecordKey> | undefined {
+    const heard = this.#heard.get(key);
+    this.#heard.delete(key);
+    return heard;
+  }
+
+  /** Note, per in-flight overlay (one not already waiting), which of its rows the server reached. */
+  hear(
+    inFlight: Iterable<[string, ReadonlySet<RecordKey>]>,
+    reached: (rk: RecordKey) => boolean,
+  ): void {
+    for (const [key, touched] of inFlight) {
+      if (this.#awaiting.has(key)) continue;
+      const wrote = [...touched].filter(reached);
+      if (wrote.length === 0) continue;
+      const heard = this.#heard.get(key) ?? new Set<RecordKey>();
+      for (const rk of wrote) heard.add(rk);
+      this.#heard.set(key, heard);
+    }
+  }
+
+  /**
+   * The waits the server just answered — or whose overlay is already gone — ended. Answers the
+   * keys whose overlay the caller must now drop because server truth reached it.
+   */
+  resolve(pending: (key: string) => boolean, moved: ReadonlySet<RecordKey>): readonly string[] {
+    const answered: string[] = [];
+    for (const [key, waiting] of [...this.#awaiting]) {
+      const gone = !pending(key);
+      if (!gone && ![...waiting.rows].some((rk) => moved.has(rk))) continue;
+      waiting.cancel();
+      this.#awaiting.delete(key);
+      if (!gone) answered.push(key);
+    }
+    return answered;
+  }
+
+  clear(): void {
+    for (const waiting of this.#awaiting.values()) waiting.cancel();
+    this.#awaiting.clear();
+    this.#heard.clear();
+  }
+}

package/src/record-key.ts

@@ -0,0 +1,34 @@
+// How a record is named in the page store: `type:key`, and the set of names an answer carried.
+// Apart from the store so a module that only names records never loads the store's machinery.
+
+import type { RecordRows, Row } from '@ultimat3/core/page';
+
+/** `type:key`. A record type is an entity name, which never holds `:`, so the split is unambiguous. */
+export type RecordKey = string;
+
+export function recordKey(type: string, key: string): RecordKey {
+  return `${type}:${key}`;
+}
+
+/**
+ * Every record an answer's envelope names — adopted or removed — as `type:key`, added to `into`.
+ * What an overlay settles against: a row the answer did not name keeps its overlay.
+ */
+export function carriedBy(
+  envelope: {
+    readonly records?: Readonly<Record<string, RecordRows>>;
+    readonly removed?: Readonly<Record<string, readonly string[]>>;
+  },
+  into: Set<RecordKey>,
+): void {
+  for (const [type, rows] of Object.entries(envelope.records ?? {})) {
+    for (const key of Object.keys(rows)) into.add(recordKey(type, key));
+  }
+  for (const [type, keys] of Object.entries(envelope.removed ?? {})) {
+    for (const key of keys) into.add(recordKey(type, key));
+  }
+}
+
+/** A row is a plain object: an array, `null` or a scalar off the wire is never merged. */
+export const isRow = (value: unknown): value is Row =>
+  typeof value === 'object' && value !== null && !Array.isArray(value);

package/src/record-names.ts

@@ -0,0 +1,45 @@
+// Which pending overlay a `records` frame's `write` names. The node stamps a frame with the digest
+// of the idempotency key its write arrived under (`writeDigest`, `@ultimat3/core`); the page digests
+// its own keys as it pushes them, so a frame is recognised as this page's own echo by lookup.
+
+import { writeDigest } from '@ultimat3/core/page';
+
+export class WriteNames {
+  readonly #byDigest = new Map<string, string>();
+  readonly #byKey = new Map<string, string>();
+
+  /**
+   * Digest `key` and remember it while `pending(key)` still holds. Started when the overlay is
+   * pushed, before its request is sent: the digest resolves in microseconds and the echo needs the
+   * request to reach the node, commit and fan back out first. No `crypto.subtle` (plain HTTP off
+   * `localhost`) names nothing, and the echo is a plain merge — the behaviour before frames named
+   * their write.
+   */
+  name(key: string, pending: (key: string) => boolean): void {
+    void writeDigest(key).then(
+      (digest) => {
+        if (digest === undefined || !pending(key)) return;
+        this.#byDigest.set(digest, key);
+        this.#byKey.set(key, digest);
+      },
+      () => undefined,
+    );
+  }
+
+  /** The overlay `digest` names, if it is one this page pushed and still holds. */
+  keyOf(digest: string): string | undefined {
+    return this.#byDigest.get(digest);
+  }
+
+  forget(key: string): void {
+    const digest = this.#byKey.get(key);
+    if (digest === undefined) return;
+    this.#byKey.delete(key);
+    this.#byDigest.delete(digest);
+  }
+
+  clear(): void {
+    this.#byDigest.clear();
+    this.#byKey.clear();
+  }
+}

package/src/record-persister.ts

@@ -0,0 +1,156 @@
+/**
+ * Keeps the record types an app marked `persist: true` on disk, per principal, and puts them back
+ * on the next load BEFORE the socket connects. Writes are debounced and flushed when the page is
+ * hidden or leaves; a restored row is stale until the server confirms it, and the first server row
+ * for that key wins outright.
+ *
+ * Only SYNCED truth is written — never an optimistic overlay: a write the server later refuses must
+ * not survive a reload as if it had landed. Its intent survives in the outbox instead.
+ */
+
+import type { ClientScope, RecordRows, Row } from '@ultimat3/core/page';
+import { CLIENT_PERSIST_META, finiteCount, onRescope, pageClient } from '@ultimat3/core/page';
+import type { LocalStore } from './local-store-idb';
+import { scopeKey } from './local-store-idb';
+
+/**
+ * What the persister needs from the page's store. `synced` and `restore` are the two methods
+ * `RecordStore` owes this module (plan 101 slice 12): the server's row without the overlay, and a
+ * restore that yields to — and is replaced by — the first server row for the same key.
+ */
+export interface PersistableStore {
+  subscribe(listener: (changed: ReadonlySet<string>) => void): () => void;
+  synced(type: string, key: string): Row | undefined;
+  restore(type: string, rows: RecordRows): void;
+}
+
+export interface RecordPersisterOptions {
+  readonly store: PersistableStore;
+  readonly local: LocalStore;
+  /** The record types to keep — `persistedTypes()` in a browser. */
+  readonly types: ReadonlySet<string>;
+  readonly principal?: (() => ClientScope['principal']) | undefined;
+  readonly debounceMs?: number | undefined;
+  readonly schedule?: ((fn: () => void, ms: number) => () => void) | undefined;
+  /** Subscribes to "the page is going away": `pagehide`, and `visibilitychange` to hidden. */
+  readonly onLeave?: ((flush: () => void) => () => void) | undefined;
+}
+
+export interface RecordPersister {
+  /** Puts this principal's rows back. Resolves with how many were restored. */
+  restore(): Promise<number>;
+  /** Writes everything changed since the last flush, now. */
+  flush(): Promise<void>;
+  stop(): void;
+}
+
+export function recordPersister(options: RecordPersisterOptions): RecordPersister {
+  const principal =
+    options.principal ?? ((): ClientScope['principal'] => pageClient().scope.principal);
+  const schedule =
+    options.schedule ??
+    ((fn: () => void, ms: number): (() => void) => {
+      const timer = setTimeout(fn, ms);
+      return () => clearTimeout(timer);
+    });
+  // A whole number of ms, at least 1: `NaN` would arm a timer that fires at once, forever.
+  const debounceMs = finiteCount('recordPersister', 'debounceMs', options.debounceMs ?? 250, 1);
+  const dirty = new Set<string>();
+  let cancel: (() => void) | undefined;
+
+  const flush = async (): Promise<void> => {
+    cancel?.();
+    cancel = undefined;
+    const scope = scopeKey(principal());
+    const changed = [...dirty];
+    dirty.clear();
+    if (scope === undefined || changed.length === 0) return;
+    const puts: { type: string; key: string; row: Row }[] = [];
+    const deletes: { type: string; key: string }[] = [];
+    for (const rk of changed) {
+      const at = rk.indexOf(':');
+      const type = rk.slice(0, at);
+      const key = rk.slice(at + 1);
+      const row = options.store.synced(type, key);
+      if (row === undefined) deletes.push({ type, key });
+      else puts.push({ type, key, row });
+    }
+    await options.local.write(scope, puts, deletes);
+  };
+
+  const unsubscribe = options.store.subscribe((changed) => {
+    for (const rk of changed) {
+      if (options.types.has(rk.slice(0, rk.indexOf(':')))) dirty.add(rk);
+    }
+    if (dirty.size > 0 && cancel === undefined) {
+      cancel = schedule(() => void flush(), debounceMs);
+    }
+  });
+
+  const restore = async (): Promise<number> => {
+    const scope = scopeKey(principal());
+    if (scope === undefined) return 0;
+    let restored = 0;
+    for (const [type, rows] of await options.local.rows(scope)) {
+      if (!options.types.has(type)) continue;
+      options.store.restore(type, rows);
+      restored += Object.keys(rows).length;
+    }
+    return restored;
+  };
+
+  // A principal change: the previous principal's rows leave the disk with it, nothing it changed is
+  // written under the next one, and the next one's own rows come back.
+  const unscope = onRescope((_next, prev) => {
+    cancel?.();
+    cancel = undefined;
+    dirty.clear();
+    const gone = scopeKey(prev.principal);
+    void (gone === undefined ? Promise.resolve() : options.local.wipe(gone)).then(restore);
+  });
+  const unleave = (options.onLeave ?? onPageLeave)(() => void flush());
+
+  return {
+    restore,
+    flush,
+    stop: (): void => {
+      cancel?.();
+      unsubscribe();
+      unscope();
+      unleave();
+    },
+  };
+}
+
+/** `pagehide` and a hidden `visibilitychange` — the last moments a page can still write. */
+function onPageLeave(flush: () => void): () => void {
+  const doc: (EventTarget & { visibilityState?: string }) | undefined = Reflect.get(
+    globalThis,
+    'document',
+  );
+  // A partial `document` (a test's stand-in) has no events to hear: nothing to subscribe to.
+  if (doc === undefined || typeof doc.addEventListener !== 'function') return () => {};
+  const hidden = (): void => {
+    if (doc.visibilityState === 'hidden') flush();
+  };
+  globalThis.addEventListener('pagehide', flush);
+  doc.addEventListener('visibilitychange', hidden);
+  return () => {
+    globalThis.removeEventListener('pagehide', flush);
+    doc.removeEventListener('visibilitychange', hidden);
+  };
+}
+
+/** The types the server rendered as persisted (`<meta name="ultimate-persist">`). None = none. */
+export function persistedTypes(): ReadonlySet<string> {
+  const doc: { querySelector?: (selector: string) => { content?: unknown } | null } | undefined =
+    Reflect.get(globalThis, 'document');
+  const content = doc?.querySelector?.(`meta[name="${CLIENT_PERSIST_META}"]`)?.content;
+  if (typeof content !== 'string') return new Set();
+  return new Set(
+    content
+      .split(',')
+      .map((type) => type.trim())
+      .filter((type) => type !== ''),
+  );
+}