@ultimat3/realtime 27.10.2 → 27.10.3

package/CLAUDE.md

@@ -236,9 +236,13 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
 - **The socket is READ-ONLY (protocol 3)**. A write is `useMutation`: the twin into the overlay, then
   `POST actionPath(name)` through core's `clientTransport` with an idempotency key; the answer's
   records are adopted inside the transport call, before `settle`. An `ack` is only a refusal.
+  KNOWN offline (`knownOffline()`), it is queued with no POST and painted once on disk (#777).
 - **`LiveClient` holds no signal** — plain reads plus `onStatus` / `onChange`; hooks wrap them.
 - **An answer that did not carry a touched row keeps that row's overlay** until the server's next row
   for it, bounded by `DEFAULT_AWAIT_SERVER_MS` (10 s).
+- **A catch-up read waits for the page's writes in flight over its types**
+  (`RecordStore.afterWrites`): it lands once each is answered or echoed, within that bound; a row
+  moved meanwhile is kept (#778).
 - **A frame names the write that produced it**: `records.write`, `patch.writes` (a live patch
   one, a resume each it replays, off `RowPatch.write` in the ring) — `writeDigest(key)`, from a
   REQUIRED `ChangeEvent.write: string | null`. `RecordStore.push` digests each overlay key first
@@ -294,15 +298,12 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
 
 ## Map
 
-`index.ts` / `server.ts` are the two barrels. Server side: `sync-node.ts` (+ `sync-auth`,
-`sync-frames`, `sync-upgrade`, `sync-listen`, `drain-evictions`), `channel.ts`, `presence.ts`,
-`socket.ts`, `live-query.ts` (+ `live-definition`, `live-refusal`, `live-tenant`, `live-resume`, `query-window`, `live-fanout`, `window-lock`,
-`subscriber-gate`, `policy-gate` — the only authz seam — and `matcher-bridge`), `replicator.ts` (+ `replicator-envelope`) /
-`live-replicator.ts`, the Postgres client (`pg-*.ts`, `pgoutput.ts`), the bus (`nats-*.ts`,
-`transport-env.ts`, `fanout.ts`). Client side: `client*.ts`, `socket-engine.ts` / `socket-host.ts` /
-`sync-worker.ts`, `record-*.ts`, `page-*.ts`, `boot.ts`, `offline-queue.ts`, `use-*.ts`,
-`reactivity.ts`. Shared: `sync-protocol.ts` (the wire), `cursor.ts`, `change-buffer.ts`,
-`thundering-herd.ts`, `live-contract.ts`, `json.ts` (no hash), `type-pins.ts`.
+`index.ts` / `server.ts` are the two barrels. Server side: `sync-*.ts`, `channel*.ts`, `presence.ts`,
+`socket.ts`, `live-*.ts` (`policy-gate.ts` is the only authz seam), `replicator*.ts`, the Postgres
+client (`pg-*.ts`, `pgoutput.ts`), the bus (`nats-*.ts`, `transport-env.ts`, `fanout.ts`). Client
+side: `client*.ts`, `socket-*.ts`, `sync-worker.ts`, `record-*.ts`, `page-*.ts`, `catch-up-hold.ts`,
+`boot.ts`, `offline-queue.ts`, `use-*.ts`, `reactivity.ts`. Shared: `sync-protocol.ts` (the wire),
+`cursor.ts`, `change-buffer.ts`, `thundering-herd.ts`, `live-contract.ts`, `json.ts`, `type-pins.ts`.
 
 ## Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/realtime",
-  "version": "27.10.2",
+  "version": "27.10.3",
   "description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
   "license": "MIT",
   "type": "module",
@@ -40,10 +40,10 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "27.10.2",
-    "@ultimat3/entity": "27.10.2",
-    "@ultimat3/policy": "27.10.2",
-    "@ultimat3/query": "27.10.2",
+    "@ultimat3/core": "27.10.3",
+    "@ultimat3/entity": "27.10.3",
+    "@ultimat3/policy": "27.10.3",
+    "@ultimat3/query": "27.10.3",
     "nats": "2.29.3"
   }
 }

package/src/catch-up-hold.ts

@@ -0,0 +1,76 @@
+// A channel's catch-up read, held until this page's writes in flight over its record types are
+// answered (#778): the read names no write, and its snapshot may already hold one whose twin is
+// still on screen — adopted then, the like is counted twice until the write's answer or echo.
+// The store bounds the hold (`afterWrites`); a row the server moved while it was held is left alone.
+
+import type { Row } from '@ultimat3/core/page';
+import type { CatchUpRecords } from './channel-catch-up';
+import { recordKey } from './record-key';
+import type { HeldLanding } from './record-landings';
+import type { RecordStore } from './record-store';
+
+/**
+ * `land` gets the read once nothing blocks it, minus every COLUMN whose server truth moved
+ * meanwhile: that value ARRIVED after the read did — a write's own answer, another subscription's
+ * frame — and the store's rule is that the later arrival stands. A column nothing moved is still
+ * the read's (the store merges by column: a narrower query moving `likes` says nothing of `title`).
+ * A row gone since — removed, or evicted — is not brought back, and a removal is skipped for a row
+ * that moved at all.
+ */
+export function holdCatchUp(
+  store: RecordStore,
+  answered: CatchUpRecords | undefined,
+  echoed: () => Iterable<string>,
+  land: (fresh: CatchUpRecords | undefined) => void,
+): HeldLanding {
+  if (answered === undefined) {
+    land(undefined);
+    return { recheck: () => undefined };
+  }
+  const seen = new Map<string, Row | undefined>();
+  const types = new Set<string>();
+  for (const [type, rows] of Object.entries(answered.records ?? {})) {
+    types.add(type);
+    for (const key of Object.keys(rows)) seen.set(recordKey(type, key), store.synced(type, key));
+  }
+  for (const [type, keys] of Object.entries(answered.removed ?? {})) {
+    types.add(type);
+    for (const key of keys) seen.set(recordKey(type, key), store.synced(type, key));
+  }
+  return store.afterWrites(types, () => land(unmovedOnly(store, answered, seen)), echoed);
+}
+
+/** Built with `fromEntries`, never by assignment: a record type and a column are the server's data. */
+function unmovedOnly(
+  store: RecordStore,
+  answered: CatchUpRecords,
+  seen: ReadonlyMap<string, Row | undefined>,
+): CatchUpRecords {
+  const records = Object.fromEntries(
+    Object.entries(answered.records ?? {}).map(([type, rows]) => [
+      type,
+      Object.fromEntries(
+        Object.entries(rows).flatMap(([key, row]) => {
+          const kept = unmovedColumns(seen.get(recordKey(type, key)), store.synced(type, key), row);
+          return kept === undefined ? [] : [[key, kept]];
+        }),
+      ),
+    ]),
+  );
+  const removed = Object.fromEntries(
+    Object.entries(answered.removed ?? {}).map(([type, keys]) => [
+      type,
+      keys.filter((key) => seen.get(recordKey(type, key)) === store.synced(type, key)),
+    ]),
+  );
+  return { ...answered, records, removed };
+}
+
+/** The read's row without the columns that moved between `before` and `now`; none if it is gone. */
+function unmovedColumns(before: Row | undefined, now: Row | undefined, read: Row): Row | undefined {
+  if (before === now) return read;
+  if (now === undefined) return undefined;
+  return Object.fromEntries(
+    Object.entries(read).filter(([column]) => before?.[column] === now[column]),
+  );
+}

package/src/client-channels.ts

@@ -2,10 +2,12 @@
 // per-channel cursor that decides duplicate from new, and the catch-up read a `replay-gap` or a
 // new epoch triggers. `records` frames go to the store and nowhere else; `events` go to handlers.
 
+import { holdCatchUp } from './catch-up-hold';
 import type { CatchUpRecords } from './channel-catch-up';
 import { type PresenceEvent, readPresence } from './channel-presence';
 import type { JsonObject } from './json';
 import { carriedBy, type RecordKey } from './record-key';
+import type { HeldLanding } from './record-landings';
 import type { RecordStore } from './record-store';
 import type {
   ChannelEventsFrame,
@@ -80,6 +82,8 @@ interface Entry {
   failures: number;
   /** Disarms the scheduled retry. */
   disarm: (() => void) | null;
+  /** The read ANSWERED and waits on this page's writes in flight (`catch-up-hold.ts`). */
+  landing: HeldLanding | null;
 }
 
 export interface ChannelMembership extends Disposable {
@@ -124,6 +128,7 @@ export class ChannelBook {
         waiting: false,
         failures: 0,
         disarm: null,
+        landing: null,
       };
       this.#byTopic.set(topic, entry);
       if (this.#deps.connected()) this.#deps.send(this.#frame(entry, 'add', true));
@@ -206,6 +211,8 @@ export class ChannelBook {
     if (entry === undefined) return;
     if (entry.buffered !== null) {
       entry.buffered.push(frame);
+      // The echo of a write the answered read waits on: it lands with the read, settling it there.
+      if (frame.write !== undefined) entry.landing?.recheck();
       return;
     }
     if (entry.epoch !== null && entry.epoch !== frame.epoch) {
@@ -310,14 +317,26 @@ export class ChannelBook {
     entry.disarm?.();
     entry.disarm = null;
     this.#set(entry, 'catching-up');
-    // The read names no write (O-507, reopened 2026-10-14 on an observed double paint). FIXED: a
-    // pending write whose echo is HELD behind the read is settled in the same notification as the
-    // read's rows (`#drain`). NOT fixed: an echo that arrives AFTER the read answered (replication
-    // slower than the read's round trip) — the rows are truth under the still-pending twin until
-    // that echo or the write's own answer settles it. Closing it needs the read to name the writes
-    // its snapshot reflects, which is a server-side change (O-507 in full).
+    // The read names no write (O-507). A pending write whose echo is HELD behind the read is settled
+    // in the same notification as the read's rows (`#drain`). One whose echo comes AFTER the read
+    // answered (replication slower than the read) is waited for (#778): the read lands once the
+    // writes in flight over its types are answered or echoed, bounded by the store's server wait.
     this.#deps.catchUp(entry.catchUp, entry.params).then(
-      (answered) => this.#drain(entry, answered),
+      (answered) => {
+        if (this.#byTopic.get(entry.topic) !== entry) return;
+        let landed = false;
+        const held = holdCatchUp(
+          this.#deps.store,
+          answered,
+          () => echoes(entry),
+          (fresh) => {
+            landed = true;
+            entry.landing = null;
+            this.#drain(entry, fresh);
+          },
+        );
+        if (!landed) entry.landing = held;
+      },
       (error: unknown) => {
         this.#deps.report(error);
         // NOT live: the frames held behind the read would land over a store it never refreshed.
@@ -341,6 +360,16 @@ export class ChannelBook {
     // Released while the read was out: its rows and frames are a seat nobody holds, and a new hold
     // of the topic may already hold newer rows. Nothing lands and nothing is re-read.
     if (this.#byTopic.get(entry.topic) !== entry) return;
+    // The principal the read was made for has left (a sign-out cleared the store while the read
+    // was held): the frames held behind it are theirs too. Dropped, never applied to the next
+    // principal's store, and the channel is read again — as whoever holds the page now.
+    if (answered?.current?.() === false) {
+      entry.buffered = [];
+      entry.again = null;
+      this.#restart(entry, entry.epoch);
+      this.#read(entry);
+      return;
+    }
     const held = entry.buffered ?? [];
     // A newer epoch arrived while the read was in flight (the node restarted mid-read): the read
     // answered for the OLD one, so it is re-read in the new one, keeping only that epoch's frames.
@@ -416,6 +445,13 @@ export function isTransientRefusal(error: unknown): boolean {
   );
 }
 
+/** The writes the frames held behind a read name — each lands, and settles, with the read. */
+function echoes(entry: Entry): string[] {
+  const named: string[] = [];
+  for (const frame of entry.buffered ?? []) if (frame.write !== undefined) named.push(frame.write);
+  return named;
+}
+
 /** A channel membership's sid is its topic behind this — what the socket engine routes on too. */
 export const CHANNEL_SID_PREFIX = 'channel:';
 

package/src/outbox-slot.ts

@@ -41,3 +41,19 @@ export type OutboxHost = { [OUTBOX_KEY]?: OutboxHandle };
 export function peekOutbox(): OutboxHandle | undefined {
   return (globalThis as OutboxHost)[OUTBOX_KEY];
 }
+
+/**
+ * The browser says it has no network. A request then is an attempt that can only fail — and a
+ * failed attempt is still a request on the wire: measured, a reload taken offline replayed on open,
+ * that POST failed, and the real one followed on `online`, so one write went out twice. The
+ * `online` event is the outbox's replay trigger, so nothing is lost by waiting for it. Unknown (no
+ * `navigator`) is not offline.
+ */
+export function knownOffline(): boolean {
+  const navigator: unknown = Reflect.get(globalThis, 'navigator');
+  return (
+    typeof navigator === 'object' &&
+    navigator !== null &&
+    Reflect.get(navigator, 'onLine') === false
+  );
+}

package/src/page-outbox.ts

@@ -18,7 +18,13 @@ import type { LocalStore } from './local-store-idb';
 import { pageLocalStore, scopeKey } from './local-store-idb';
 import type { DrainReport, QueuedMutation, QueueState, QueueStore } from './offline-queue';
 import { memoryQueueStore, OfflineQueue, toQueueError } from './offline-queue';
-import { OUTBOX_KEY, type OutboxEntry, type OutboxHandle, type OutboxHost } from './outbox-slot';
+import {
+  knownOffline,
+  OUTBOX_KEY,
+  type OutboxEntry,
+  type OutboxHandle,
+  type OutboxHost,
+} from './outbox-slot';
 import { type OutboxTabs, type OutboxTabsWindow, outboxTabs } from './outbox-tabs';
 import { LocalStoreUnavailableError, OfflineQueueAbandonedError } from './page-errors';
 import { peekPageRealtime } from './page-store';
@@ -427,19 +433,3 @@ export function listenForDrain(outbox: Pick<PageOutbox, 'replay'>): () => void {
     globalThis.removeEventListener('online', replay);
   };
 }
-
-/**
- * The browser says it has no network. A replay then is an attempt that can only fail — and a
- * failed attempt is still a request on the wire: measured, a reload taken offline replayed on open,
- * that POST failed, and the real one followed on `online`, so one write went out twice. The
- * `online` event is the replay's own trigger, so nothing is lost by waiting for it. Unknown (no
- * `navigator`) is not offline.
- */
-function knownOffline(): boolean {
-  const navigator: unknown = Reflect.get(globalThis, 'navigator');
-  return (
-    typeof navigator === 'object' &&
-    navigator !== null &&
-    Reflect.get(navigator, 'onLine') === false
-  );
-}

package/src/record-await.ts

@@ -44,6 +44,16 @@ export class ServerWait {
     this.#awaiting.set(key, { rows, cancel });
   }
 
+  /** Whether `key`'s write was answered and its overlay now waits for server truth. */
+  awaiting(key: string): boolean {
+    return this.#awaiting.has(key);
+  }
+
+  /** `fire` after this wait's bound — the one bound on waiting for the server (`awaitMs`). */
+  bound(fire: () => void): () => void {
+    return this.#schedule(fire, this.#ms);
+  }
+
   /** The overlay went some other way: its wait and what it heard go with it. */
   forget(key: string): void {
     this.#awaiting.get(key)?.cancel();

package/src/record-landings.ts

@@ -0,0 +1,69 @@
+// Server truth that must wait for the page's own writes before it lands (#778): a channel's
+// catch-up read whose snapshot may already hold a write still in flight. Adopted under that write's
+// twin it is the write counted twice, so it lands once nothing blocks it — or when the store's one
+// server-wait bound passes, and then exactly as it always did.
+
+/** A held landing. `recheck` asks again now, for what changed outside the store (a held echo). */
+export interface HeldLanding {
+  recheck(): void;
+}
+
+interface Held {
+  readonly blocked: () => boolean;
+  fire(): void;
+}
+
+const LANDED: HeldLanding = { recheck: () => undefined };
+
+export class Landings {
+  readonly #held = new Set<Held>();
+  readonly #bound: (fire: () => void) => () => void;
+  readonly #report: (error: unknown) => void;
+
+  /**
+   * `bound` arms the ceiling on one hold: the store's server wait, never a second number. A held
+   * landing runs inside whatever batch released it, so what it throws is `report`ed, never raised
+   * into that unrelated caller (a write's settle) or left to stop the landings behind it.
+   */
+  constructor(bound: (fire: () => void) => () => void, report: (error: unknown) => void) {
+    this.#bound = bound;
+    this.#report = report;
+  }
+
+  /** `land` runs at once when nothing blocks it; otherwise at the first check that finds none. */
+  hold(blocked: () => boolean, land: () => void): HeldLanding {
+    if (!blocked()) {
+      land();
+      return LANDED;
+    }
+    let done = false;
+    let cancel = (): void => undefined;
+    const held: Held = {
+      blocked,
+      fire: () => {
+        if (done) return;
+        done = true;
+        this.#held.delete(held);
+        cancel();
+        try {
+          land();
+        } catch (error) {
+          this.#report(error);
+        }
+      },
+    };
+    this.#held.add(held);
+    cancel = this.#bound(() => held.fire());
+    return {
+      recheck: () => {
+        if (!done && !blocked()) held.fire();
+      },
+    };
+  }
+
+  /** After every store notification: whatever nothing blocks any longer lands, in hold order. */
+  check(): void {
+    if (this.#held.size === 0) return;
+    for (const held of [...this.#held]) if (!held.blocked()) held.fire();
+  }
+}

package/src/record-store.ts

@@ -8,6 +8,7 @@ import { resolveConflict } from '@ultimat3/core/page';
 import { RebaseConflictError, RecordRejectedError } from './page-errors';
 import { ServerWait } from './record-await';
 import { isRow, type RecordKey, recordKey } from './record-key';
+import { type HeldLanding, Landings } from './record-landings';
 import { WriteNames } from './record-names';
 import { SyncedLayer } from './record-synced';
 import { type OverlayEntry, replayOverlays, sameRow } from './record-tx';
@@ -45,10 +46,16 @@ export class RecordStore implements RecordSink {
   readonly #wait: ServerWait;
   /** Which pending overlay each write digest names — what a frame's `write` is looked up in. */
   readonly #names = new WriteNames();
+  /** Server truth held behind this page's writes in flight (`afterWrites`). */
+  readonly #landings: Landings;
 
   constructor(options: RecordStoreOptions = {}) {
     this.#report = options.report ?? ((error) => console.error(error));
     this.#wait = new ServerWait(options.schedule, options.awaitMs);
+    this.#landings = new Landings(
+      (fire) => this.#wait.bound(fire),
+      (error) => this.#report(error),
+    );
   }
 
   /** The record as every holder sees it: the overlay's answer over synced truth. */
@@ -260,6 +267,33 @@ export class RecordStore implements RecordSink {
     this.#wait.start(key, this.#touched.get(key) ?? new Set(), () => this.drop(key));
   }
 
+  /**
+   * Runs `land` once every write IN FLIGHT NOW over a record of `types` is answered — settled,
+   * dropped, or waiting for server truth — or its echo is among the `echoed` digests (a frame held
+   * to land with `land`), or once the server-wait bound (`awaitMs`) passes, whichever is first. A
+   * write made later does not extend it. What a catch-up read's rows wait on (#778): adopted under
+   * a twin whose write the read already holds, they paint that write twice.
+   */
+  afterWrites(
+    types: ReadonlySet<string>,
+    land: () => void,
+    echoed: () => Iterable<string> = () => [],
+  ): HeldLanding {
+    const prefixes = [...types].map((type) => `${type}:`);
+    const over = (rk: RecordKey): boolean => prefixes.some((prefix) => rk.startsWith(prefix));
+    const writes = [...this.#overlays.keys()].filter((key) =>
+      [...(this.#touched.get(key) ?? [])].some(over),
+    );
+    const inFlight = (key: string): boolean => this.#overlays.has(key) && !this.#wait.awaiting(key);
+    return this.#landings.hold(() => {
+      const live = writes.filter(inFlight);
+      if (live.length === 0) return false;
+      const named = new Set<string | undefined>();
+      for (const digest of echoed()) named.add(this.#names.keyOf(digest));
+      return live.some((key) => !named.has(key));
+    }, land);
+  }
+
   /** Pending optimistic writes, oldest first. */
   pending(): readonly string[] {
     return [...this.#overlays.keys()];
@@ -302,6 +336,8 @@ export class RecordStore implements RecordSink {
       this.#moved.clear();
       this.#changed = null;
       if (collected.size > 0) for (const listener of this.#listeners) listener(collected);
+      // A write that settled in this batch may be the last a held landing waited on.
+      this.#landings.check();
     }
   }
 

package/src/use-mutation.ts

@@ -6,7 +6,7 @@
 import type { ConflictPolicy } from '@ultimat3/core/page';
 import { actionPath, isSuperseded, isUltimateError, pageClient, uuidV7 } from '@ultimat3/core/page';
 import type { JsonValue } from './json';
-import { type OutboxHandle, peekOutbox } from './outbox-slot';
+import { knownOffline, type OutboxHandle, peekOutbox } from './outbox-slot';
 import { OfflineQueueAbandonedError, ServerRenderLiveError } from './page-errors';
 import { installedPage, type PageWrites, pageRealtime } from './page-store';
 import { isServerRender, signalFor } from './reactivity';
@@ -125,14 +125,21 @@ export function useMutation(mutator: MutatorLike): Mutate {
     const key = `${mutator.name}:${uuidV7()}`;
     const local = mutator.local;
     // Called back through the mutator: `local` may be a method, and an unbound one loses `this`.
-    if (local !== undefined) {
+    const paint = (): void => {
+      if (local === undefined) return;
       page.store.push(key, (tx) => mutator.local?.(tx, input), mutator.conflict ?? 'server-wins');
-    }
-    count(writes, mutator.name, 1);
+    };
     // Older writes still wait in the outbox: this one queues BEHIND them rather than overtaking
     // them over HTTP — a like queued offline and the unlike made once the network was back could
     // otherwise land swapped. The replay sends the queue in order, this write last, under its key.
     const queued = peekOutbox();
+    // The browser SAYS it is offline (#777): a POST can only fail, so the write goes to the outbox
+    // without one, and reaches the disk BEFORE its twin reaches the screen. Painted first, it was
+    // durable only after the failed POST, the persister's flush and the queue write — a reload in
+    // that window came back with the old count and an empty outbox. The `online` event replays it.
+    const offline = queued !== undefined && knownOffline();
+    if (!offline) paint();
+    count(writes, mutator.name, 1);
     // The outbox may REFUSE: its queue's principal left mid-write (`X_OFFLINE_QUEUE_ABANDONED`),
     // or the disk would not take it. Nothing will ever send this write then, so its twin comes
     // off the screen and the caller is told — a rejected `mutate`, by code.
@@ -156,8 +163,9 @@ export function useMutation(mutator: MutatorLike): Mutate {
       }
     };
     // Asked of the DURABLE queue, re-read now: another tab of this user may have queued writes this
-    // tab's copy has not heard of yet, and sending past them reorders the user's intent.
-    if (queued !== undefined) {
+    // tab's copy has not heard of yet, and sending past them reorders the user's intent. Offline,
+    // nothing is sent, so the question is moot and its read would only delay the disk write.
+    if (queued !== undefined && !offline) {
       await queued.refresh().catch(() => undefined);
       // That read is an await: a principal that left meanwhile must not have this write POSTed, or
       // queued, as the next one. Refused by the code `enqueue` uses for the same departure.
@@ -167,9 +175,16 @@ export function useMutation(mutator: MutatorLike): Mutate {
         throw new OfflineQueueAbandonedError({ name: mutator.name });
       }
     }
-    if (queued !== undefined && queued.pending().length > 0) {
+    if (queued !== undefined && (offline || queued.pending().length > 0)) {
       try {
         await enqueue(queued);
+        if (offline) {
+          // Queued under the principal that made it; one that left since takes its page with it.
+          if (pageClient().scope.epoch !== issued) {
+            throw new OfflineQueueAbandonedError({ name: mutator.name });
+          }
+          paint();
+        }
         void queued.replay().catch(() => undefined);
         return undefined;
       } finally {