@ultimat3/realtime 21.0.0 → 22.0.0

package/src/nats-jetstream.ts

@@ -181,8 +181,16 @@ export async function kvGet(
 }
 
 /**
- * Every current value under a wildcard, in one request. A batch direct read answers with the
- * messages and then an empty `204 EOB`; a prefix nobody has written answers `404` and nothing else.
+ * Every current value under a wildcard. A batch direct read answers at most `batch` messages and
+ * then an empty `204 EOB`; a range nobody has written answers `404` and nothing else.
+ *
+ * PAGED, on the last sequence read, and by `next_by_subj` rather than `multi_last`. It was one
+ * `multi_last` batch, and that fails twice past 1,000 members: the batch truncated the set — and
+ * `sweep()`, which differences the full set, announced a `leave` for every member past the cut —
+ * and a real nats-server (2.11, measured) refuses a `multi_last` matching more than 1,024 subjects
+ * outright with `413 Too Many Results`, which the old loop skipped as a marker and returned an
+ * EMPTY set. The bucket keeps one message per subject (`max_msgs_per_subject: 1`), so walking every
+ * message under the filter IS the last value per key. A page shorter than `batch` is the last.
  */
 export async function kvLast(
   client: NatsClient,
@@ -191,19 +199,33 @@ export async function kvLast(
   batch = 1_000,
 ): Promise<readonly KvRecord[]> {
   const subject = `$JS.API.DIRECT.GET.${kvStream(bucket)}`;
-  const body = { multi_last: [kvSubject(bucket, filter)], batch };
-  const replies = await client.requestMany(subject, encoder.encode(JSON.stringify(body)), {
-    until: (message) => message.status === STATUS_EOB || message.status === STATUS_NOT_FOUND,
-  });
-  const records: KvRecord[] = [];
-  for (const reply of replies) {
-    // A status on a batch reply is a marker, never a value — the terminator is filtered by `until`,
-    // and anything else the server slips in (a `408` heartbeat) carries no message to read.
-    if (reply.status !== 0) continue;
-    const record = recordOf(reply, bucket);
-    if (record) records.push(record);
+  const byKey = new Map<string, KvRecord>();
+  let from = 1;
+  for (;;) {
+    const body = { seq: from, next_by_subj: kvSubject(bucket, filter), batch };
+    const replies = await client.requestMany(subject, encoder.encode(JSON.stringify(body)), {
+      until: (message) => message.status === STATUS_EOB || message.status === STATUS_NOT_FOUND,
+    });
+    let last = 0;
+    let read = 0;
+    for (const reply of replies) {
+      // The terminators are filtered by `until`. Any OTHER status — a `408` timeout, a `503` — is
+      // a batch that did not finish, and reading it as the whole set is the truncation above.
+      if (reply.status !== 0) {
+        throw new TransportUnavailableError({
+          transport: 'nats',
+          reason: `${subject} answered status ${reply.status} mid-batch, so the set read was incomplete`,
+        });
+      }
+      read += 1;
+      const seq = Number.parseInt(reply.header('Nats-Sequence') ?? '', 10);
+      if (Number.isSafeInteger(seq) && seq > last) last = seq;
+      const record = recordOf(reply, bucket);
+      if (record) byKey.set(record.key, record);
+    }
+    if (read < batch || last === 0) return [...byKey.values()];
+    from = last + 1;
   }
-  return records;
 }
 
 /** A KV write is a publish that waits for JetStream's ack — a lost put must not read as stored. */

package/src/nats-transport.ts

@@ -18,7 +18,7 @@ import { parseNatsUrl } from './nats-client';
 import { ensureKvBucket } from './nats-jetstream';
 import { NatsKvSet } from './nats-kv';
 import { openNatsClient } from './nats-lib-client';
-import { type BackoffPolicy, backoffDelay, defaultBackoff, type Rng } from './thundering-herd';
+import { type BackoffPolicy, defaultBackoff, policyDelay, type Rng } from './thundering-herd';
 
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
@@ -156,7 +156,7 @@ export class NatsTransport implements Transport {
       maxReconnectAttempts: this.#attempts,
       // The library retries; the spread is ours, so a cluster restart does not bring every node
       // back on the same millisecond.
-      reconnectDelay: () => backoffDelay(this.#retries++, this.#backoff, this.#rng),
+      reconnectDelay: () => policyDelay(this.#backoff, ++this.#retries, this.#rng),
       onError: (error) => this.#report(error, this.name),
       onReconnect: () => this.#recovered(),
     });

package/src/offline-queue.ts

@@ -33,21 +33,38 @@ export interface QueueState {
   readonly nextSeq: number;
 }
 
+/**
+ * One change to the durable queue: these entries written BY KEY, these keys deleted, and the
+ * sequence floor. Never the whole queue — two tabs of one user share the store, and a whole-queue
+ * save let the last tab to save erase the other's queued write.
+ */
+export interface QueueChange {
+  readonly puts: readonly QueuedMutation[];
+  readonly deletes: readonly string[];
+  readonly nextSeq: number;
+}
+
 /** Durability seam: IndexedDB in the browser (`page-outbox.ts`), memory in tests. */
 export interface QueueStore {
   load(): Promise<QueueState>;
-  save(state: QueueState): Promise<void>;
+  write(change: QueueChange): Promise<void>;
 }
 
 export class MemoryQueueStore implements QueueStore {
-  #state: QueueState = { mutations: [], nextSeq: 1 };
+  readonly #mutations = new Map<string, QueuedMutation>();
+  #nextSeq = 1;
 
   async load(): Promise<QueueState> {
-    return this.#state;
+    return {
+      mutations: [...this.#mutations.values()].map((m) => ({ ...m })),
+      nextSeq: this.#nextSeq,
+    };
   }
 
-  async save(state: QueueState): Promise<void> {
-    this.#state = { mutations: state.mutations.map((m) => ({ ...m })), nextSeq: state.nextSeq };
+  async write(change: QueueChange): Promise<void> {
+    for (const key of change.deletes) this.#mutations.delete(key);
+    for (const mutation of change.puts) this.#mutations.set(mutation.key, { ...mutation });
+    this.#nextSeq = Math.max(this.#nextSeq, change.nextSeq);
   }
 }
 
@@ -81,9 +98,37 @@ export class OfflineQueue {
     this.#nextSeq = state.nextSeq;
   }
 
-  /** Rehydrates from durable storage, so a reload resumes the same queue with the same sequence. */
+  /**
+   * Rehydrates from durable storage, so a reload resumes the same queue with the same sequence.
+   *
+   * An `inflight` entry on disk belonged to a page that is gone, so it goes back to `pending`. Left
+   * as it was, `#sendable` skipped it forever — no ack was coming to a page that no longer exists —
+   * and every later write overtook it. The replay carries its idempotency key, so a write the old
+   * page did get through is answered from the action's idempotency store, never applied twice.
+   */
   static async open(store: QueueStore): Promise<OfflineQueue> {
-    return new OfflineQueue(store, await store.load());
+    const queue = new OfflineQueue(store, await store.load());
+    await queue.#reclaimInflight();
+    return queue;
+  }
+
+  /**
+   * Re-reads the durable queue, which another tab of the same user may have written since this one
+   * opened. Call ONLY while this queue is the one draining — `page-outbox.ts` holds a Web Lock for
+   * exactly that — because it reclaims every `inflight` entry as `pending`: with the lock held, no
+   * other pass can have one on the wire.
+   */
+  async reload(): Promise<void> {
+    const state = await this.#store.load();
+    this.#mutations = state.mutations.map((mutation) => ({ ...mutation }));
+    this.#nextSeq = Math.max(this.#nextSeq, state.nextSeq);
+    await this.#reclaimInflight();
+  }
+
+  async #reclaimInflight(): Promise<void> {
+    const reclaimed = this.#mutations.filter((mutation) => mutation.status === 'inflight');
+    for (const mutation of reclaimed) mutation.status = 'pending';
+    if (reclaimed.length > 0) await this.#persist(reclaimed);
   }
 
   get size(): number {
@@ -133,6 +178,8 @@ export class OfflineQueue {
     // entry is dropped and this one takes a new sequence at the back of the queue.
     // By identity: `existing` IS the entry for this key, found above — no second key comparison.
     if (existing) this.#mutations = this.#mutations.filter((entry) => entry !== existing);
+    // Another tab of the same user may have taken sequence numbers since this one loaded.
+    this.#nextSeq = Math.max(this.#nextSeq, (await this.#store.load()).nextSeq);
     const mutation: QueuedMutation = {
       key: args.key,
       seq: this.#nextSeq,
@@ -145,7 +192,7 @@ export class OfflineQueue {
     };
     this.#nextSeq += 1;
     this.#mutations.push(mutation);
-    await this.#persist();
+    await this.#persist([mutation]);
     return mutation;
   }
 
@@ -190,12 +237,14 @@ export class OfflineQueue {
   async requeueInflight(): Promise<number> {
     this.#epoch += 1;
     let returned = 0;
+    const back: QueuedMutation[] = [];
     for (const mutation of this.#mutations) {
       if (mutation.status !== 'inflight') continue;
       mutation.status = 'pending';
       returned += 1;
+      back.push(mutation);
     }
-    if (returned > 0) await this.#persist();
+    if (returned > 0) await this.#persist(back);
     return returned;
   }
 
@@ -206,7 +255,7 @@ export class OfflineQueue {
     mutation.status = 'acked';
     // By identity: `find` already matched it, and a second comparison of the key is the same question.
     this.#mutations = this.#mutations.filter((entry) => entry !== mutation);
-    await this.#persist();
+    await this.#persist([], [key]);
   }
 
   /** Terminal failure (policy denial, validation): kept for the UI, never retried blindly. */
@@ -215,7 +264,7 @@ export class OfflineQueue {
     if (!mutation) return;
     mutation.status = 'failed';
     mutation.error = error;
-    await this.#persist();
+    await this.#persist([mutation]);
   }
 
   /**
@@ -252,6 +301,7 @@ export class OfflineQueue {
       return { sent: 0, collapsed: this.#collapsed, remaining: 0, stoppedAt: null };
     }
     let sent = 0;
+    const touched: QueuedMutation[] = [];
     for (const mutation of sendable) {
       // The connection this pass was draining into is gone, and `requeueInflight` has already
       // handed back what was on it. Everything left stays `pending` for the pass the next
@@ -269,6 +319,7 @@ export class OfflineQueue {
       if (!this.#stillSendable(mutation)) continue;
       mutation.status = 'inflight';
       mutation.attempts += 1;
+      touched.push(mutation);
       try {
         await send(mutation);
         // Stays `inflight`. `send` resolving means the frame reached a socket — a browser
@@ -280,7 +331,7 @@ export class OfflineQueue {
       } catch (error) {
         mutation.status = 'pending';
         mutation.error = toQueueError(error);
-        await this.#persist();
+        await this.#persist([mutation]);
         return {
           sent,
           collapsed: this.#collapsed,
@@ -289,7 +340,9 @@ export class OfflineQueue {
         };
       }
     }
-    await this.#persist();
+    // Only what this pass touched AND the queue still holds: an entry the server settled during the
+    // pass was already deleted by `ack`, and writing it back would resurrect it.
+    await this.#persist(touched.filter((mutation) => this.#mutations.includes(mutation)));
     return {
       sent,
       collapsed: this.#collapsed,
@@ -299,19 +352,21 @@ export class OfflineQueue {
   }
 
   async clear(): Promise<void> {
+    const keys = this.#mutations.map((mutation) => mutation.key);
     this.#mutations = [];
-    await this.#persist();
+    await this.#persist([], keys);
   }
 
   /**
-   * A snapshot, never the live entries. `save` is a durable write — OPFS, IndexedDB — and it is
-   * allowed to await before it reads. Handed the array itself, a store that resolves after the next
-   * pass has moved on persists a status that was never true when it was called; `inflight` is the
-   * one a reload cannot recover from, because `#sendable` skips it and no ack is coming.
+   * Snapshots, never the live entries. `write` is a durable write — IndexedDB — and it is allowed
+   * to await before it reads. Handed an entry itself, a store that resolves after the next pass has
+   * moved on persists a status that was never true when it was called. BY KEY, never the whole
+   * queue: a second tab's entries are not this tab's to overwrite or delete.
    */
-  async #persist(): Promise<void> {
-    await this.#store.save({
-      mutations: this.#mutations.map((mutation) => ({ ...mutation })),
+  async #persist(puts: readonly QueuedMutation[], deletes: readonly string[] = []): Promise<void> {
+    await this.#store.write({
+      puts: puts.map((mutation) => ({ ...mutation })),
+      deletes,
       nextSeq: this.#nextSeq,
     });
   }

package/src/page-outbox.ts

@@ -74,12 +74,17 @@ export function createOutbox(options: OutboxOptions): PageOutbox {
   const overlays =
     options.overlays ?? ((): OutboxOverlays | undefined => peekPageRealtime()?.store);
   let queue: OfflineQueue | undefined;
+  /** The scope the open queue belongs to — what the drain lock is named after. */
+  let scope: string | undefined;
 
   const open = async (): Promise<void> => {
-    queue = await OfflineQueue.open(queueStore(await options.local, scopeKey(principal())));
+    scope = scopeKey(principal());
+    queue = await OfflineQueue.open(queueStore(await options.local, scope));
   };
   let ready = open();
   let running: Promise<DrainReport> | undefined;
+  /** A trigger landed while a pass ran: one more pass follows it, never one per trigger. */
+  let again = false;
 
   const deliver = async (mutation: QueuedMutation): Promise<void> => {
     const current = queue;
@@ -116,7 +121,7 @@ export function createOutbox(options: OutboxOptions): PageOutbox {
     });
   });
 
-  return {
+  const self: PageOutbox = {
     enqueue: async (entry) => {
       // A disk that refused the rows must not also cost the write: the intent still goes on disk.
       await options.beforeEnqueue?.().catch(() => undefined);
@@ -127,9 +132,13 @@ export function createOutbox(options: OutboxOptions): PageOutbox {
       // Single flight: a trigger that lands while a replay is running JOINS it. Open, `online`,
       // the socket's reconnect and the service worker's drain arrive together, and each chaining
       // a pass of its own sent the head of the queue once per trigger whenever a send failed —
-      // one write, several POSTs. What a joined trigger would have sent is still queued for the
-      // next one; nothing is dropped.
-      if (running !== undefined) return running;
+      // one write, several POSTs. What the running pass could not have seen — a write queued
+      // after it re-read the store — gets exactly ONE follow-up pass, however many triggers
+      // joined: `again` is a flag, never a count.
+      if (running !== undefined) {
+        again = true;
+        return running;
+      }
       const pass = (async (): Promise<DrainReport> => {
         await ready;
         if (queue === undefined) return EMPTY;
@@ -137,13 +146,29 @@ export function createOutbox(options: OutboxOptions): PageOutbox {
         // attempt the browser already knows cannot leave is a failed request on the wire and
         // nothing more. `online` asks again.
         if (knownOffline()) return { ...EMPTY, remaining: queue.pending().length };
-        return queue.drain(deliver);
+        const draining = queue;
+        // One tab drains a principal's outbox at a time, and it drains what EVERY tab queued: the
+        // queue is re-read under the lock, so a write another tab made since this one opened is
+        // sent too, and an `inflight` entry — which only a pass holding this lock could have put
+        // on the wire, and it is over — goes back to `pending`.
+        return await exclusive(`ultimate-outbox:${scope ?? 'memory'}`, async () => {
+          await draining.reload();
+          return await draining.drain(deliver);
+        });
       })();
-      const settled = (): void => {
-        if (running === pass) running = undefined;
+      const settled = (report?: DrainReport): void => {
+        if (running !== pass) return;
+        running = undefined;
+        // Only after a pass that reached the end: one stopped by a failure leaves the rest for
+        // the next trigger, as it always did.
+        if (again && report !== undefined && report.stoppedAt === null) {
+          again = false;
+          void self.replay().catch(() => undefined);
+        }
+        again = false;
       };
       running = pass;
-      pass.then(settled, settled);
+      pass.then(settled, () => settled());
       return pass;
     },
     get size(): number {
@@ -154,6 +179,7 @@ export function createOutbox(options: OutboxOptions): PageOutbox {
       return ready;
     },
   };
+  return self;
 }
 
 /** Persisted under the principal; an UNSCOPED page queues in memory only, and loses it on reload. */
@@ -162,10 +188,54 @@ function queueStore(local: LocalStore, scope: string | undefined): QueueStore {
   return {
     load: async (): Promise<QueueState> =>
       (await local.queue(scope)) ?? { mutations: [], nextSeq: 1 },
-    save: (state) => local.saveQueue(scope, state),
+    write: (change) => local.writeQueue(scope, change),
   };
 }
 
+/** The slice of the Web Locks API a drain needs. */
+interface LockManagerLike {
+  request<T>(name: string, callback: () => Promise<T>): Promise<T>;
+}
+
+/**
+ * `work` under the browser's Web Lock named `name`, so two tabs of one user never drain one outbox
+ * at once — they would each send the head of the queue. With no `navigator.locks` the same
+ * exclusion is kept inside this realm, on a chain held on `globalThis` (every island bundle carries
+ * its own copy of this module): tabs cannot be excluded there, but two outboxes in one page can.
+ */
+function exclusive<T>(name: string, work: () => Promise<T>): Promise<T> {
+  const navigator: unknown = Reflect.get(globalThis, 'navigator');
+  const locks: unknown =
+    typeof navigator === 'object' && navigator !== null
+      ? Reflect.get(navigator, 'locks')
+      : undefined;
+  if (
+    typeof locks === 'object' &&
+    locks !== null &&
+    typeof Reflect.get(locks, 'request') === 'function'
+  ) {
+    return (locks as LockManagerLike).request(name, work);
+  }
+  const host = globalThis as LockHost;
+  const chains = host[LOCAL_LOCKS] ?? new Map<string, Promise<unknown>>();
+  if (host[LOCAL_LOCKS] === undefined) {
+    Object.defineProperty(host, LOCAL_LOCKS, { value: chains, configurable: true });
+  }
+  const ahead = chains.get(name) ?? Promise.resolve();
+  const turn = ahead.then(work, work);
+  const tail = turn.then(
+    () => undefined,
+    () => undefined,
+  );
+  chains.set(name, tail);
+  void tail.then(() => {
+    if (chains.get(name) === tail) chains.delete(name);
+  });
+  return turn;
+}
+
+const LOCAL_LOCKS: unique symbol = Symbol.for('ultimate.outbox-locks');
+type LockHost = { [LOCAL_LOCKS]?: Map<string, Promise<unknown>> };
 function sendOverHttp(entry: OutboxEntry, carried: Set<string>): Promise<unknown> {
   return clientTransport({
     method: 'POST',

package/src/page-socket.ts

@@ -5,10 +5,17 @@
 import { onRescope, pageClient } from '@ultimat3/core/page';
 import { queryClientMethodFor } from '@ultimat3/query/client';
 import { LiveClient } from './client';
+import { DEFAULT_HEARTBEAT_MS } from './client-heartbeat';
 import { peekOutbox } from './outbox-slot';
 import { SyncUnconfiguredError } from './page-errors';
 import { pageRealtime } from './page-store';
-import { openHost, type SocketHost, type SocketHostOptions } from './socket-host';
+import {
+  openHost,
+  type RehostingHost,
+  rehosting,
+  type SocketHost,
+  type SocketHostOptions,
+} from './socket-host';
 import { pageSyncTarget, syncWorkerFromMeta } from './sync-meta';
 
 /** Get-or-create, and connect on creation. `hook` names the caller in the refusal. */
@@ -20,7 +27,7 @@ export function pageSocket(hook: string): LiveClient {
   // What the bootstrap passed, else what the document shell rendered into `<head>`.
   const target = page.sync ?? pageSyncTarget();
   if (target === undefined) throw new SyncUnconfiguredError({ hook });
-  let host = hostFor(pageClient().scope.principal ?? null);
+  let host = rehostingFor(pageClient().scope.principal ?? null, target.buildId);
   const client = new LiveClient({
     connect: () => host.socket(target),
     buildId: target.buildId,
@@ -46,15 +53,31 @@ export function pageSocket(hook: string): LiveClient {
   // A new principal gets its own worker — never the previous principal's socket — and redials.
   const offRescope = onRescope((next) => {
     host.bye();
-    host = hostFor(next.principal ?? null);
+    host = rehostingFor(next.principal ?? null, target.buildId);
     client.connect();
   });
-  const bye = (): void => host.bye();
+  // `bye` only for a page that is really going: a `pagehide` into the back/forward cache is a page
+  // that may come back, and one that said bye came back to a port the engine had released.
+  const bye = (event: Event): void => {
+    if (Reflect.get(event, 'persisted') !== true) host.bye();
+  };
+  // ...and a page restored from that cache re-hosts rather than trusting the port it left with.
+  const restored = (event: Event): void => {
+    if (Reflect.get(event, 'persisted') !== true) return;
+    host.rehost();
+    client.connect();
+  };
   const listens = typeof addEventListener === 'function';
-  if (listens) addEventListener('pagehide', bye);
+  if (listens) {
+    addEventListener('pagehide', bye);
+    addEventListener('pageshow', restored);
+  }
   teardowns.add(() => {
     offRescope();
-    if (listens) removeEventListener('pagehide', bye);
+    if (listens) {
+      removeEventListener('pagehide', bye);
+      removeEventListener('pageshow', restored);
+    }
     client.close();
     host.bye();
     if (page.socket === client) page.socket = undefined;
@@ -99,10 +122,18 @@ export function hasPageSocket(): boolean {
   );
 }
 
-function hostFor(scope: string | null): SocketHost {
+function hostFor(scope: string | null, buildId: string): SocketHost {
   const workerUrl =
     typeof document === 'undefined' || typeof location === 'undefined'
       ? undefined
       : syncWorkerFromMeta(document, location.href);
-  return hosts({ workerUrl, scope });
+  return hosts({ workerUrl, scope, buildId });
+}
+
+/**
+ * The host for one principal, re-made when an `open` goes unanswered for two heartbeats — a port
+ * the engine reaped, or a worker that died — rather than leaving the tab dialling a dead port.
+ */
+function rehostingFor(scope: string | null, buildId: string): RehostingHost {
+  return rehosting(() => hostFor(scope, buildId), { openTimeoutMs: 2 * DEFAULT_HEARTBEAT_MS });
 }