@ultimat3/realtime 22.15.0 → 24.0.0

package/src/offline-queue.ts

@@ -11,6 +11,7 @@
 
 import { renderThrowable, stringField } from '@ultimat3/core/page';
 import type { JsonValue } from './json';
+import { OfflineQueueAbandonedError } from './page-errors';
 import type { WireError } from './sync-protocol';
 
 export type MutationStatus = 'pending' | 'inflight' | 'acked' | 'failed';
@@ -91,6 +92,8 @@ export class OfflineQueue {
    * is parked inside `send`. Bumped by every loss so a pass that resumes afterwards claims nothing.
    */
   #epoch = 0;
+  /** Set by `abandon`, never cleared: this queue's principal is gone. */
+  #abandoned = false;
 
   private constructor(store: QueueStore, state: QueueState) {
     this.#store = store;
@@ -103,8 +106,10 @@ export class OfflineQueue {
    *
    * An `inflight` entry on disk belonged to a page that is gone, so it goes back to `pending`. Left
    * as it was, `#sendable` skipped it forever — no ack was coming to a page that no longer exists —
-   * and every later write overtook it. The replay carries its idempotency key, so a write the old
-   * page did get through is answered from the action's idempotency store, never applied twice.
+   * and every later write overtook it. The replay carries its idempotency key, and a `mutator()`
+   * is REQUIRED to declare `idempotent: true` (`X_MUTATOR_NOT_IDEMPOTENT` at declaration), so a
+   * write the old page did get through is answered from the action's idempotency store and is
+   * never applied twice. That requirement is what makes this resend safe, not the key alone.
    */
   static async open(store: QueueStore): Promise<OfflineQueue> {
     const queue = new OfflineQueue(store, await store.load());
@@ -119,6 +124,7 @@ export class OfflineQueue {
    * other pass can have one on the wire.
    */
   async reload(): Promise<void> {
+    if (this.#abandoned) return;
     const state = await this.#store.load();
     this.#mutations = state.mutations.map((mutation) => ({ ...mutation }));
     this.#nextSeq = Math.max(this.#nextSeq, state.nextSeq);
@@ -167,6 +173,17 @@ export class OfflineQueue {
     input: JsonValue;
     at?: number;
   }): Promise<QueuedMutation> {
+    // Read BEFORE the key is looked up, never between the lookup and the push: with the await in
+    // between, two enqueues of one key both found nothing and both pushed — one intent, two
+    // entries under one sequence number. Another tab of the same user may have taken sequence
+    // numbers since this one loaded, which is what the read is for.
+    //
+    // An abandoned queue REFUSES, before the read and after it (the principal can leave during
+    // it): a write accepted here would sit in memory, shown as queued, with nothing to send it.
+    if (this.#abandoned) throw new OfflineQueueAbandonedError({ name: args.name });
+    const floor = (await this.#store.load()).nextSeq;
+    if (this.#abandoned) throw new OfflineQueueAbandonedError({ name: args.name });
+    this.#nextSeq = Math.max(this.#nextSeq, floor);
     const existing = this.find(args.key);
     if (existing && existing.status !== 'failed') {
       this.#collapsed += 1;
@@ -178,8 +195,6 @@ export class OfflineQueue {
     // entry is dropped and this one takes a new sequence at the back of the queue.
     // By identity: `existing` IS the entry for this key, found above — no second key comparison.
     if (existing) this.#mutations = this.#mutations.filter((entry) => entry !== existing);
-    // Another tab of the same user may have taken sequence numbers since this one loaded.
-    this.#nextSeq = Math.max(this.#nextSeq, (await this.#store.load()).nextSeq);
     const mutation: QueuedMutation = {
       key: args.key,
       seq: this.#nextSeq,
@@ -248,6 +263,22 @@ export class OfflineQueue {
     return returned;
   }
 
+  /**
+   * The principal this queue belongs to is gone (`page-outbox.ts` calls it on a `rescope`), and it
+   * is final: nothing more is SENT, WRITTEN or ACCEPTED (`enqueue` refuses). The epoch bump stops a pass parked
+   * inside `send` before its next entry — which would otherwise leave under the next principal's
+   * session — and the flag stops a pass that starts later, and every write-back: the store was
+   * wiped for this principal, and a pass that then saved what it touched put the writes back.
+   */
+  abandon(): void {
+    this.#epoch += 1;
+    this.#abandoned = true;
+  }
+
+  get abandoned(): boolean {
+    return this.#abandoned;
+  }
+
   /** Server acknowledged: the mutation leaves the queue and its rebase entry can be committed. */
   async ack(key: string): Promise<void> {
     const mutation = this.find(key);
@@ -294,7 +325,7 @@ export class OfflineQueue {
   /** One drain pass. Never called concurrently with itself — `drain` owns that. */
   async #pass(send: MutationSender): Promise<DrainReport> {
     const epoch = this.#epoch;
-    const sendable = this.#sendable();
+    const sendable = this.#abandoned ? [] : this.#sendable();
     // Nothing to do: a pass chained behind one that already sent everything must not rewrite the
     // durable state for the privilege of reporting zero.
     if (sendable.length === 0) {
@@ -326,6 +357,12 @@ export class OfflineQueue {
         // `WebSocket.send` on a CLOSING socket discards it and returns normally — so calling that
         // an ack drops the mutation on exactly the socket death this queue exists to survive.
         // Only `ack`/`fail` (the server) or `requeueInflight` (a lost connection) moves it on.
+        //
+        // A sender may settle the write itself before it returns — the page outbox does, over
+        // HTTP. One it REFUSED keeps the error `fail` just recorded and is not counted as sent:
+        // clearing it here put `{ status: 'failed', error: null }` on disk and reported a refused
+        // write as a sent one.
+        if (statusOf(mutation) === 'failed') continue;
         mutation.error = null;
         sent += 1;
       } catch (error) {
@@ -364,6 +401,7 @@ export class OfflineQueue {
    * queue: a second tab's entries are not this tab's to overwrite or delete.
    */
   async #persist(puts: readonly QueuedMutation[], deletes: readonly string[] = []): Promise<void> {
+    if (this.#abandoned) return;
     await this.#store.write({
       puts: puts.map((mutation) => ({ ...mutation })),
       deletes,
@@ -372,6 +410,11 @@ export class OfflineQueue {
   }
 }
 
+/** Read through a call: `send` awaited in between, and the compiler still narrows to `inflight`. */
+function statusOf(mutation: QueuedMutation): MutationStatus {
+  return mutation.status;
+}
+
 /** A thrown value as the queue records it — `code`, `cause`, `fix`, never a throw of its own. */
 export function toQueueError(error: unknown): WireError {
   return {

package/src/page-errors.ts

@@ -93,6 +93,23 @@ export class LocalStoreUnavailableError extends RealtimeError {
   }
 }
 
+/**
+ * A write reached a queue whose principal is gone: the page signed out, or in as someone else,
+ * while the write was being queued. It is not stored and never sent — a write issued under one
+ * session must not leave under the next — and it is RAISED, because a user's write dropped in
+ * silence is a loss nobody is told about. `name` is the mutator's registered name; the write's
+ * idempotency key is deliberately not rendered.
+ */
+export class OfflineQueueAbandonedError extends RealtimeError {
+  constructor(args: { name: string }) {
+    super({
+      code: 'X_OFFLINE_QUEUE_ABANDONED',
+      cause: `the write "${args.name}" was queued after its page changed principal, so it was not stored and will not be sent — call the mutation again under the current session`,
+      fix: 'x errors explain X_OFFLINE_QUEUE_ABANDONED --json',
+    });
+  }
+}
+
 /**
  * Something that can only mean "talk to the socket" ran on the server client — a mutation, a
  * publish, a topic subscription, a dial. There is no socket during a server render and there never

package/src/page-outbox.ts

@@ -1,13 +1,10 @@
 /**
- * The page's ONE outbox: writes a page could not send are queued in `OfflineQueue`, persisted in
- * the page's durable store under the current principal, and replayed IN ORDER over HTTP — each to
- * `actionPath(name)` through core's `clientTransport`, each with its own idempotency key, so a
- * replay after a lost response is answered from the action's idempotency store and never applied
- * twice. Replayed on open, on `online`, and on the service worker's `OUTBOX_DRAIN_MESSAGE`.
- * A principal change wipes the previous principal's queue: its writes are never sent as the next.
+ * The page's ONE outbox: writes a page could not send, persisted under the current principal and
+ * replayed IN ORDER over HTTP, each under its own idempotency key. AT LEAST once — an entry leaves
+ * the disk when its ack is written. A principal change abandons the pass and wipes the queue.
  */
 
-import type { ClientScope } from '@ultimat3/core/page';
+import type { ClientScope, UltimateError } from '@ultimat3/core/page';
 import {
   actionPath,
   classifyThrown,
@@ -15,12 +12,14 @@ import {
   OUTBOX_DRAIN_MESSAGE,
   onRescope,
   pageClient,
+  renderThrowable,
 } from '@ultimat3/core/page';
 import type { LocalStore } from './local-store-idb';
 import { pageLocalStore, scopeKey } from './local-store-idb';
 import type { DrainReport, QueuedMutation, QueueState, QueueStore } from './offline-queue';
 import { MemoryQueueStore, OfflineQueue, toQueueError } from './offline-queue';
 import { OUTBOX_KEY, type OutboxEntry, type OutboxHandle, type OutboxHost } from './outbox-slot';
+import { LocalStoreUnavailableError, OfflineQueueAbandonedError } from './page-errors';
 import { peekPageRealtime } from './page-store';
 import { carriedBy } from './record-store';
 
@@ -63,70 +62,142 @@ export interface OutboxOptions {
    * the write queued and the post it liked missing, and showed the old count.
    */
   readonly beforeEnqueue?: (() => Promise<void>) | undefined;
+  /** Where a disk that refused the queue is said, by code. Default `console.warn`. */
+  readonly warn?: ((error: UltimateError) => void) | undefined;
 }
 
 const EMPTY: DrainReport = { sent: 0, collapsed: 0, remaining: 0, stoppedAt: null };
 
+/** One principal's open queue, and the scope the drain lock is named after. */
+interface Held {
+  readonly queue: OfflineQueue;
+  readonly scope: string | undefined;
+}
+
 export function createOutbox(options: OutboxOptions): PageOutbox {
   const principal =
     options.principal ?? ((): ClientScope['principal'] => pageClient().scope.principal);
   const send = options.send ?? sendOverHttp;
   const overlays =
     options.overlays ?? ((): OutboxOverlays | undefined => peekPageRealtime()?.store);
-  let queue: OfflineQueue | undefined;
-  /** The scope the open queue belongs to — what the drain lock is named after. */
-  let scope: string | undefined;
+  /**
+   * The open queue AND the scope it belongs to, as one value: a pass captures both when it starts,
+   * so nothing it does later can be read off a principal that arrived since.
+   */
+  let held: Held | undefined;
+  /**
+   * Bumped by every principal change, SYNCHRONOUSLY. Anything here that resolves a queue after an
+   * await compares it first: the answer to "whose queue" must be the one from before the await.
+   */
+  let epoch = 0;
+  const warn = options.warn ?? ((error: UltimateError): void => console.warn(error));
+  const unavailable = (what: string, error: unknown): void =>
+    warn(new LocalStoreUnavailableError({ reason: `${what}: ${renderThrowable(error)}` }));
 
+  /** Never rejects: a queue the disk would not open is a memory queue, said once by code. */
   const open = async (): Promise<void> => {
-    scope = scopeKey(principal());
-    queue = await OfflineQueue.open(queueStore(await options.local, scope));
-  };
-  let ready = open();
-  let running: Promise<DrainReport> | undefined;
-  /** A trigger landed while a pass ran: one more pass follows it, never one per trigger. */
-  let again = false;
-
-  const deliver = async (mutation: QueuedMutation): Promise<void> => {
-    const current = queue;
-    const carried = new Set<string>();
+    const at = epoch;
+    const scope = scopeKey(principal());
+    let queue: OfflineQueue;
     try {
-      await send({ key: mutation.key, name: mutation.name, input: mutation.input }, carried);
+      queue = await OfflineQueue.open(queueStore(await options.local, scope));
     } catch (error) {
-      const kind = classifyThrown(error);
-      // The network, or a server asking to be asked again: stays queued, and the pass stops so
-      // nothing behind it overtakes it.
-      if (kind === 'retryable' || kind === 'retry-after') throw error;
-      // Anything else is the server's decision about this write — kept for the UI, never resent.
-      await current?.fail(mutation.key, toQueueError(error));
-      overlays()?.drop(mutation.key);
+      unavailable('the outbox could not be opened', error);
+      queue = await OfflineQueue.open(new MemoryQueueStore());
+    }
+    // The principal left while the disk was being read: this queue is the one that left with it,
+    // and the handler that saw the change has already chained the next open.
+    if (at !== epoch) {
+      queue.abandon();
       return;
     }
-    await current?.ack(mutation.key);
-    const store = overlays();
-    try {
-      // Exactly as a live write settles: a row the answer did not carry keeps its overlay until
-      // the server's row for it arrives.
-      store?.settle(mutation.key, carried);
-    } catch {
-      // A custom merge that answered no row: the server's truth stands.
-      store?.drop(mutation.key);
+    held?.queue.abandon();
+    held = { queue, scope };
+  };
+  let ready = open();
+  /** The current principal's queue: `ready` is re-made by a rescope, so it is re-read until still. */
+  const current = async (): Promise<Held | undefined> => {
+    for (;;) {
+      const opening = ready;
+      await opening;
+      if (opening === ready) return held;
     }
   };
+  let running: Promise<DrainReport> | undefined;
+  /** A trigger landed while a pass ran: one more pass follows it, never one per trigger. */
+  let again = false;
+
+  const deliver =
+    (queue: OfflineQueue) =>
+    async (mutation: QueuedMutation): Promise<void> => {
+      const carried = new Set<string>();
+      try {
+        await send({ key: mutation.key, name: mutation.name, input: mutation.input }, carried);
+      } catch (error) {
+        const kind = classifyThrown(error);
+        // The network, or a server asking to be asked again: stays queued, and the pass stops so
+        // nothing behind it overtakes it. So does a queue whose principal left mid-send.
+        if (kind === 'retryable' || kind === 'retry-after' || queue.abandoned) throw error;
+        // Anything else is the server's decision about this write — kept for the UI, never resent.
+        await queue.fail(mutation.key, toQueueError(error));
+        overlays()?.drop(mutation.key);
+        return;
+      }
+      // The principal changed while this was on the wire: its store and its twins are already
+      // gone, and what is on the page now is somebody else's.
+      if (queue.abandoned) return;
+      await queue.ack(mutation.key);
+      const store = overlays();
+      try {
+        // Exactly as a live write settles: a row the answer did not carry keeps its overlay until
+        // the server's row for it arrives.
+        store?.settle(mutation.key, carried);
+      } catch {
+        // A custom merge that answered no row: the server's truth stands.
+        store?.drop(mutation.key);
+      }
+    };
 
   onRescope((_next, prev) => {
     const gone = scopeKey(prev.principal);
+    // SYNCHRONOUSLY, before anything is awaited: a pass parked inside `send` resumes into the next
+    // principal's session, and the wipe below reaches the disk but never the pass. Abandoned, it
+    // sends nothing more and writes nothing back.
+    held?.queue.abandon();
+    held = undefined;
+    epoch += 1;
+    // The single-flight slot is the principal's too: left set, the next principal's trigger JOINED
+    // the abandoned pass — handed its report, another principal's key in it — and its own queue
+    // was not drained until some later trigger. `settled` only clears a slot that is still its own.
+    running = undefined;
+    again = false;
     ready = ready.then(async () => {
-      if (gone !== undefined) await (await options.local).wipe(gone);
+      try {
+        if (gone !== undefined) await (await options.local).wipe(gone);
+      } catch (error) {
+        // Still on disk, under the scope that left: unreachable from this principal, and the next
+        // boot's `wipeOthers` takes it. Said, never thrown — a rejection here used to poison the
+        // chain, so every later enqueue, replay and rescope on this tab failed with it.
+        unavailable("the previous principal's outbox could not be wiped", error);
+      }
       await open();
     });
   });
 
   const self: PageOutbox = {
     enqueue: async (entry) => {
+      // WHOSE queue is decided now, before anything is awaited: resolved after the flush below,
+      // a write issued under one principal and parked there was queued — and sent — as the next.
+      // A queue still opening is waited for, and belongs to this call only if nobody left since.
+      const at = epoch;
+      const mine = held ?? (await current());
+      if (mine === undefined || at !== epoch) {
+        throw new OfflineQueueAbandonedError({ name: entry.name });
+      }
       // A disk that refused the rows must not also cost the write: the intent still goes on disk.
       await options.beforeEnqueue?.().catch(() => undefined);
-      await ready;
-      await queue?.enqueue(entry);
+      // Abandoned meanwhile, it refuses by the same code.
+      await mine.queue.enqueue(entry);
     },
     replay: () => {
       // Single flight: a trigger that lands while a replay is running JOINS it. Open, `online`,
@@ -140,20 +211,19 @@ export function createOutbox(options: OutboxOptions): PageOutbox {
         return running;
       }
       const pass = (async (): Promise<DrainReport> => {
-        await ready;
-        if (queue === undefined) return EMPTY;
+        const now = await current();
+        if (now === undefined) return EMPTY;
         // Checked when the pass STARTS, whoever asked (the socket's reconnect asks too): an
         // attempt the browser already knows cannot leave is a failed request on the wire and
         // nothing more. `online` asks again.
-        if (knownOffline()) return { ...EMPTY, remaining: queue.pending().length };
-        const draining = queue;
+        if (knownOffline()) return { ...EMPTY, remaining: now.queue.pending().length };
         // One tab drains a principal's outbox at a time, and it drains what EVERY tab queued: the
         // queue is re-read under the lock, so a write another tab made since this one opened is
         // sent too, and an `inflight` entry — which only a pass holding this lock could have put
         // on the wire, and it is over — goes back to `pending`.
-        return await exclusive(`ultimate-outbox:${scope ?? 'memory'}`, async () => {
-          await draining.reload();
-          return await draining.drain(deliver);
+        return await exclusive(`ultimate-outbox:${now.scope ?? 'memory'}`, async () => {
+          await now.queue.reload();
+          return await now.queue.drain(deliver(now.queue));
         });
       })();
       const settled = (report?: DrainReport): void => {
@@ -172,9 +242,10 @@ export function createOutbox(options: OutboxOptions): PageOutbox {
       return pass;
     },
     get size(): number {
-      return queue?.pending().length ?? 0;
+      return held?.queue.pending().length ?? 0;
     },
-    pending: () => (queue?.pending() ?? []).map(({ key, name, input }) => ({ key, name, input })),
+    pending: () =>
+      (held?.queue.pending() ?? []).map(({ key, name, input }) => ({ key, name, input })),
     get ready(): Promise<void> {
       return ready;
     },

package/src/pg-advisory-lock.ts

@@ -3,11 +3,11 @@
 // is scoped to this connection's Postgres *session*, so acquiring means keeping the connection
 // open and releasing means closing it: no lease, no renewal, no fencing token, ever.
 
+import type { AdvisoryLock } from './advisory-lock';
 import { ReplicationFailedError } from './errors';
 import { PgConnection, type PgRows } from './pg-connection';
 import { bunPgStream, type PgTarget, parsePgUrl } from './pg-socket';
 import type { PgStream } from './pg-wire';
-import type { AdvisoryLock } from './replicator';
 import type { Rng } from './thundering-herd';
 
 /** A key reaches a simple query unparameterised, so its charset is the injection boundary. */
@@ -45,6 +45,7 @@ export class PgAdvisoryLock implements AdvisoryLock {
    * the process dies. A standby then never takes over a slot whose owner has already stopped.
    */
   #acquiring: Promise<boolean> | null = null;
+  readonly #lostListeners = new Set<(reason: string) => void>();
 
   constructor(options: PgAdvisoryLockOptions) {
     if (!KEY_PATTERN.test(options.key)) {
@@ -91,6 +92,7 @@ export class PgAdvisoryLock implements AdvisoryLock {
       database: target.database,
       applicationName: `ultimate-replicator-lock:${this.key}`,
       rng: this.#options.rng,
+      ssl: target.ssl,
     });
     let rows: PgRows;
     try {
@@ -110,9 +112,51 @@ export class PgAdvisoryLock implements AdvisoryLock {
     // `release()` — which is exactly what lets a crashed replicator free the slot with nothing
     // left to clean up.
     this.#connection = connection;
+    // The session is idle from here on, and an idle session has no reader: without this a server
+    // that ended it was never noticed, and `tryAcquire()` went on answering `true` from memory
+    // for a lock Postgres had already handed to another process.
+    connection.watchIdle((reason) => this.#lost(connection, reason));
     return true;
   }
 
+  /**
+   * Drop the session without the unlock statement. The socket is closed outright — no goodbye is
+   * written, because a write to a black-holed socket is one more thing that never settles — and
+   * Postgres releases the lock when the session ends. An acquisition still in flight is dropped
+   * the moment it lands.
+   */
+  abandon(): void {
+    const acquiring = this.#acquiring;
+    if (acquiring !== null) {
+      // Voided: its own caller has the outcome; this only makes sure nothing stays held after it.
+      void acquiring.then(
+        () => this.abandon(),
+        () => undefined,
+      );
+      return;
+    }
+    const connection = this.#connection;
+    if (connection === null) return;
+    this.#connection = null;
+    connection.destroy();
+  }
+
+  onLost(listener: (reason: string) => void): () => void {
+    this.#lostListeners.add(listener);
+    return () => {
+      this.#lostListeners.delete(listener);
+    };
+  }
+
+  /** The session ended underneath the lock. A release that got there first owns the goodbye. */
+  #lost(connection: PgConnection, reason: string): void {
+    if (this.#connection !== connection) return;
+    this.#connection = null;
+    // The socket is already gone: nothing is said to it, and the listeners hear first.
+    connection.destroy();
+    for (const listener of [...this.#lostListeners]) listener(reason);
+  }
+
   async release(): Promise<void> {
     // A release that arrives mid-acquisition waits it out. `#connection` is still `null` there, so
     // the early return below would answer "nothing to release" and the session would open behind

package/src/pg-auth.ts

@@ -63,8 +63,10 @@ class ScramSha256Session implements ScramSession {
     if (!serverNonce.startsWith(this.#clientNonce)) {
       throw new ReplicationProtocolError({
         stage: 'auth',
-        detail: `server nonce "${serverNonce}" does not extend client nonce "${this.#clientNonce}"`,
-        fix: 'x doctor db — the server did not echo the client nonce; check for a proxy on the replication URL',
+        detail:
+          `server nonce "${serverNonce}" does not extend client nonce "${this.#clientNonce}" — ` +
+          'the server did not echo the client nonce; check for a proxy on the replication URL',
+        fix: 'x doctor --json',
       });
     }
 
@@ -216,8 +218,11 @@ function parseServerFirst(text: string): ServerFirst {
   if (iterations > MAX_SCRAM_ITERATIONS) {
     throw new ReplicationProtocolError({
       stage: 'auth',
-      detail: `server-first-message asked for ${iterationsText} PBKDF2 iterations, above the ${MAX_SCRAM_ITERATIONS} ceiling`,
-      fix: 'x doctor db — point the replication URL at postgres itself; a real server asks for 4096',
+      detail:
+        `server-first-message asked for ${iterationsText} PBKDF2 iterations, above the ` +
+        `${MAX_SCRAM_ITERATIONS} ceiling — a real server asks for 4096, so point the replication ` +
+        'URL at postgres itself',
+      fix: 'x doctor --json',
     });
   }
   const salt = decodeBase64('salt', saltB64);

package/src/pg-bytes.ts

@@ -129,7 +129,10 @@ export class ByteWriter {
   }
 
   uint8(value: number): this {
-    this.#bytes[this.#room(1)] = value & 0xff;
+    // The offset FIRST: `#room` may replace `#bytes`, and `this.#bytes[this.#room(1)]` reads the
+    // old buffer before it does — the byte that caused a growth was written into the discard.
+    const at = this.#room(1);
+    this.#bytes[at] = value & 0xff;
     return this;
   }