@ultimat3/realtime 23.0.0 → 25.0.0

package/src/nats-transport.ts

@@ -1,7 +1,8 @@
 // Single responsibility: the production `Transport` — core NATS for fanout, JetStream KV for the
 // shared presence sets. The client underneath owns the wire and the reconnect, including
-// re-establishing subscriptions, which is what makes a `sync` node stateless: a lost connection is
-// re-dialled and re-subscribed underneath the caller, and this file keeps no socket state at all.
+// re-establishing subscriptions, which is what makes a `sync` node stateless. The one loss it does
+// not own is its own end: a client closed for good once its reconnect budget is spent is replaced
+// here, on our backoff, and every kept subscription is bound again on the new one.
 
 import {
   type Clock,
@@ -18,6 +19,7 @@ import { parseNatsUrl } from './nats-client';
 import { ensureKvBucket } from './nats-jetstream';
 import { NatsKvSet } from './nats-kv';
 import { openNatsClient } from './nats-open';
+import { NatsSubscriptions } from './nats-subscriptions';
 import { type BackoffPolicy, defaultBackoff, policyDelay, type Rng } from './thundering-herd';
 
 const encoder = new TextEncoder();
@@ -55,6 +57,11 @@ export class NatsTransport implements Transport {
   #dialing: Promise<NatsClient> | undefined;
   #retries = 0;
   #closed = false;
+  /** Set while a client the library gave up on is being replaced; the attempt in flight. */
+  #redialing: number | undefined;
+  #wakeRedial: (() => void) | undefined;
+  readonly #subscriptions = new NatsSubscriptions();
+  readonly #reconnectListeners = new Set<() => void>();
 
   constructor(options: NatsTransportOptions) {
     // Parsed here rather than at the first publish: a malformed NATS_URL is a boot-time fault, and
@@ -97,27 +104,44 @@ export class NatsTransport implements Transport {
   }
 
   /**
-   * The subscription is the client's to keep: it survives a drop and comes back with the reconnect,
-   * so there is no intent map here to re-bind from — and therefore no way for a re-bind to run
-   * twice and double every change on the subject.
+   * A drop is the client's to recover: the subscription comes back with the library's reconnect,
+   * untouched by this file. It is also KEPT here, for the one loss the library does not recover —
+   * a client closed for good — and bound once on its replacement, never twice on one client.
    */
   async subscribe(subject: string, handler: TransportHandler): Promise<TransportSubscription> {
     const client = await this.#ensure();
     // Same seam as `publish`: a permissions violation on the subject is refused here, not later.
-    const live = this.#translating(`subscribe to ${subject}`, () =>
-      client.subscribe(subject, (message) => {
+    const release = this.#subscriptions.add(
+      client,
+      subject,
+      (message) => {
         try {
           handler(decoder.decode(message.payload), message.subject);
         } catch (error) {
           this.#report(error, message.subject);
         }
-      }),
+      },
+      (call) => this.#translating(`subscribe to ${subject}`, call),
     );
-    return { subject, unsubscribe: () => live.unsubscribe() };
+    return { subject, unsubscribe: release };
+  }
+
+  /**
+   * Told every time the client comes back from a drop. Fanout is at-most-once, so the gap between
+   * the drop and this call is changes the subscriber never saw — and with no later message there
+   * is no sequence number to notice it by. A `sync` node invalidates its windows on it. Returns
+   * the unsubscribe.
+   */
+  onReconnect(listener: () => void): () => void {
+    this.#reconnectListeners.add(listener);
+    return () => {
+      this.#reconnectListeners.delete(listener);
+    };
   }
 
   async close(): Promise<void> {
     this.#closed = true;
+    this.#wakeRedial?.();
     const client = this.#client;
     this.#client = undefined;
     await client?.close();
@@ -127,7 +151,9 @@ export class NatsTransport implements Transport {
    * One dial, shared by every caller that races it. A client that has already been handed out is
    * reused whatever its state: while it is reconnecting the library is re-establishing that same
    * connection and its subscriptions, and a second dial alongside it would double every delivery.
-   * A budget that ran out is a readiness failure, not a reason to start an unbounded retry here.
+   * A client closed for good is not handed out: `#lost` clears it and `#redial` replaces it, and a
+   * caller that lands meanwhile is refused at once rather than parked behind a dial that may take
+   * the library's whole connect budget.
    */
   #ensure(): Promise<NatsClient> {
     if (this.#closed) {
@@ -137,6 +163,19 @@ export class NatsTransport implements Transport {
     }
     const current = this.#client;
     if (current !== undefined) return Promise.resolve(current);
+    if (this.#redialing !== undefined) {
+      return Promise.reject(
+        new TransportUnavailableError({
+          transport: this.name,
+          reason: `the connection closed once its reconnect budget was spent, and re-dial attempt ${this.#redialing} is in progress`,
+        }),
+      );
+    }
+    return this.#dialOnce();
+  }
+
+  /** One dial, shared by every caller that races it. */
+  #dialOnce(): Promise<NatsClient> {
     this.#dialing ??= this.#dial().finally(() => {
       this.#dialing = undefined;
     });
@@ -150,6 +189,10 @@ export class NatsTransport implements Transport {
    * leaking one per retry.
    */
   async #dial(): Promise<NatsClient> {
+    // The client is only known once the dial resolves, and the library may give up on it before
+    // the bucket is asserted: both cases are caught by identity, never by a flag on the transport.
+    let opened: NatsClient | undefined;
+    let closedEarly = false;
     const client = await this.#connect({
       url: this.#options.url,
       name: 'ultimate',
@@ -159,9 +202,19 @@ export class NatsTransport implements Transport {
       reconnectDelay: () => policyDelay(this.#backoff, ++this.#retries, this.#rng),
       onError: (error) => this.#report(error, this.name),
       onReconnect: () => this.#recovered(),
+      onClosed: () => {
+        if (opened === undefined) closedEarly = true;
+        else this.#lost(opened);
+      },
     });
     try {
       await this.#ensureBucket(client);
+      if (closedEarly) {
+        throw new TransportUnavailableError({
+          transport: this.name,
+          reason: 'the connection closed for good while the KV bucket was being asserted',
+        });
+      }
       // `close()` can land while a dial is in flight, and it only closes what it can see:
       // publishing now would leave a connection open that nothing will ever close again.
       if (this.#closed) {
@@ -174,11 +227,76 @@ export class NatsTransport implements Transport {
       await client.close();
       throw error;
     }
+    opened = client;
     this.#client = client;
     this.#retries = 0;
     return client;
   }
 
+  /**
+   * The library spent its reconnect budget and closed the client. Handing it out again is a node
+   * that refuses every publish until somebody restarts it — and the liveness probe never asks the
+   * bus — so it is forgotten here and replaced in the background, subscriptions included.
+   */
+  #lost(client: NatsClient): void {
+    if (this.#closed || this.#client !== client) return;
+    this.#client = undefined;
+    this.#subscriptions.orphan();
+    this.#report(
+      new TransportUnavailableError({
+        transport: this.name,
+        reason: 'the connection closed once its reconnect budget was spent; re-dialling',
+      }),
+      this.name,
+    );
+    void this.#redial();
+  }
+
+  /**
+   * Re-dial until a client is up or the transport is closed. Bounded by our backoff between
+   * attempts (capped at the policy's `maxMs`), and each attempt is itself a library dial with its
+   * own budget, so a long outage costs one dial per interval rather than a spin. A dial that lands
+   * re-binds every kept subscription and is announced: changes published in the gap are gone.
+   */
+  async #redial(): Promise<void> {
+    if (this.#redialing !== undefined) return;
+    try {
+      for (let attempt = 1; !this.#closed && this.#client === undefined; attempt += 1) {
+        this.#redialing = attempt;
+        try {
+          const client = await this.#dialOnce();
+          // Coded like a first subscribe's refusal; the subscription stays kept for the next client.
+          for (const failure of this.#subscriptions.bindAll(client)) {
+            this.#report(
+              this.#coded(`re-subscribe to ${failure.subject}`, failure.error),
+              failure.subject,
+            );
+          }
+          this.#announce();
+        } catch (error) {
+          if (this.#closed) return;
+          this.#report(error, this.name);
+          await this.#pause(policyDelay(this.#backoff, attempt, this.#rng));
+        }
+      }
+    } finally {
+      this.#redialing = undefined;
+    }
+  }
+
+  /** A wait `close()` can cut short, so a closed transport never holds a timer for `maxMs`. */
+  #pause(ms: number): Promise<void> {
+    return new Promise((resolve) => {
+      const done = (): void => {
+        clearTimeout(timer);
+        this.#wakeRedial = undefined;
+        resolve();
+      };
+      const timer = setTimeout(done, ms);
+      this.#wakeRedial = done;
+    });
+  }
+
   #ensureBucket(client: NatsClient): Promise<void> {
     return ensureKvBucket(
       client,
@@ -200,7 +318,19 @@ export class NatsTransport implements Transport {
     this.#retries = 0;
     const client = this.#client;
     if (client === undefined) return;
+    // Not awaited: the library calls this from its status loop. The failure has one place to go.
     void this.#ensureBucket(client).catch((error: unknown) => this.#report(error, this.name));
+    this.#announce();
+  }
+
+  #announce(): void {
+    for (const listener of [...this.#reconnectListeners]) {
+      try {
+        listener();
+      } catch (error) {
+        this.#report(error, this.name);
+      }
+    }
   }
 
   /**
@@ -214,14 +344,18 @@ export class NatsTransport implements Transport {
     try {
       return call();
     } catch (error) {
-      if (isUltimateError(error)) throw error;
-      throw new TransportUnavailableError({
-        transport: this.name,
-        reason: `${what} was refused: ${renderThrowable(error)}`,
-      });
+      throw this.#coded(what, error);
     }
   }
 
+  #coded(what: string, error: unknown): unknown {
+    if (isUltimateError(error)) return error;
+    return new TransportUnavailableError({
+      transport: this.name,
+      reason: `${what} was refused: ${renderThrowable(error)}`,
+    });
+  }
+
   /**
    * Every background failure lands here — a lost connection, a throwing subscriber, an exhausted
    * reconnect. Dropping it when the caller passed no handler is what turns "no changes arrive"

package/src/offline-queue.ts

@@ -11,6 +11,7 @@
 
 import { renderThrowable, stringField } from '@ultimat3/core/page';
 import type { JsonValue } from './json';
+import { OfflineQueueAbandonedError } from './page-errors';
 import type { WireError } from './sync-protocol';
 
 export type MutationStatus = 'pending' | 'inflight' | 'acked' | 'failed';
@@ -68,6 +69,11 @@ export class MemoryQueueStore implements QueueStore {
   }
 }
 
+/** The one way to build the in-process queue store; the class is a type in the barrel only (`X_FACTORY_NAME_SPELLING`). */
+export function memoryQueueStore(): MemoryQueueStore {
+  return new MemoryQueueStore();
+}
+
 export interface DrainReport {
   readonly sent: number;
   readonly collapsed: number;
@@ -91,6 +97,8 @@ export class OfflineQueue {
    * is parked inside `send`. Bumped by every loss so a pass that resumes afterwards claims nothing.
    */
   #epoch = 0;
+  /** Set by `abandon`, never cleared: this queue's principal is gone. */
+  #abandoned = false;
 
   private constructor(store: QueueStore, state: QueueState) {
     this.#store = store;
@@ -103,8 +111,10 @@ export class OfflineQueue {
    *
    * An `inflight` entry on disk belonged to a page that is gone, so it goes back to `pending`. Left
    * as it was, `#sendable` skipped it forever — no ack was coming to a page that no longer exists —
-   * and every later write overtook it. The replay carries its idempotency key, so a write the old
-   * page did get through is answered from the action's idempotency store, never applied twice.
+   * and every later write overtook it. The replay carries its idempotency key, and a `mutator()`
+   * is REQUIRED to declare `idempotent: true` (`X_MUTATOR_NOT_IDEMPOTENT` at declaration), so a
+   * write the old page did get through is answered from the action's idempotency store and is
+   * never applied twice. That requirement is what makes this resend safe, not the key alone.
    */
   static async open(store: QueueStore): Promise<OfflineQueue> {
     const queue = new OfflineQueue(store, await store.load());
@@ -119,6 +129,7 @@ export class OfflineQueue {
    * other pass can have one on the wire.
    */
   async reload(): Promise<void> {
+    if (this.#abandoned) return;
     const state = await this.#store.load();
     this.#mutations = state.mutations.map((mutation) => ({ ...mutation }));
     this.#nextSeq = Math.max(this.#nextSeq, state.nextSeq);
@@ -167,6 +178,17 @@ export class OfflineQueue {
     input: JsonValue;
     at?: number;
   }): Promise<QueuedMutation> {
+    // Read BEFORE the key is looked up, never between the lookup and the push: with the await in
+    // between, two enqueues of one key both found nothing and both pushed — one intent, two
+    // entries under one sequence number. Another tab of the same user may have taken sequence
+    // numbers since this one loaded, which is what the read is for.
+    //
+    // An abandoned queue REFUSES, before the read and after it (the principal can leave during
+    // it): a write accepted here would sit in memory, shown as queued, with nothing to send it.
+    if (this.#abandoned) throw new OfflineQueueAbandonedError({ name: args.name });
+    const floor = (await this.#store.load()).nextSeq;
+    if (this.#abandoned) throw new OfflineQueueAbandonedError({ name: args.name });
+    this.#nextSeq = Math.max(this.#nextSeq, floor);
     const existing = this.find(args.key);
     if (existing && existing.status !== 'failed') {
       this.#collapsed += 1;
@@ -178,8 +200,6 @@ export class OfflineQueue {
     // entry is dropped and this one takes a new sequence at the back of the queue.
     // By identity: `existing` IS the entry for this key, found above — no second key comparison.
     if (existing) this.#mutations = this.#mutations.filter((entry) => entry !== existing);
-    // Another tab of the same user may have taken sequence numbers since this one loaded.
-    this.#nextSeq = Math.max(this.#nextSeq, (await this.#store.load()).nextSeq);
     const mutation: QueuedMutation = {
       key: args.key,
       seq: this.#nextSeq,
@@ -248,6 +268,22 @@ export class OfflineQueue {
     return returned;
   }
 
+  /**
+   * The principal this queue belongs to is gone (`page-outbox.ts` calls it on a `rescope`), and it
+   * is final: nothing more is SENT, WRITTEN or ACCEPTED (`enqueue` refuses). The epoch bump stops a pass parked
+   * inside `send` before its next entry — which would otherwise leave under the next principal's
+   * session — and the flag stops a pass that starts later, and every write-back: the store was
+   * wiped for this principal, and a pass that then saved what it touched put the writes back.
+   */
+  abandon(): void {
+    this.#epoch += 1;
+    this.#abandoned = true;
+  }
+
+  get abandoned(): boolean {
+    return this.#abandoned;
+  }
+
   /** Server acknowledged: the mutation leaves the queue and its rebase entry can be committed. */
   async ack(key: string): Promise<void> {
     const mutation = this.find(key);
@@ -294,7 +330,7 @@ export class OfflineQueue {
   /** One drain pass. Never called concurrently with itself — `drain` owns that. */
   async #pass(send: MutationSender): Promise<DrainReport> {
     const epoch = this.#epoch;
-    const sendable = this.#sendable();
+    const sendable = this.#abandoned ? [] : this.#sendable();
     // Nothing to do: a pass chained behind one that already sent everything must not rewrite the
     // durable state for the privilege of reporting zero.
     if (sendable.length === 0) {
@@ -326,6 +362,12 @@ export class OfflineQueue {
         // `WebSocket.send` on a CLOSING socket discards it and returns normally — so calling that
         // an ack drops the mutation on exactly the socket death this queue exists to survive.
         // Only `ack`/`fail` (the server) or `requeueInflight` (a lost connection) moves it on.
+        //
+        // A sender may settle the write itself before it returns — the page outbox does, over
+        // HTTP. One it REFUSED keeps the error `fail` just recorded and is not counted as sent:
+        // clearing it here put `{ status: 'failed', error: null }` on disk and reported a refused
+        // write as a sent one.
+        if (statusOf(mutation) === 'failed') continue;
         mutation.error = null;
         sent += 1;
       } catch (error) {
@@ -364,6 +406,7 @@ export class OfflineQueue {
    * queue: a second tab's entries are not this tab's to overwrite or delete.
    */
   async #persist(puts: readonly QueuedMutation[], deletes: readonly string[] = []): Promise<void> {
+    if (this.#abandoned) return;
     await this.#store.write({
       puts: puts.map((mutation) => ({ ...mutation })),
       deletes,
@@ -372,6 +415,11 @@ export class OfflineQueue {
   }
 }
 
+/** Read through a call: `send` awaited in between, and the compiler still narrows to `inflight`. */
+function statusOf(mutation: QueuedMutation): MutationStatus {
+  return mutation.status;
+}
+
 /** A thrown value as the queue records it — `code`, `cause`, `fix`, never a throw of its own. */
 export function toQueueError(error: unknown): WireError {
   return {

package/src/page-errors.ts

@@ -1,9 +1,8 @@
 // The refusals a BROWSER can reach — the page store, the hooks, the socket's wire check, the local
-// store. Apart from `errors.ts` for bytes: that module registers the whole code table at import,
-// and an island that renders one record has no use for sixty titles. The codes stay in `errors.ts`
-// (its `registerErrorCodes()` is what `package.json`'s `sideEffects` names, anchored by the
-// barrel); in a browser that loaded no table a code titles itself from its name, and `code`,
-// `cause` and `fix` — what a reader acts on — are unchanged. This module runs nothing at import.
+// store. Apart from `errors.ts` for bytes: that module holds the sync node's refusal classes, which
+// no island constructs. The codes and their titles are `error-titles.ts` (the module
+// `package.json`'s `sideEffects` names, imported bare by the barrel), so a code constructed here is
+// titled wherever the barrel was reached. This module runs nothing at import.
 
 import { renderFixShellArg } from '@ultimat3/core/page';
 import { RealtimeError } from './realtime-error';
@@ -36,8 +35,8 @@ export class RebaseConflictError extends RealtimeError {
 }
 
 /**
- * A hook ran IN A BROWSER in an island bundle whose bootstrap never called `installRealtime()`.
- * Per BUNDLE, not per page: every island carries its own copy of solid-js, so the signal factory a
+ * A hook ran IN A BROWSER in an island bundle whose bootstrap never called `installRealtime()`,
+ * or before the page runtime (`page-runtime.ts`, carried by the page boot) was installed. Per BUNDLE, not per page: every island carries its own copy of solid-js, so the signal factory a
  * hook renders through has to be that island's own — the page-wide store cannot hold one.
  *
  * A server render is deliberately not this error: no DOM means no reactive runtime to install,
@@ -47,8 +46,8 @@ export class RealtimeUninstalledError extends RealtimeError {
   constructor(args: { hook: string }) {
     super({
       code: 'X_REALTIME_UNINSTALLED',
-      cause: `${args.hook}() ran in a browser island whose bundle never called installRealtime()`,
-      fix: "x build   # the island bootstrap installs it; a hand-built island calls installRealtime({ signal: createSignal }) from '@ultimat3/realtime' before its first render",
+      cause: `${args.hook}() ran in a browser island whose bundle never called installRealtime(), or on a page whose runtime (@ultimat3/realtime/boot) never loaded`,
+      fix: "x build   # the island bootstrap installs it; a hand-built island imports '@ultimat3/realtime/boot' (the page runtime) and calls installRealtime({ signal: createSignal }) from '@ultimat3/realtime' before its first render",
     });
   }
 }
@@ -93,6 +92,23 @@ export class LocalStoreUnavailableError extends RealtimeError {
   }
 }
 
+/**
+ * A write reached a queue whose principal is gone: the page signed out, or in as someone else,
+ * while the write was being queued. It is not stored and never sent — a write issued under one
+ * session must not leave under the next — and it is RAISED, because a user's write dropped in
+ * silence is a loss nobody is told about. `name` is the mutator's registered name; the write's
+ * idempotency key is deliberately not rendered.
+ */
+export class OfflineQueueAbandonedError extends RealtimeError {
+  constructor(args: { name: string }) {
+    super({
+      code: 'X_OFFLINE_QUEUE_ABANDONED',
+      cause: `the write "${args.name}" was queued after its page changed principal, so it was not stored and will not be sent — call the mutation again under the current session`,
+      fix: 'x errors explain X_OFFLINE_QUEUE_ABANDONED --json',
+    });
+  }
+}
+
 /**
  * Something that can only mean "talk to the socket" ran on the server client — a mutation, a
  * publish, a topic subscription, a dial. There is no socket during a server render and there never