@ultimat3/realtime 1.0.0

package/src/nats-transport.ts

@@ -0,0 +1,257 @@
+// Single responsibility: the production `Transport` — core NATS for fanout, JetStream KV for the
+// shared presence sets. Subscriptions are held as intent rather than as socket state, so a lost
+// connection is re-established and re-subscribed underneath the caller: that is what makes a `sync`
+// node stateless and lets any client resubscribe to any node.
+
+import { type Clock, isUltimateError, logger, systemClock } from '@ultimat3/core';
+import { TransportUnavailableError } from './errors';
+import type { Transport, TransportHandler, TransportSet, TransportSubscription } from './fanout';
+import { NatsConnection, type NatsSubscription } from './nats-connection';
+import { ensureKvBucket } from './nats-jetstream';
+import { NatsKvSet } from './nats-kv';
+import { bunNatsStream, type NatsStream, type NatsTarget, parseNatsUrl } from './nats-socket';
+import { type BackoffPolicy, backoffDelay, defaultBackoff, type Rng } from './thundering-herd';
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+export interface NatsTransportOptions {
+  readonly url: string;
+  /** KV bucket backing `shared`. Created on first connect when the cluster has none. */
+  readonly bucket: string;
+  readonly maxReconnectAttempts?: number;
+  readonly backoff?: BackoffPolicy;
+  readonly clock?: Clock;
+  /** Presence TTL, only as the floor for the bucket's whole-stream age limit. */
+  readonly presenceTtlMs?: number;
+  /** A failing subscriber, or a connection lost in the background, must not break the process. */
+  readonly onError?: (error: unknown, subject: string) => void;
+  readonly rng?: Rng;
+  /** Injected so the whole transport — reconnect included — runs in a test with no network. */
+  readonly open?: (target: NatsTarget) => Promise<NatsStream>;
+  readonly sleep?: (ms: number) => Promise<void>;
+}
+
+interface Wanted {
+  readonly subject: string;
+  readonly handler: TransportHandler;
+  live: NatsSubscription | undefined;
+}
+
+const DEFAULT_ATTEMPTS = 10;
+
+/** The production bus: NATS subjects for fanout, a JetStream KV bucket for presence. */
+export class NatsTransport implements Transport {
+  readonly name = 'nats';
+  readonly shared: TransportSet;
+  readonly #target: NatsTarget;
+  readonly #options: NatsTransportOptions;
+  readonly #wanted = new Map<number, Wanted>();
+  readonly #backoff: BackoffPolicy;
+  readonly #attempts: number;
+  readonly #rng: Rng;
+  readonly #sleep: (ms: number) => Promise<void>;
+  #connection: NatsConnection | undefined;
+  #dialing: Promise<NatsConnection> | undefined;
+  #next = 0;
+  #losses = 0;
+  #closed = false;
+
+  constructor(options: NatsTransportOptions) {
+    this.#target = parseNatsUrl(options.url);
+    this.#options = options;
+    this.#backoff = options.backoff ?? defaultBackoff;
+    this.#attempts = options.maxReconnectAttempts ?? DEFAULT_ATTEMPTS;
+    this.#rng = options.rng ?? Math.random;
+    this.#sleep = options.sleep ?? ((ms) => Bun.sleep(ms));
+    this.shared = new NatsKvSet({
+      connection: () => this.#ensure(),
+      bucket: options.bucket,
+      clock: options.clock ?? systemClock,
+    });
+  }
+
+  /** Fail fast at boot rather than on the first change: `/readyz` is meant to catch a dead bus. */
+  async connect(): Promise<void> {
+    await this.#ensure();
+  }
+
+  get connected(): boolean {
+    return this.#connection !== undefined && !this.#connection.closed;
+  }
+
+  async publish(subject: string, payload: string): Promise<void> {
+    const connection = await this.#ensure();
+    await connection.publish(subject, encoder.encode(payload));
+  }
+
+  async subscribe(subject: string, handler: TransportHandler): Promise<TransportSubscription> {
+    this.#next += 1;
+    const id = this.#next;
+    const wanted: Wanted = { subject, handler, live: undefined };
+    this.#wanted.set(id, wanted);
+    try {
+      const connection = await this.#ensure();
+      // The dial this may have triggered already re-bound everything it found registered, this
+      // one included — binding again here would double every delivery on the subject.
+      wanted.live ??= await this.#bind(connection, wanted);
+    } catch (error) {
+      this.#wanted.delete(id);
+      throw error;
+    }
+    return {
+      subject,
+      unsubscribe: () => {
+        this.#wanted.delete(id);
+        void wanted.live?.unsubscribe().catch((error: unknown) => this.#report(error, subject));
+        wanted.live = undefined;
+      },
+    };
+  }
+
+  async close(): Promise<void> {
+    this.#closed = true;
+    this.#wanted.clear();
+    const connection = this.#connection;
+    this.#connection = undefined;
+    await connection?.close();
+  }
+
+  #bind(connection: NatsConnection, wanted: Wanted): Promise<NatsSubscription> {
+    return connection.subscribe(wanted.subject, (message) => {
+      try {
+        wanted.handler(decoder.decode(message.payload), message.subject);
+      } catch (error) {
+        this.#report(error, message.subject);
+      }
+    });
+  }
+
+  #ensure(): Promise<NatsConnection> {
+    if (this.#closed) {
+      return Promise.reject(
+        new TransportUnavailableError({ transport: this.name, reason: 'transport is closed' }),
+      );
+    }
+    const current = this.#connection;
+    if (current !== undefined && !current.closed) return Promise.resolve(current);
+    this.#dialing ??= this.#dial().finally(() => {
+      this.#dialing = undefined;
+    });
+    return this.#dialing;
+  }
+
+  /** Retry is bounded: a bus that is down for longer than the budget is a readiness failure. */
+  async #dial(): Promise<NatsConnection> {
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        return await this.#establish();
+      } catch (error) {
+        // A protocol mismatch answers the same on every attempt — a server too old for per-message
+        // TTL stays too old — so retrying only delays the one report that names the fix.
+        const fatal = isUltimateError(error) && error.code === 'X_TRANSPORT_PROTOCOL';
+        if (fatal || this.#closed || attempt >= this.#attempts) {
+          throw isUltimateError(error)
+            ? error
+            : new TransportUnavailableError({
+                transport: this.name,
+                reason: `${this.#target.host}:${this.#target.port} — ${String(error)}`,
+              });
+        }
+        await this.#sleep(backoffDelay(attempt, this.#backoff, this.#rng));
+      }
+    }
+  }
+
+  /**
+   * One attempt, published only once it is whole. A connection parked in `#connection` before its
+   * bucket and its subscriptions are up answers `connected` for a dial that rejected, keeps its
+   * `onClose` — which then clears the connection that replaced it — and leaves half a rebind
+   * behind, so the next dial binds the same subject a second time and every change arrives twice.
+   * A failed attempt therefore takes its own socket down with it rather than leaking one per retry.
+   */
+  async #establish(): Promise<NatsConnection> {
+    const connection = await this.#open();
+    try {
+      await ensureKvBucket(connection, this.#options.bucket, this.#options.presenceTtlMs ?? 30_000);
+      for (const wanted of this.#wanted.values())
+        wanted.live = await this.#bind(connection, wanted);
+      // `close()` can land while an attempt is in flight, and it only closes what it can see:
+      // publishing now would leave a socket open that nothing will ever close again.
+      if (this.#closed) {
+        throw new TransportUnavailableError({
+          transport: this.name,
+          reason: 'transport is closed',
+        });
+      }
+    } catch (error) {
+      for (const wanted of this.#wanted.values()) wanted.live = undefined;
+      await connection.close();
+      throw error;
+    }
+    this.#connection = connection;
+    this.#losses = 0;
+    return connection;
+  }
+
+  async #open(): Promise<NatsConnection> {
+    const stream = await (this.#options.open ?? bunNatsStream)(this.#target);
+    // The connection names itself in its own `onClose` so a drop can be matched against the live
+    // one. It goes through a holder rather than a `const`, because a buffered `-ERR` closes the
+    // session from inside `open()`, while a `const` binding would still be in its dead zone.
+    const held: { connection: NatsConnection | undefined } = { connection: undefined };
+    held.connection = await NatsConnection.open({
+      stream,
+      target: this.#target,
+      name: 'ultimate',
+      rng: this.#options.rng,
+      onClose: (error) => this.#lost(error, held.connection),
+      onError: (error) => this.#report(error, this.name),
+    });
+    return held.connection;
+  }
+
+  /**
+   * A lost connection re-dials on its own rather than waiting for the next publish: a `sync` node
+   * whose subscriptions are down is silently delivering nothing, which is worse than an error.
+   */
+  #lost(error: unknown, connection: NatsConnection | undefined): void {
+    // Only the live connection may declare a loss. One abandoned mid-dial drops on its own clock,
+    // and without this it would clear the subscriptions — and the reconnect budget — of whichever
+    // connection replaced it, then report a failure the transport had already recovered from.
+    if (connection === undefined || this.#connection !== connection) return;
+    this.#connection = undefined;
+    for (const wanted of this.#wanted.values()) wanted.live = undefined;
+    this.#report(error, this.name);
+    if (this.#closed || this.#wanted.size === 0) return;
+    this.#losses += 1;
+    if (this.#losses > this.#attempts) return;
+    void this.#recover();
+  }
+
+  async #recover(): Promise<void> {
+    // Backoff first: a server that accepts and immediately drops must not become a hot loop.
+    await this.#sleep(backoffDelay(this.#losses - 1, this.#backoff, this.#rng));
+    if (this.#closed || this.connected) return;
+    await this.#ensure().catch((error: unknown) => this.#report(error, this.name));
+  }
+
+  /**
+   * Every background failure lands here — a lost connection, a throwing subscriber, an exhausted
+   * reconnect. Dropping it when the caller passed no handler is what turns "no changes arrive"
+   * into a debugging session with nothing to read, so the default emits rather than swallows.
+   */
+  #report(error: unknown, subject: string): void {
+    const handler = this.#options.onError;
+    if (handler !== undefined) {
+      handler(error, subject);
+      return;
+    }
+    logger.error('nats transport error', {
+      transport: this.name,
+      subject,
+      code: isUltimateError(error) ? error.code : undefined,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}

package/src/offline-queue.ts

@@ -0,0 +1,206 @@
+// Tier 3: the durable mutation queue. Two invariants, both enforced here rather than documented:
+//
+//   1. **Order.** Mutations drain in client sequence order and stop at the first failure. A mutator
+//      that assumed `like` ran before `unlike` must never see them swapped.
+//   2. **Dedupe.** The idempotency key is the identity of the intent. Re-enqueueing a key that is
+//      already queued (double click, replay after a crash) collapses onto the existing entry and
+//      never gets a new sequence number.
+
+import type { JsonValue } from './json';
+import { type Frame, PROTOCOL_VERSION, type WireError } from './sync-protocol';
+
+export type MutationStatus = 'pending' | 'inflight' | 'acked' | 'failed';
+
+export interface QueuedMutation {
+  /** Idempotency key. Supplied by the mutator, stable across retries and reloads. */
+  readonly key: string;
+  /** Monotonic per client. Never renumbered — rebase replays in this order. */
+  readonly seq: number;
+  readonly name: string;
+  readonly input: JsonValue;
+  readonly enqueuedAt: number;
+  attempts: number;
+  status: MutationStatus;
+  error: WireError | null;
+}
+
+export interface QueueState {
+  readonly mutations: readonly QueuedMutation[];
+  readonly nextSeq: number;
+}
+
+/** Durability seam: OPFS/IndexedDB in the browser, memory in tests. */
+export interface QueueStore {
+  load(): Promise<QueueState>;
+  save(state: QueueState): Promise<void>;
+}
+
+export class MemoryQueueStore implements QueueStore {
+  #state: QueueState = { mutations: [], nextSeq: 1 };
+
+  async load(): Promise<QueueState> {
+    return this.#state;
+  }
+
+  async save(state: QueueState): Promise<void> {
+    this.#state = { mutations: state.mutations.map((m) => ({ ...m })), nextSeq: state.nextSeq };
+  }
+}
+
+export interface DrainReport {
+  readonly sent: number;
+  readonly collapsed: number;
+  readonly remaining: number;
+  readonly stoppedAt: string | null;
+}
+
+export type MutationSender = (mutation: QueuedMutation) => Promise<void>;
+
+export class OfflineQueue {
+  readonly #store: QueueStore;
+  #mutations: QueuedMutation[] = [];
+  #nextSeq = 1;
+  #collapsed = 0;
+
+  private constructor(store: QueueStore, state: QueueState) {
+    this.#store = store;
+    this.#mutations = state.mutations.map((mutation) => ({ ...mutation }));
+    this.#nextSeq = state.nextSeq;
+  }
+
+  /** Rehydrates from durable storage, so a reload resumes the same queue with the same sequence. */
+  static async open(store: QueueStore): Promise<OfflineQueue> {
+    return new OfflineQueue(store, await store.load());
+  }
+
+  get size(): number {
+    return this.#mutations.length;
+  }
+
+  get collapsed(): number {
+    return this.#collapsed;
+  }
+
+  get nextSeq(): number {
+    return this.#nextSeq;
+  }
+
+  find(key: string): QueuedMutation | undefined {
+    return this.#mutations.find((mutation) => mutation.key === key);
+  }
+
+  /** Sorted by sequence. This is the only order anything downstream is allowed to use. */
+  pending(): readonly QueuedMutation[] {
+    return this.#mutations
+      .filter((mutation) => mutation.status === 'pending' || mutation.status === 'inflight')
+      .sort((a, b) => a.seq - b.seq);
+  }
+
+  all(): readonly QueuedMutation[] {
+    return [...this.#mutations].sort((a, b) => a.seq - b.seq);
+  }
+
+  async enqueue(args: {
+    key: string;
+    name: string;
+    input: JsonValue;
+    at?: number;
+  }): Promise<QueuedMutation> {
+    const existing = this.find(args.key);
+    if (existing) {
+      this.#collapsed += 1;
+      return existing;
+    }
+    const mutation: QueuedMutation = {
+      key: args.key,
+      seq: this.#nextSeq,
+      name: args.name,
+      input: args.input,
+      enqueuedAt: args.at ?? 0,
+      attempts: 0,
+      status: 'pending',
+      error: null,
+    };
+    this.#nextSeq += 1;
+    this.#mutations.push(mutation);
+    await this.#persist();
+    return mutation;
+  }
+
+  /**
+   * Drains in sequence order and stops at the first failure. Continuing past a failure is how a
+   * sync engine reorders a user's intent — so it does not continue.
+   */
+  async drain(send: MutationSender): Promise<DrainReport> {
+    let sent = 0;
+    for (const mutation of this.pending()) {
+      mutation.status = 'inflight';
+      mutation.attempts += 1;
+      try {
+        await send(mutation);
+        mutation.status = 'acked';
+        mutation.error = null;
+        sent += 1;
+      } catch (error) {
+        mutation.status = 'pending';
+        mutation.error = toQueueError(error);
+        await this.#persist();
+        return {
+          sent,
+          collapsed: this.#collapsed,
+          remaining: this.pending().length,
+          stoppedAt: mutation.key,
+        };
+      }
+    }
+    await this.#persist();
+    return { sent, collapsed: this.#collapsed, remaining: this.pending().length, stoppedAt: null };
+  }
+
+  /** Server acknowledged: the mutation leaves the queue and its rebase entry can be committed. */
+  async ack(key: string): Promise<void> {
+    const mutation = this.find(key);
+    if (!mutation) return;
+    mutation.status = 'acked';
+    this.#mutations = this.#mutations.filter((candidate) => candidate.key !== key);
+    await this.#persist();
+  }
+
+  /** Terminal failure (policy denial, validation): kept for the UI, never retried blindly. */
+  async fail(key: string, error: WireError): Promise<void> {
+    const mutation = this.find(key);
+    if (!mutation) return;
+    mutation.status = 'failed';
+    mutation.error = error;
+    await this.#persist();
+  }
+
+  async clear(): Promise<void> {
+    this.#mutations = [];
+    await this.#persist();
+  }
+
+  async #persist(): Promise<void> {
+    await this.#store.save({ mutations: this.#mutations, nextSeq: this.#nextSeq });
+  }
+}
+
+export function mutateFrame(mutation: QueuedMutation): Frame {
+  return {
+    type: 'mutate',
+    v: PROTOCOL_VERSION,
+    key: mutation.key,
+    seq: mutation.seq,
+    name: mutation.name,
+    input: mutation.input,
+  };
+}
+
+function toQueueError(error: unknown): WireError {
+  const shape = error as { code?: unknown; cause?: unknown; fix?: unknown } | null;
+  return {
+    code: typeof shape?.code === 'string' ? shape.code : 'X_TRANSPORT_UNAVAILABLE',
+    cause: typeof shape?.cause === 'string' ? shape.cause : String(error),
+    fix: typeof shape?.fix === 'string' ? shape.fix : 'the queue retries on the next reconnect',
+  };
+}

package/src/pg-advisory-lock.ts

@@ -0,0 +1,98 @@
+// Single responsibility: the production `AdvisoryLock` — backs `replicator.ts`'s "exactly one
+// replicator per database" invariant with `SELECT pg_try_advisory_lock(hashtext(key))`. The lock
+// is scoped to this connection's Postgres *session*, so acquiring means keeping the connection
+// open and releasing means closing it: no lease, no renewal, no fencing token, ever.
+
+import { ReplicationFailedError } from './errors';
+import { PgConnection, type PgRows } from './pg-connection';
+import { bunPgStream, type PgTarget, parsePgUrl } from './pg-socket';
+import type { PgStream } from './pg-wire';
+import type { AdvisoryLock } from './replicator';
+import type { Rng } from './thundering-herd';
+
+/** A key reaches a simple query unparameterised, so its charset is the injection boundary. */
+const KEY_PATTERN = /^[A-Za-z0-9:_.-]+$/;
+
+export interface PgAdvisoryLockOptions {
+  /** Connection string for the database whose replicator this is. */
+  readonly url: string;
+  /** Lock identity, e.g. `x:replicator:<slot>`. Hashed by Postgres, not by us. */
+  readonly key: string;
+  /** The byte pipe, injected. Defaults to `bunPgStream`; a test drives a scripted server. */
+  readonly stream?: ((target: PgTarget) => Promise<PgStream>) | undefined;
+  /** Injected so the SCRAM nonce is deterministic under a seeded test. */
+  readonly rng?: Rng | undefined;
+}
+
+/**
+ * One `pg_try_advisory_lock`, held by a connection this class opens for itself. Postgres refcounts
+ * a session-level advisory lock per acquisition — a second `pg_try_advisory_lock` on a session that
+ * already holds the key would need a matching second `pg_advisory_unlock`, and `release()` only
+ * ever issues one, so taking a second grant here would be a leak, not a no-op.
+ */
+export class PgAdvisoryLock implements AdvisoryLock {
+  readonly key: string;
+  readonly #options: PgAdvisoryLockOptions;
+  #connection: PgConnection | null = null;
+
+  constructor(options: PgAdvisoryLockOptions) {
+    if (!KEY_PATTERN.test(options.key)) {
+      throw new ReplicationFailedError({
+        stage: 'preflight',
+        detail: `advisory lock key "${options.key}" is not [A-Za-z0-9:_.-]+`,
+        fix:
+          'rename the lock key to match [A-Za-z0-9:_.-]+ — it is interpolated into ' +
+          'pg_try_advisory_lock(hashtext(...))',
+      });
+    }
+    this.key = options.key;
+    this.#options = options;
+  }
+
+  async tryAcquire(): Promise<boolean> {
+    // Already ours — see the class comment on why a second `pg_try_advisory_lock` must not run.
+    if (this.#connection !== null) return true;
+    const target = parsePgUrl(this.#options.url);
+    const stream = await (this.#options.stream ?? bunPgStream)(target);
+    // Plain SQL, not `replication: 'database'` — this session runs one statement, never a feed.
+    const connection = await PgConnection.open({
+      stream,
+      user: target.user,
+      password: target.password,
+      database: target.database,
+      applicationName: `ultimate-replicator-lock:${this.key}`,
+      rng: this.#options.rng,
+    });
+    let rows: PgRows;
+    try {
+      rows = await connection.query(`SELECT pg_try_advisory_lock(hashtext('${this.key}'))`);
+    } catch (failure) {
+      // "The database refused me" and "another process holds it" are different facts; only the
+      // second is a `false`, so a query failure closes what was opened and propagates instead.
+      await connection.close();
+      throw failure;
+    }
+    if (rows[0]?.[0] !== 't') {
+      // A standby must not hold an idle session for a lock it did not get.
+      await connection.close();
+      return false;
+    }
+    // Kept open on purpose: this session *is* the lock, and closing it is the whole of
+    // `release()` — which is exactly what lets a crashed replicator free the slot with nothing
+    // left to clean up.
+    this.#connection = connection;
+    return true;
+  }
+
+  async release(): Promise<void> {
+    const connection = this.#connection;
+    if (connection === null) return;
+    this.#connection = null;
+    try {
+      await connection.query(`SELECT pg_advisory_unlock(hashtext('${this.key}'))`);
+    } finally {
+      // The close releases the session lock regardless, so it must run even if the unlock did not.
+      await connection.close();
+    }
+  }
+}