@ultimat3/realtime 1.0.0

package/src/rebase.ts

@@ -0,0 +1,198 @@
+// Tier 3: the reconcile loop. The server rebases; the client rolls back and reapplies.
+//
+// Truth is always the server — a client is never the merge authority. So reconciliation is exactly:
+// undo the optimistic writes from the acknowledged mutation onward (newest first), land server
+// truth, then replay the still-pending mutators in sequence order. Because `local` is a pure
+// function of `(tx, input)`, that replay is deterministic; that purity rule is the whole reason
+// tier 3 is affordable.
+
+import { RebaseConflictError } from './errors';
+import type { Row } from './json';
+import type { LocalStore, LocalTx, TableMap } from './local-store';
+import { type ConflictStrategyName, type Frame, PROTOCOL_VERSION } from './sync-protocol';
+
+export interface MergeArgs {
+  /** Local row as the user last saw it, before any rollback. */
+  readonly local: Row | undefined;
+  /** Local row after the optimistic writes were undone — the shared base. */
+  readonly base: Row | undefined;
+  /** Server truth. `null` means the server deleted the row. */
+  readonly server: Row | null;
+}
+
+export interface CustomMerge {
+  readonly kind: 'custom';
+  merge(args: MergeArgs): Row | null | undefined;
+}
+
+/** `conflict: custom(merge)` — exactly the name the mutator contract uses. */
+export function custom(merge: (args: MergeArgs) => Row | null | undefined): CustomMerge {
+  return { kind: 'custom', merge };
+}
+
+export type ConflictStrategy = 'server-wins' | 'last-write-wins' | CustomMerge;
+
+export function strategyName(strategy: ConflictStrategy): ConflictStrategyName {
+  return typeof strategy === 'string' ? strategy : 'custom';
+}
+
+export interface RebaseEntry<T extends TableMap = TableMap> {
+  /** Idempotency key — the same key the offline queue uses. */
+  readonly key: string;
+  readonly seq: number;
+  readonly entity: string;
+  readonly strategy: ConflictStrategy;
+  /** The mutator's `local` half, curried with its input. Pure, therefore replayable. */
+  apply(tx: LocalTx<T>): void;
+}
+
+/** Optimistic writes awaiting server truth, ordered by client sequence. */
+export class RebaseLog<T extends TableMap = TableMap> {
+  readonly #entries = new Map<string, RebaseEntry<T>>();
+
+  record(entry: RebaseEntry<T>): void {
+    this.#entries.set(entry.key, entry);
+  }
+
+  get(key: string): RebaseEntry<T> | undefined {
+    return this.#entries.get(key);
+  }
+
+  drop(key: string): void {
+    this.#entries.delete(key);
+  }
+
+  pending(): readonly RebaseEntry<T>[] {
+    return [...this.#entries.values()].sort((a, b) => a.seq - b.seq);
+  }
+
+  get size(): number {
+    return this.#entries.size;
+  }
+}
+
+export interface ServerAck {
+  readonly key: string;
+  readonly entity: string;
+  readonly id: string;
+  readonly row: Row | null;
+}
+
+export interface ReconcileOptions {
+  /** Field compared by `last-write-wins`. Must be a number (epoch ms) written by the server. */
+  readonly clockField?: string;
+}
+
+export interface ReconcileResult {
+  readonly strategy: ConflictStrategyName;
+  readonly rolledBack: readonly string[];
+  readonly reapplied: readonly string[];
+  readonly winner: 'server' | 'local' | 'merge';
+}
+
+/**
+ * One acknowledgement, one rebase. Rolls back the acked mutation and every later optimistic write,
+ * lands server truth under the mutator's conflict strategy, then replays the rest in sequence order.
+ */
+export function reconcile<T extends TableMap = TableMap>(
+  args: {
+    store: LocalStore<T>;
+    log: RebaseLog<T>;
+    ack: ServerAck;
+  },
+  options: ReconcileOptions = {},
+): ReconcileResult {
+  const { store, log, ack } = args;
+  const entry = log.get(ack.key);
+  const strategy = entry?.strategy ?? 'server-wins';
+  const table = store.table(ack.entity);
+  const local = table.get(ack.id);
+
+  // Everything at or after the acked sequence is optimistic and must be undone newest-first.
+  const affected = log.pending().filter((candidate) => candidate.seq >= (entry?.seq ?? 0));
+  const rolledBack: string[] = [];
+  for (const candidate of [...affected].reverse()) {
+    store.rollback(candidate.key);
+    rolledBack.push(candidate.key);
+  }
+
+  const base = store.table(ack.entity).get(ack.id);
+  const winner = land(store, ack, strategy, { local, base }, options);
+  log.drop(ack.key);
+
+  const reapplied: string[] = [];
+  for (const candidate of affected) {
+    if (candidate.key === ack.key) continue;
+    store.apply(candidate.key, (tx) => candidate.apply(tx));
+    reapplied.push(candidate.key);
+  }
+
+  return { strategy: strategyName(strategy), rolledBack, reapplied, winner };
+}
+
+function land<T extends TableMap>(
+  store: LocalStore<T>,
+  ack: ServerAck,
+  strategy: ConflictStrategy,
+  rows: { local: Row | undefined; base: Row | undefined },
+  options: ReconcileOptions,
+): 'server' | 'local' | 'merge' {
+  const table = store.table(ack.entity);
+
+  if (strategy === 'server-wins') {
+    write(store, ack.entity, ack.id, ack.row);
+    return 'server';
+  }
+
+  if (strategy === 'last-write-wins') {
+    const field = options.clockField ?? 'updatedAt';
+    const localAt = numberAt(rows.local, field);
+    const serverAt = numberAt(ack.row, field);
+    if (localAt !== null && serverAt !== null && localAt > serverAt) {
+      // The local write is newer by the server's own clock field: keep it, do not clobber.
+      if (rows.local) table.upsert(rows.local);
+      return 'local';
+    }
+    write(store, ack.entity, ack.id, ack.row);
+    return 'server';
+  }
+
+  const merged = strategy.merge({ local: rows.local, base: rows.base, server: ack.row });
+  if (merged === undefined) {
+    throw new RebaseConflictError({
+      key: ack.key,
+      entity: ack.entity,
+      reason: 'custom(merge) returned undefined; return a row, or null to accept the delete',
+    });
+  }
+  write(store, ack.entity, ack.id, merged);
+  return 'merge';
+}
+
+function write<T extends TableMap>(
+  store: LocalStore<T>,
+  entity: string,
+  id: string,
+  row: Row | null,
+): void {
+  const table = store.table(entity);
+  if (row === null) table.delete(id);
+  else table.upsert(row);
+}
+
+function numberAt(row: Row | null | undefined, field: string): number | null {
+  if (!row) return null;
+  const value = row[field];
+  return typeof value === 'number' ? value : null;
+}
+
+export function rebaseFrame(ack: ServerAck, strategy: ConflictStrategy): Frame {
+  return {
+    type: 'rebase',
+    v: PROTOCOL_VERSION,
+    key: ack.key,
+    entity: ack.entity,
+    strategy: strategyName(strategy),
+    row: ack.row,
+  };
+}

package/src/replicator.ts

@@ -0,0 +1,185 @@
+// The `replicator` role: consume the change feed, normalize, publish to the transport. Nothing else.
+//
+// **Exactly one per database.** Two replicators on one slot means every change is fanned out twice:
+// duplicate patches, duplicate lsns, and a client digest that can never converge. The invariant is
+// enforced by a Postgres session-level advisory lock, not by deployment discipline:
+//
+//   SELECT pg_try_advisory_lock(hashtext('x:replicator:<slot>'))
+//
+// The lock is tied to the *session*, so a crashed replicator releases it automatically — no lease
+// renewal, no fencing token, no split brain. A process that fails to take the lock does not start
+// its feed and reports `/readyz` false; it retries with jittered backoff and takes over the moment
+// the holder dies. Scaling the replicator is therefore always vertical, and that is by design.
+
+import { logger, withSpan } from '@ultimat3/core';
+import type { ChangeEvent, ChangeFeed } from './changefeed';
+import type { Transport } from './fanout';
+import { type BackoffPolicy, backoffDelay, defaultBackoff, type Rng } from './thundering-herd';
+
+export const CHANGE_SUBJECT_PREFIX = 'x.change';
+
+/** Session-scoped mutual exclusion. Postgres-backed in production, in-memory for `x dev`. */
+export interface AdvisoryLock {
+  readonly key: string;
+  /** `false` means another process holds it — never block, never steal. */
+  tryAcquire(): Promise<boolean>;
+  release(): Promise<void>;
+}
+
+/** Single-process default: correct for `x dev` and tests, useless across containers (by design). */
+export class InMemoryAdvisoryLock implements AdvisoryLock {
+  static readonly #held = new Set<string>();
+  readonly key: string;
+  #mine = false;
+
+  constructor(key: string) {
+    this.key = key;
+  }
+
+  async tryAcquire(): Promise<boolean> {
+    if (InMemoryAdvisoryLock.#held.has(this.key)) return false;
+    InMemoryAdvisoryLock.#held.add(this.key);
+    this.#mine = true;
+    return true;
+  }
+
+  async release(): Promise<void> {
+    if (!this.#mine) return;
+    InMemoryAdvisoryLock.#held.delete(this.key);
+    this.#mine = false;
+  }
+}
+
+/** `x.change.<entity>.<orgId>` — tenant in the subject, so fanout filters without parsing a row. */
+export function changeSubject(change: ChangeEvent): string {
+  return `${CHANGE_SUBJECT_PREFIX}.${change.entity}.${change.orgId ?? '_'}`;
+}
+
+export interface ReplicatorOptions {
+  readonly feed: ChangeFeed;
+  readonly transport: Transport;
+  readonly lock: AdvisoryLock;
+  /** Resume position, normally the last lsn this replicator persisted. */
+  readonly from?: string;
+  readonly subjectOf?: (change: ChangeEvent) => string;
+  readonly backoff?: BackoffPolicy;
+  readonly rng?: Rng;
+}
+
+export interface ReplicatorStats {
+  readonly published: number;
+  readonly skipped: number;
+  readonly outOfOrder: number;
+}
+
+export interface Replicator {
+  /** `false` = another replicator holds the lock; this process must stay `/readyz` false. */
+  start(): Promise<boolean>;
+  stop(): Promise<void>;
+  readonly running: boolean;
+  lastLsn(): string | null;
+  stats(): ReplicatorStats;
+  /** Delay before the next takeover attempt, jittered so N standbys do not collide. */
+  retryDelayMs(attempt: number): number;
+}
+
+export function createReplicator(options: ReplicatorOptions): Replicator {
+  const subjectOf = options.subjectOf ?? changeSubject;
+  const backoff = options.backoff ?? defaultBackoff;
+  let running = false;
+  let lastLsn: string | null = null;
+  let published = 0;
+  let skipped = 0;
+  let outOfOrder = 0;
+
+  const onChange = async (raw: ChangeEvent): Promise<void> => {
+    const change = normalize(raw);
+    if (!change) {
+      skipped += 1;
+      return;
+    }
+    if (lastLsn !== null && change.lsn <= lastLsn) {
+      // At-least-once delivery is the feed's contract, so a repeat is expected, not an error.
+      outOfOrder += 1;
+      return;
+    }
+    await withSpan('realtime.replicate', async () => {
+      await options.transport.publish(subjectOf(change), JSON.stringify(change));
+      lastLsn = change.lsn;
+      published += 1;
+    });
+  };
+
+  return {
+    async start(): Promise<boolean> {
+      if (running) return true;
+      if (!(await options.lock.tryAcquire())) {
+        logger.warn('replicator standby: advisory lock held elsewhere', { key: options.lock.key });
+        return false;
+      }
+      running = true;
+      await options.feed.start(
+        options.from === undefined ? { onChange } : { from: options.from, onChange },
+      );
+      logger.info('replicator started', { source: options.feed.source, key: options.lock.key });
+      return true;
+    },
+
+    async stop(): Promise<void> {
+      if (!running) return;
+      running = false;
+      await options.feed.stop();
+      await options.lock.release();
+    },
+
+    get running(): boolean {
+      return running;
+    },
+
+    lastLsn(): string | null {
+      return lastLsn ?? options.feed.lastLsn();
+    },
+
+    stats(): ReplicatorStats {
+      return { published, skipped, outOfOrder };
+    },
+
+    retryDelayMs(attempt: number): number {
+      return backoffDelay(attempt, backoff, options.rng ?? Math.random);
+    },
+  };
+}
+
+/** Drops events the pipeline cannot use and hoists the tenant id out of the row. */
+export function normalize(change: ChangeEvent): ChangeEvent | null {
+  const row = change.after ?? change.before;
+  if (!row) return null;
+  if (change.op === 'insert' && change.after === null) return null;
+  if (change.op === 'delete' && change.before === null) return null;
+  if (change.orgId !== null) return change;
+  const orgId = row['orgId'];
+  return typeof orgId === 'string' ? { ...change, orgId } : change;
+}
+
+/** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
+export function parseChange(payload: string): ChangeEvent | null {
+  try {
+    const parsed: unknown = JSON.parse(payload);
+    if (typeof parsed !== 'object' || parsed === null) return null;
+    const shape = parsed as Partial<ChangeEvent>;
+    if (typeof shape.entity !== 'string' || typeof shape.lsn !== 'string') return null;
+    if (shape.op !== 'insert' && shape.op !== 'update' && shape.op !== 'delete') return null;
+    return {
+      entity: shape.entity,
+      op: shape.op,
+      before: shape.before ?? null,
+      after: shape.after ?? null,
+      lsn: shape.lsn,
+      txid: typeof shape.txid === 'string' ? shape.txid : '',
+      orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
+      at: typeof shape.at === 'number' ? shape.at : 0,
+    };
+  } catch {
+    return null;
+  }
+}

package/src/socket.ts

@@ -0,0 +1,208 @@
+// One WS connection. Bun's WS profile is what makes a million sockets affordable, so this object
+// is deliberately lean: ~8 fields plus two small sets. Budget is ~1KB of JS heap per connection on
+// top of Bun's own per-socket cost — anything richer (row caches, per-socket buffers) belongs in
+// the transport or the change buffer, never here. `sync` is stateless: nothing on this object
+// survives a restart, and nothing needs to.
+
+import { type Actor, type Clock, systemClock, uuid } from '@ultimat3/core';
+import { encode, type Frame } from './sync-protocol';
+
+export const CLOSE = {
+  normal: 1000,
+  goingAway: 1001,
+  policy: 1008,
+  overloaded: 1013,
+  versionSkew: 4000,
+  idle: 4001,
+  drain: 4002,
+} as const;
+
+/** The slice of Bun's `ServerWebSocket` this package uses. Structural, so tests need no server. */
+export interface WsLike {
+  send(data: string): number;
+  close(code?: number, reason?: string): void;
+  subscribe(topic: string): void;
+  unsubscribe(topic: string): void;
+  getBufferedAmount(): number;
+}
+
+export interface SyncSocketOptions {
+  readonly ws: WsLike;
+  /** Build id the *client* reported in `hello`. Version skew is a first-class connection state. */
+  readonly clientBuildId: string;
+  readonly serverBuildId: string;
+  readonly actor?: Actor | null;
+  readonly id?: string;
+  readonly clock?: Clock;
+  /** Frames are dropped rather than queued past this. See `desynced`. */
+  readonly maxBufferedBytes?: number;
+  readonly maxDroppedFrames?: number;
+}
+
+export function actorIdOf(actor: Actor | null): string | null {
+  return actor === null ? null : actor.id;
+}
+
+export class SyncSocket {
+  readonly id: string;
+  readonly clientBuildId: string;
+  readonly serverBuildId: string;
+  readonly openedAt: number;
+  /** Channel topics (tier 1). Live-query subscriptions are keyed separately, by sid. */
+  readonly topics = new Set<string>();
+  /** sid -> qid for live queries (tier 2/3) on this connection. */
+  readonly queries = new Map<string, string>();
+  /**
+   * Subscriptions whose patch stream was interrupted by backpressure. Dropping a patch is safe
+   * *only* because the cursor makes a re-snapshot cheap; the drop is recorded here so the next
+   * flush re-snapshots instead of silently diverging.
+   */
+  readonly desynced = new Set<string>();
+
+  actor: Actor | null;
+  lastSeenAt: number;
+  droppedFrames = 0;
+  sentFrames = 0;
+
+  readonly #ws: WsLike;
+  readonly #clock: Clock;
+  readonly #maxBufferedBytes: number;
+  readonly #maxDroppedFrames: number;
+  #closed = false;
+
+  constructor(options: SyncSocketOptions) {
+    this.#ws = options.ws;
+    this.#clock = options.clock ?? systemClock;
+    this.id = options.id ?? uuid();
+    this.clientBuildId = options.clientBuildId;
+    this.serverBuildId = options.serverBuildId;
+    this.actor = options.actor ?? null;
+    this.#maxBufferedBytes = options.maxBufferedBytes ?? 1024 * 1024;
+    this.#maxDroppedFrames = options.maxDroppedFrames ?? 32;
+    this.openedAt = this.#clock.now().getTime();
+    this.lastSeenAt = this.openedAt;
+  }
+
+  get actorId(): string | null {
+    return actorIdOf(this.actor);
+  }
+
+  get closed(): boolean {
+    return this.#closed;
+  }
+
+  /** A skewed client gets an `update-available` frame; it is never silently served a new shape. */
+  get skewed(): boolean {
+    return this.clientBuildId !== this.serverBuildId;
+  }
+
+  /** `false` means the frame was dropped by backpressure — the caller must mark state stale. */
+  send(frame: Frame): boolean {
+    if (this.#closed) return false;
+    if (this.#ws.getBufferedAmount() > this.#maxBufferedBytes) {
+      this.droppedFrames += 1;
+      if (this.droppedFrames > this.#maxDroppedFrames) {
+        this.close(CLOSE.overloaded, 'backpressure');
+      }
+      return false;
+    }
+    this.#ws.send(encode(frame));
+    this.sentFrames += 1;
+    return true;
+  }
+
+  /** Record a dropped or invalidated subscription so the next flush re-snapshots it. */
+  markDesynced(sid: string): void {
+    this.desynced.add(sid);
+  }
+
+  clearDesynced(sid: string): void {
+    this.desynced.delete(sid);
+  }
+
+  subscribeTopic(topic: string): void {
+    this.topics.add(topic);
+    this.#ws.subscribe(topic);
+  }
+
+  unsubscribeTopic(topic: string): void {
+    this.topics.delete(topic);
+    this.#ws.unsubscribe(topic);
+  }
+
+  touch(): void {
+    this.lastSeenAt = this.#clock.now().getTime();
+  }
+
+  idleFor(now: number): number {
+    return now - this.lastSeenAt;
+  }
+
+  close(code: number = CLOSE.normal, reason = ''): void {
+    if (this.#closed) return;
+    this.#closed = true;
+    this.#ws.close(code, reason);
+  }
+}
+
+export interface SocketRegistryOptions {
+  readonly clock?: Clock;
+  /** Bun also enforces its own `idleTimeout`; this sweep catches half-open connections. */
+  readonly idleTimeoutMs?: number;
+}
+
+/** Per-node socket table. Intentionally the only in-memory map on a `sync` node. */
+export class SocketRegistry {
+  readonly #sockets = new Map<string, SyncSocket>();
+  readonly #clock: Clock;
+  readonly #idleTimeoutMs: number;
+
+  constructor(options: SocketRegistryOptions = {}) {
+    this.#clock = options.clock ?? systemClock;
+    this.#idleTimeoutMs = options.idleTimeoutMs ?? 120_000;
+  }
+
+  add(socket: SyncSocket): void {
+    this.#sockets.set(socket.id, socket);
+  }
+
+  remove(id: string): void {
+    this.#sockets.delete(id);
+  }
+
+  get(id: string): SyncSocket | undefined {
+    return this.#sockets.get(id);
+  }
+
+  all(): Iterable<SyncSocket> {
+    return this.#sockets.values();
+  }
+
+  get count(): number {
+    return this.#sockets.size;
+  }
+
+  /** Closes and returns everything past the idle budget. Called on an interval by `sync-node`. */
+  sweepIdle(): SyncSocket[] {
+    const now = this.#clock.now().getTime();
+    const closed: SyncSocket[] = [];
+    for (const socket of this.#sockets.values()) {
+      if (socket.idleFor(now) > this.#idleTimeoutMs) {
+        socket.close(CLOSE.idle, 'idle timeout');
+        this.#sockets.delete(socket.id);
+        closed.push(socket);
+      }
+    }
+    return closed;
+  }
+
+  /** Local delivery for a channel topic. Bun's native pub/sub handles the common case; this is
+   *  the fallback used when a frame must be filtered per socket (policy, desync bookkeeping). */
+  deliver(topic: string, frame: Frame): number {
+    let sent = 0;
+    for (const socket of this.#sockets.values()) {
+      if (socket.topics.has(topic) && socket.send(frame)) sent += 1;
+    }
+    return sent;
+  }
+}