@ultimat3/realtime 1.0.0

package/src/pgoutput.ts

@@ -0,0 +1,297 @@
+// Decodes pgoutput logical-replication messages (protocol version 1, Postgres >= 12) into typed
+// PgOutputMessage values, and the postgres text-format values inside each tuple into JsonValue.
+// Pure byte decoding: no sockets, no I/O. A decoder instance owns the per-connection relation
+// cache that later Insert/Update/Delete/Truncate messages reference by oid.
+
+import { ReplicationProtocolError } from './errors';
+import type { JsonObject, JsonValue } from './json';
+import { ByteReader, pgTimestampToEpochMs } from './pg-bytes';
+
+export interface PgColumn {
+  /** part of the replica identity key — set by the `flags & 1` bit. */
+  readonly key: boolean;
+  /** the physical, snake_case column name as postgres reports it. */
+  readonly name: string;
+  readonly typeOid: number;
+  readonly typeMod: number;
+}
+
+export interface PgRelation {
+  readonly oid: number;
+  readonly schema: string;
+  readonly name: string;
+  /** 'd' default | 'n' nothing | 'f' full | 'i' index */
+  readonly replicaIdentity: string;
+  readonly columns: readonly PgColumn[];
+}
+
+export type PgOutputMessage =
+  | {
+      readonly kind: 'begin';
+      readonly commitLsn: bigint;
+      readonly commitAt: number;
+      readonly xid: number;
+    }
+  | {
+      readonly kind: 'commit';
+      readonly commitLsn: bigint;
+      readonly endLsn: bigint;
+      readonly commitAt: number;
+    }
+  | { readonly kind: 'relation'; readonly relation: PgRelation }
+  | { readonly kind: 'insert'; readonly relation: PgRelation; readonly after: JsonObject }
+  | {
+      readonly kind: 'update';
+      readonly relation: PgRelation;
+      readonly before: JsonObject | null;
+      readonly after: JsonObject;
+    }
+  | { readonly kind: 'delete'; readonly relation: PgRelation; readonly before: JsonObject }
+  | { readonly kind: 'truncate'; readonly relations: readonly PgRelation[] }
+  /** origin / type / logical message — decoded far enough to be skipped safely. */
+  | { readonly kind: 'other'; readonly tag: string };
+
+/**
+ * Postgres sends every value as text (we never negotiate binary). Decoding depends on the
+ * column's type oid — the wire gives us nothing else to go on, so this switch is the one place
+ * that type catalogue is encoded.
+ */
+function decodeValue(typeOid: number, text: string): JsonValue {
+  switch (typeOid) {
+    case 16: // bool
+      return text === 't';
+
+    case 20: {
+      // int8: only safe as a number if it round-trips exactly; otherwise keep the digits —
+      // a rounded bigint is a worse lie than a string that still parses correctly downstream.
+      const asNumber = Number(text);
+      return Number.isSafeInteger(asNumber) ? asNumber : text;
+    }
+
+    case 21: // int2
+    case 23: // int4
+    case 26: // oid
+      return Number(text);
+
+    case 700: // float4
+    case 701: // float8
+      // JSON has no literal for these three, so the text form survives the round trip instead of
+      // silently becoming a number `JSON.stringify` would otherwise turn into `null`.
+      if (text === 'NaN' || text === 'Infinity' || text === '-Infinity') return text;
+      return Number(text);
+
+    case 1700: // numeric — exactness beats convenience; money is never a float here.
+      return text;
+
+    case 114: // json
+    case 3802: {
+      // jsonb
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(text);
+      } catch (cause) {
+        throw new ReplicationProtocolError({
+          stage: 'value',
+          detail: `type oid ${typeOid} carried invalid json: ${String(cause)}`,
+        });
+      }
+      return parsed as JsonValue;
+    }
+
+    case 1082: // date
+    case 1114: // timestamp
+    case 1184: // timestamptz
+      return text; // an ISO-ish string; never a `Date` — the row must stay JSON.
+
+    case 17: // bytea — the `\x...` text form, as-is.
+      return text;
+
+    default: // text, varchar, uuid, enum, and everything else not called out above.
+      return text;
+  }
+}
+
+/**
+ * Int16 ncolumns + that many columns. Every tuple kind (insert's new row, update/delete's old
+ * row, a `'K'` key-only row) shares this decoder: postgres always sends one byte per column,
+ * `'u'` standing in for the columns a key-only tuple leaves out — so the column count always
+ * matches the relation, and only the per-column byte tells us whether a value is actually there.
+ */
+function decodeTupleData(reader: ByteReader, relation: PgRelation): JsonObject {
+  const count = reader.int16();
+  if (count !== relation.columns.length) {
+    throw new ReplicationProtocolError({
+      stage: 'tuple',
+      detail:
+        `relation "${relation.schema}.${relation.name}" has ${relation.columns.length} columns ` +
+        `but a tuple for it carried ${count}`,
+    });
+  }
+
+  const row: JsonObject = {};
+  for (const column of relation.columns) {
+    const kind = reader.tag();
+    if (kind === 'n') {
+      row[column.name] = null;
+      continue;
+    }
+    if (kind === 'u') {
+      // Omitted, not nulled: "unchanged" and "set to null" are different facts about the row,
+      // and a missing key is the only encoding that keeps those two facts distinguishable.
+      continue;
+    }
+    if (kind === 't') {
+      const length = reader.int32();
+      row[column.name] = decodeValue(column.typeOid, reader.utf8(length));
+      continue;
+    }
+    if (kind === 'b') {
+      throw new ReplicationProtocolError({
+        stage: 'tuple',
+        detail: `column "${column.name}" arrived as binary; this decoder only requests text-format values`,
+      });
+    }
+    throw new ReplicationProtocolError({
+      stage: 'tuple',
+      detail: `column "${column.name}" has an unrecognised tuple kind "${kind}"`,
+    });
+  }
+  return row;
+}
+
+/**
+ * Holds the relation cache: postgres sends a `Relation` message once per table per connection and
+ * every later tuple references it by oid, so a decoder instance is per-connection and is thrown
+ * away with it.
+ */
+export class PgOutputDecoder {
+  readonly #relations = new Map<number, PgRelation>();
+
+  decode(payload: Uint8Array): PgOutputMessage {
+    const reader = new ByteReader(payload, 'pgoutput');
+    const tag = reader.tag();
+    switch (tag) {
+      case 'B':
+        return this.#decodeBegin(reader);
+      case 'C':
+        return this.#decodeCommit(reader);
+      case 'R':
+        return this.#decodeRelation(reader);
+      case 'I':
+        return this.#decodeInsert(reader);
+      case 'U':
+        return this.#decodeUpdate(reader);
+      case 'D':
+        return this.#decodeDelete(reader);
+      case 'T':
+        return this.#decodeTruncate(reader);
+      // 'O' (origin), 'Y' (type), 'M' (logical message), and any tag a newer server invents:
+      // nothing downstream needs them decoded, and guessing at an unknown tag's shape is how a
+      // truncated read turns into a silent misread instead of a clean skip.
+      default:
+        return { kind: 'other', tag };
+    }
+  }
+
+  /** The relation behind an oid, for a caller that needs it after the fact. */
+  relation(oid: number): PgRelation | undefined {
+    return this.#relations.get(oid);
+  }
+
+  #relationOrThrow(oid: number): PgRelation {
+    const relation = this.#relations.get(oid);
+    if (relation === undefined) {
+      throw new ReplicationProtocolError({
+        stage: 'tuple',
+        detail: `no Relation message has been seen yet for oid ${oid}`,
+      });
+    }
+    return relation;
+  }
+
+  #decodeBegin(reader: ByteReader): PgOutputMessage {
+    const commitLsn = reader.int64();
+    const commitAt = pgTimestampToEpochMs(reader.int64());
+    const xid = reader.int32();
+    return { kind: 'begin', commitLsn, commitAt, xid };
+  }
+
+  #decodeCommit(reader: ByteReader): PgOutputMessage {
+    reader.uint8(); // flags: reserved by the protocol, unused today.
+    const commitLsn = reader.int64();
+    const endLsn = reader.int64();
+    const commitAt = pgTimestampToEpochMs(reader.int64());
+    return { kind: 'commit', commitLsn, endLsn, commitAt };
+  }
+
+  #decodeRelation(reader: ByteReader): PgOutputMessage {
+    const oid = reader.int32();
+    const schema = reader.cstring();
+    const name = reader.cstring();
+    const replicaIdentity = reader.tag();
+    const columnCount = reader.int16();
+    const columns: PgColumn[] = [];
+    for (let i = 0; i < columnCount; i += 1) {
+      const flags = reader.uint8();
+      const columnName = reader.cstring();
+      const typeOid = reader.int32();
+      const typeMod = reader.int32();
+      columns.push({ key: (flags & 1) === 1, name: columnName, typeOid, typeMod });
+    }
+    // Always overwrites: postgres re-sends a Relation after a DDL change, and a stale column
+    // list would silently mis-name every later value decoded against this oid.
+    const relation: PgRelation = { oid, schema, name, replicaIdentity, columns };
+    this.#relations.set(oid, relation);
+    return { kind: 'relation', relation };
+  }
+
+  #decodeInsert(reader: ByteReader): PgOutputMessage {
+    const relation = this.#relationOrThrow(reader.int32());
+    reader.tag(); // always 'N' — a new row has no other kind.
+    const after = decodeTupleData(reader, relation);
+    return { kind: 'insert', relation, after };
+  }
+
+  #decodeUpdate(reader: ByteReader): PgOutputMessage {
+    const relation = this.#relationOrThrow(reader.int32());
+    let marker = reader.tag();
+    let before: JsonObject | null = null;
+    if (marker === 'K' || marker === 'O') {
+      before = decodeTupleData(reader, relation);
+      marker = reader.tag();
+    }
+    if (marker !== 'N') {
+      throw new ReplicationProtocolError({
+        stage: 'update',
+        detail: `expected the new-tuple marker "N" but got "${marker}"`,
+      });
+    }
+    const after = decodeTupleData(reader, relation);
+    return { kind: 'update', relation, before, after };
+  }
+
+  #decodeDelete(reader: ByteReader): PgOutputMessage {
+    const relation = this.#relationOrThrow(reader.int32());
+    const marker = reader.tag();
+    if (marker !== 'K' && marker !== 'O') {
+      throw new ReplicationProtocolError({
+        stage: 'delete',
+        detail: `expected the old-tuple marker "K" or "O" but got "${marker}"`,
+      });
+    }
+    const before = decodeTupleData(reader, relation);
+    return { kind: 'delete', relation, before };
+  }
+
+  #decodeTruncate(reader: ByteReader): PgOutputMessage {
+    const count = reader.int32();
+    reader.uint8(); // flags: CASCADE / RESTART IDENTITY bits — advisory, not modelled downstream.
+    const relations: PgRelation[] = [];
+    for (let i = 0; i < count; i += 1) {
+      const relation = this.#relations.get(reader.int32());
+      // An oid with no cached Relation is skipped, not fatal: truncate is advisory for us.
+      if (relation !== undefined) relations.push(relation);
+    }
+    return { kind: 'truncate', relations };
+  }
+}

package/src/policy-gate.ts

@@ -0,0 +1,56 @@
+// The single seam between realtime and authz. It goes through `@ultimat3/query`'s `guard`, which
+// is itself the only point of contact with `@ultimat3/policy` — one authz system, never two, and
+// realtime does not get its own opinion about what a decision means.
+//
+// The row gate turns a denial into "not visible" instead of an error: a row that fails an actor's
+// policy is dropped, never sent. That is the rule from the live-query pipeline, implemented once.
+
+import type { Actor, Ctx } from '@ultimat3/core';
+import { guard, type QueryPolicy, type QuerySubject } from '@ultimat3/query';
+import type { JsonValue, Row } from './json';
+
+export interface GateOptions {
+  /** Query name, for the denial reason and the policy trace. */
+  readonly query: string;
+  readonly ctx: Ctx;
+}
+
+/** Subscribe-time gate for `LiveQueryDefinition.authorize`. Throws the policy's denial error. */
+export function authorizeWithPolicy(
+  policy: QueryPolicy,
+  options: GateOptions,
+): (args: { actor: Actor | null; input: JsonValue }) => Promise<void> {
+  return async (args) => {
+    // No row exists yet at subscribe time; `null` says so rather than leaving the predicate
+    // to infer it from an absent field.
+    await guard(policy, subjectOf(options, args.actor, args.input, null), 'live');
+  };
+}
+
+/**
+ * Row gate for `LiveQueryDefinition.visible`. Called once per subscriber per row — never once per
+ * query. The row travels as `row`, the same field an HTTP or job row rule reads: a predicate is
+ * written once as `({ actor, row }) => …` and works on every surface.
+ */
+export function visibleWithPolicy<R extends Row = Row>(
+  policy: QueryPolicy,
+  options: GateOptions,
+): (args: { actor: Actor | null; row: R; input: JsonValue }) => Promise<boolean> {
+  return async (args) => {
+    try {
+      await guard(policy, subjectOf(options, args.actor, args.input, args.row), 'live');
+      return true;
+    } catch {
+      return false;
+    }
+  };
+}
+
+function subjectOf(
+  options: GateOptions,
+  actor: Actor | null,
+  input: unknown,
+  row: unknown,
+): QuerySubject {
+  return { actor, input, row, ctx: options.ctx, query: options.query };
+}

package/src/presence.ts

@@ -0,0 +1,219 @@
+// Tier 1: presence. Who is here, where their cursor is, what they are typing.
+//
+// Presence lives in `transport.shared`, never in a node's heap: when a `sync` node dies its members
+// simply stop heartbeating and expire, and every other node already sees the same set. Ephemeral
+// state is never modelled as rows — that rule is what keeps presence off the write path entirely.
+
+import { type Clock, systemClock } from '@ultimat3/core';
+import type { ChannelHub, Topic } from './channel';
+import type { Transport } from './fanout';
+import type { JsonObject } from './json';
+import { type Frame, PROTOCOL_VERSION, type PresenceMember } from './sync-protocol';
+
+export const PRESENCE_KEY_PREFIX = 'presence';
+
+export interface PresenceOptions {
+  readonly transport: Transport;
+  /** Optional: without a hub, presence is queryable but silent (no join/leave frames). */
+  readonly hub?: ChannelHub;
+  readonly clock?: Clock;
+  /** Member TTL. Clients should heartbeat at ttl/3 so one lost beat is not a false leave. */
+  readonly ttlMs?: number;
+}
+
+export interface PresenceInput {
+  readonly id: string;
+  readonly actorId: string | null;
+  readonly meta?: JsonObject;
+  /** Logical time from the member. Ties and older writes are dropped — last write wins. */
+  readonly updatedAt?: number;
+}
+
+export class PresenceRegistry {
+  readonly #transport: Transport;
+  readonly #hub: ChannelHub | undefined;
+  readonly #clock: Clock;
+  readonly #ttlMs: number;
+  /** Diffing cache only — the truth is always `transport.shared`. Safe to lose. */
+  readonly #seen = new Map<string, Set<string>>();
+
+  constructor(options: PresenceOptions) {
+    this.#transport = options.transport;
+    this.#hub = options.hub;
+    this.#clock = options.clock ?? systemClock;
+    this.#ttlMs = options.ttlMs ?? 30_000;
+  }
+
+  get ttlMs(): number {
+    return this.#ttlMs;
+  }
+
+  /** Recommended client heartbeat interval: one lost beat must not read as a leave. */
+  get heartbeatMs(): number {
+    return Math.max(1_000, Math.floor(this.#ttlMs / 3));
+  }
+
+  async join(name: Topic, input: PresenceInput): Promise<readonly PresenceMember[]> {
+    const member: PresenceMember = {
+      id: input.id,
+      actorId: input.actorId,
+      meta: input.meta ?? {},
+      updatedAt: input.updatedAt ?? this.#clock.now().getTime(),
+    };
+    await this.#write(name, member);
+    this.#track(name).add(member.id);
+    await this.#emit(name, 'join', [member]);
+    return await this.list(name);
+  }
+
+  /** `false` means the member had already expired: the caller must `join` again, not `heartbeat`. */
+  async heartbeat(name: Topic, id: string): Promise<boolean> {
+    const alive = await this.#transport.shared.touch(this.#key(name), id, this.#ttlMs);
+    if (!alive) this.#track(name).delete(id);
+    return alive;
+  }
+
+  /** Last-write-wins per member: an out-of-order cursor update is dropped, never merged. */
+  async update(name: Topic, input: PresenceInput): Promise<PresenceMember | null> {
+    const current = await this.#find(name, input.id);
+    const updatedAt = input.updatedAt ?? this.#clock.now().getTime();
+    if (current && updatedAt <= current.updatedAt) return current;
+    const member: PresenceMember = {
+      id: input.id,
+      actorId: input.actorId,
+      meta: input.meta ?? {},
+      updatedAt,
+    };
+    await this.#write(name, member);
+    this.#track(name).add(member.id);
+    await this.#emit(name, current ? 'update' : 'join', [member]);
+    return member;
+  }
+
+  async leave(name: Topic, id: string): Promise<void> {
+    const current = await this.#find(name, id);
+    await this.#transport.shared.drop(this.#key(name), id);
+    this.#track(name).delete(id);
+    await this.#emit(
+      name,
+      'leave',
+      current
+        ? [current]
+        : [{ id, actorId: null, meta: {}, updatedAt: this.#clock.now().getTime() }],
+    );
+  }
+
+  async list(name: Topic): Promise<readonly PresenceMember[]> {
+    const entries = await this.#transport.shared.entries(this.#key(name));
+    const members: PresenceMember[] = [];
+    for (const entry of entries) {
+      const member = parseMember(entry.member, entry.value);
+      if (member) members.push(member);
+    }
+    members.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+    return members;
+  }
+
+  /** Turns TTL expiry into explicit `leave` frames. Called on an interval by the `sync` node. */
+  async sweep(name: Topic): Promise<readonly PresenceMember[]> {
+    const live = await this.list(name);
+    const liveIds = new Set(live.map((member) => member.id));
+    const tracked = this.#track(name);
+    const gone: PresenceMember[] = [];
+    for (const id of tracked) {
+      if (!liveIds.has(id)) {
+        tracked.delete(id);
+        gone.push({ id, actorId: null, meta: {}, updatedAt: this.#clock.now().getTime() });
+      }
+    }
+    for (const id of liveIds) tracked.add(id);
+    if (gone.length > 0) await this.#emit(name, 'leave', gone);
+    return gone;
+  }
+
+  /**
+   * Every topic this node has seen, in one pass. Expiry is silent by design — a member whose node
+   * died simply stops heartbeating — so with nothing sweeping, the survivors keep rendering a
+   * cursor that stopped moving until they reconnect. The `sync` node calls this on an interval;
+   * nothing else is in a position to.
+   *
+   * Why an interval and not a call on demand: `sweep` can only report a member it has already
+   * seen, and a member that joined on *another* node is first seen by a sweep. One pass while it
+   * is alive is what makes the next pass able to say it left.
+   */
+  async sweepAll(): Promise<readonly PresenceMember[]> {
+    const gone: PresenceMember[] = [];
+    for (const name of [...this.#seen.keys()] as Topic[]) {
+      gone.push(...(await this.sweep(name)));
+      // A room nobody is in is not a room. Without this the cache keeps one entry per topic ever
+      // subscribed to, for the life of the process, and the sweep walks all of them forever.
+      if ((this.#seen.get(name)?.size ?? 0) === 0) this.#seen.delete(name);
+    }
+    return gone;
+  }
+
+  /** Full-set frame for a client that just (re)connected — presence has no delta protocol. */
+  async syncFrame(name: Topic): Promise<Frame> {
+    return presenceFrame(name, 'sync', await this.list(name));
+  }
+
+  #key(name: Topic): string {
+    return `${PRESENCE_KEY_PREFIX}.${name}`;
+  }
+
+  #track(name: Topic): Set<string> {
+    const existing = this.#seen.get(name);
+    if (existing) return existing;
+    const created = new Set<string>();
+    this.#seen.set(name, created);
+    return created;
+  }
+
+  async #write(name: Topic, member: PresenceMember): Promise<void> {
+    const value = JSON.stringify({
+      actorId: member.actorId,
+      meta: member.meta,
+      updatedAt: member.updatedAt,
+    });
+    await this.#transport.shared.put(this.#key(name), member.id, value, this.#ttlMs);
+  }
+
+  async #find(name: Topic, id: string): Promise<PresenceMember | null> {
+    const entries = await this.#transport.shared.entries(this.#key(name));
+    const entry = entries.find((candidate) => candidate.member === id);
+    return entry ? parseMember(entry.member, entry.value) : null;
+  }
+
+  async #emit(
+    name: Topic,
+    op: 'join' | 'leave' | 'update',
+    members: readonly PresenceMember[],
+  ): Promise<void> {
+    if (!this.#hub) return;
+    await this.#hub.publishFrame(name, presenceFrame(name, op, members));
+  }
+}
+
+export function presenceFrame(
+  name: Topic,
+  op: 'join' | 'leave' | 'update' | 'sync',
+  members: readonly PresenceMember[],
+): Frame {
+  return { type: 'presence', v: PROTOCOL_VERSION, topic: name, op, members };
+}
+
+function parseMember(id: string, value: string): PresenceMember | null {
+  try {
+    const parsed: unknown = JSON.parse(value);
+    if (typeof parsed !== 'object' || parsed === null) return null;
+    const shape = parsed as { actorId?: unknown; meta?: unknown; updatedAt?: unknown };
+    return {
+      id,
+      actorId: typeof shape.actorId === 'string' ? shape.actorId : null,
+      meta: typeof shape.meta === 'object' && shape.meta !== null ? (shape.meta as JsonObject) : {},
+      updatedAt: typeof shape.updatedAt === 'number' ? shape.updatedAt : 0,
+    };
+  } catch {
+    return null;
+  }
+}