@ultimat3/realtime 1.0.0

package/src/pg-connection.ts

@@ -0,0 +1,337 @@
+// Single responsibility: one Postgres session over a `PgStream` — startup, authentication, simple
+// queries, and the CopyBoth switch that a replication stream lives inside. It speaks only in
+// messages, never in bytes on a socket, so the whole handshake is driven by hand in the tests.
+
+import { logger } from '@ultimat3/core';
+import { ReplicationFailedError, ReplicationProtocolError } from './errors';
+import {
+  chooseMechanism,
+  md5Password,
+  type ScramSession,
+  scramNonce,
+  scramSession,
+} from './pg-auth';
+import { ByteReader } from './pg-bytes';
+import {
+  copyDoneMessage,
+  frame,
+  MessageReader,
+  type PgMessage,
+  type PgStream,
+  passwordMessage,
+  queryMessage,
+  responseFields,
+  saslInitialResponse,
+  saslResponse,
+  serverError,
+  startupMessage,
+  terminateMessage,
+} from './pg-wire';
+import type { Rng } from './thundering-herd';
+
+export interface PgConnectionOptions {
+  readonly stream: PgStream;
+  readonly user: string;
+  readonly password?: string | undefined;
+  readonly database: string;
+  /**
+   * `database` opens a logical-replication walsender that *also* answers ordinary SQL — which is
+   * what lets one connection check `wal_level` and then stream from a slot.
+   */
+  readonly replication?: 'database' | undefined;
+  readonly applicationName?: string | undefined;
+  /** Injected so the SCRAM nonce is deterministic under a seeded test. */
+  readonly rng?: Rng | undefined;
+}
+
+/** A result set as text, exactly as the wire carries it. `null` is SQL NULL, never `''`. */
+export type PgRows = readonly (readonly (string | null)[])[];
+
+const AUTH_OK = 0;
+const AUTH_CLEARTEXT = 3;
+const AUTH_MD5 = 5;
+const AUTH_SASL = 10;
+const AUTH_SASL_CONTINUE = 11;
+const AUTH_SASL_FINAL = 12;
+
+const needPassword = (method: string): ReplicationFailedError =>
+  new ReplicationFailedError({
+    stage: 'auth',
+    detail: `the server asked for ${method} but the replication URL carries no password`,
+    fix: 'put the credentials in the URL: postgres://user:password@host:5432/db',
+  });
+
+export class PgConnection {
+  readonly #stream: PgStream;
+  readonly #reader: MessageReader;
+  readonly #parameters = new Map<string, string>();
+  #copyBoth = false;
+  #closed = false;
+
+  private constructor(stream: PgStream) {
+    this.#stream = stream;
+    this.#reader = new MessageReader(stream);
+  }
+
+  /** Startup, authentication, and everything up to the first `ReadyForQuery`. */
+  static async open(options: PgConnectionOptions): Promise<PgConnection> {
+    const connection = new PgConnection(options.stream);
+    const parameters: Record<string, string> = {
+      user: options.user,
+      database: options.database,
+      application_name: options.applicationName ?? 'ultimate-replicator',
+    };
+    // A walsender rejects most GUCs, so only the two it accepts are sent.
+    if (options.replication !== undefined) parameters['replication'] = options.replication;
+    // A handshake fails on ordinary conditions — no password, an ErrorResponse, an EOF — and on
+    // every one of them the caller gets an exception instead of an object, so nothing is left
+    // holding the socket. Closing it here is what stops a retrying supervisor from accumulating
+    // one file descriptor per failed attempt.
+    try {
+      await options.stream.write(startupMessage(parameters));
+      await connection.#authenticate(options);
+      await connection.#awaitReady();
+    } catch (failure) {
+      closeQuietly(options.stream);
+      throw failure;
+    }
+    return connection;
+  }
+
+  /** A `ParameterStatus` the server volunteered — `server_version`, `integer_datetimes`, … */
+  parameter(name: string): string | undefined {
+    return this.#parameters.get(name);
+  }
+
+  get inCopyBoth(): boolean {
+    return this.#copyBoth;
+  }
+
+  /** One simple query. Returns the rows as text; a `CommandComplete` with no rows returns `[]`. */
+  async query(sql: string): Promise<PgRows> {
+    await this.#stream.write(queryMessage(sql));
+    const rows: (readonly (string | null)[])[] = [];
+    for (;;) {
+      const message = await this.#expect('query');
+      switch (message.tag) {
+        case 'D':
+          rows.push(dataRow(message.body));
+          break;
+        case 'E':
+          // Drain to `ReadyForQuery` first: leaving the session mid-result desynchronises reuse.
+          await this.#drainToReady();
+          throw serverError('query', message.body);
+        case 'Z':
+          return rows;
+        default:
+          this.#note(message);
+      }
+    }
+  }
+
+  /**
+   * `START_REPLICATION` — the connection stops being request/response and becomes a duplex copy
+   * stream. There is no way back short of closing it, which is why this is one-way on purpose.
+   */
+  async startCopyBoth(sql: string): Promise<void> {
+    await this.#stream.write(queryMessage(sql));
+    for (;;) {
+      const message = await this.#expect('start-replication');
+      if (message.tag === 'W') {
+        this.#copyBoth = true;
+        return;
+      }
+      if (message.tag === 'E') throw serverError('start-replication', message.body);
+      this.#note(message);
+    }
+  }
+
+  /** The next `CopyData` payload, or `undefined` when the server ended the stream. */
+  async nextCopyData(): Promise<Uint8Array | undefined> {
+    for (;;) {
+      const message = await this.#reader.next();
+      if (message === undefined) return undefined;
+      switch (message.tag) {
+        case 'd':
+          return message.body;
+        case 'c':
+          this.#copyBoth = false;
+          return undefined;
+        case 'E':
+          throw serverError('stream', message.body);
+        default:
+          this.#note(message);
+      }
+    }
+  }
+
+  /** Frontend `CopyData` — how a standby status update reaches the walsender. */
+  async sendCopyData(payload: Uint8Array): Promise<void> {
+    await this.#stream.write(frame('d', payload));
+  }
+
+  /**
+   * End the copy stream from this side. Saying so is what lets the server release the slot at
+   * once; dropping the socket instead leaves it `active` until the backend notices.
+   */
+  async endCopy(): Promise<void> {
+    if (!this.#copyBoth) return;
+    await this.#stream.write(copyDoneMessage());
+  }
+
+  async close(): Promise<void> {
+    if (this.#closed) return;
+    this.#closed = true;
+    // Best effort: a walsender that already died does not need to hear the goodbye.
+    try {
+      await this.#stream.write(terminateMessage());
+    } catch {
+      // fall through to the socket close below
+    }
+    this.#stream.close();
+  }
+
+  async #authenticate(options: PgConnectionOptions): Promise<void> {
+    let scram: ScramSession | undefined;
+    for (;;) {
+      const message = await this.#expect('auth');
+      if (message.tag === 'E') throw serverError('auth', message.body);
+      if (message.tag !== 'R') {
+        this.#note(message);
+        continue;
+      }
+      const reader = new ByteReader(message.body, 'auth');
+      const method = reader.int32();
+      switch (method) {
+        case AUTH_OK:
+          return;
+        case AUTH_CLEARTEXT: {
+          const password = options.password ?? '';
+          if (password === '') throw needPassword('a cleartext password');
+          await this.#stream.write(passwordMessage(password));
+          break;
+        }
+        case AUTH_MD5: {
+          const password = options.password ?? '';
+          if (password === '') throw needPassword('an md5 password');
+          const salt = reader.take(4);
+          await this.#stream.write(
+            passwordMessage(md5Password({ user: options.user, password, salt })),
+          );
+          break;
+        }
+        case AUTH_SASL: {
+          const password = options.password ?? '';
+          if (password === '') throw needPassword('a SCRAM password');
+          const mechanism = chooseMechanism(mechanisms(reader));
+          // No `?? Math.random`: an absent `rng` must reach `scramNonce`'s CSPRNG default, which
+          // is the only source RFC 5802 allows for a client nonce.
+          scram = scramSession({ password, nonce: scramNonce(options.rng) });
+          await this.#stream.write(saslInitialResponse(mechanism, scram.clientFirst()));
+          break;
+        }
+        case AUTH_SASL_CONTINUE: {
+          if (scram === undefined) throw outOfOrder('SASLContinue');
+          await this.#stream.write(saslResponse(await scram.clientFinal(reader.rest())));
+          break;
+        }
+        case AUTH_SASL_FINAL: {
+          if (scram === undefined) throw outOfOrder('SASLFinal');
+          await scram.verify(reader.rest());
+          break;
+        }
+        default:
+          throw new ReplicationProtocolError({
+            stage: 'auth',
+            detail: `the server asked for authentication method ${method}, which this client does not speak`,
+            fix: 'set password_encryption = scram-sha-256 and give the replication role a password',
+          });
+      }
+    }
+  }
+
+  /** Everything between `AuthenticationOk` and the first `ReadyForQuery` is session metadata. */
+  async #awaitReady(): Promise<void> {
+    for (;;) {
+      const message = await this.#expect('startup');
+      if (message.tag === 'Z') return;
+      if (message.tag === 'E') throw serverError('startup', message.body);
+      this.#note(message);
+    }
+  }
+
+  async #drainToReady(): Promise<void> {
+    for (;;) {
+      const message = await this.#reader.next();
+      if (message === undefined || message.tag === 'Z') return;
+    }
+  }
+
+  async #expect(stage: string): Promise<PgMessage> {
+    const message = await this.#reader.next();
+    if (message !== undefined) return message;
+    throw new ReplicationFailedError({
+      stage,
+      detail:
+        'the server closed the connection without answering — pg_hba.conf needs a ' +
+        '"host replication <user> <cidr> scram-sha-256" line before it will hold one open',
+      fix: 'psql "$DATABASE_URL" -c "SELECT pg_reload_conf()" -c "TABLE pg_hba_file_rules"',
+    });
+  }
+
+  /** Messages that carry session state or server chatter, in one place so nothing is dropped. */
+  #note(message: PgMessage): void {
+    switch (message.tag) {
+      case 'S': {
+        const reader = new ByteReader(message.body, 'parameter-status');
+        this.#parameters.set(reader.cstring(), reader.cstring());
+        return;
+      }
+      case 'N':
+        logger.warn('postgres notice', responseFields(message.body));
+        return;
+      // BackendKeyData, RowDescription, CommandComplete, EmptyQuery, NoticeResponse, Notification:
+      // nothing downstream reads them, and dropping them silently is the point of this branch.
+      default:
+        return;
+    }
+  }
+}
+
+const outOfOrder = (what: string): ReplicationProtocolError =>
+  new ReplicationProtocolError({
+    stage: 'auth',
+    detail: `the server sent ${what} before it offered a SASL mechanism`,
+    fix: 'x doctor db — the SASL exchange arrived out of order; check for a pooler or proxy between this client and postgres',
+  });
+
+/** A `close()` that throws must not replace the handshake failure that is worth reporting. */
+const closeQuietly = (stream: PgStream): void => {
+  try {
+    stream.close();
+  } catch {
+    // the original failure is the one the caller needs
+  }
+};
+
+/** The mechanism list is cstrings until an empty one. */
+const mechanisms = (reader: ByteReader): readonly string[] => {
+  const offered: string[] = [];
+  for (;;) {
+    const name = reader.cstring();
+    if (name === '') return offered;
+    offered.push(name);
+  }
+};
+
+/** `DataRow`: Int16 column count, then Int32 length (-1 = NULL) + that many bytes, per column. */
+const dataRow = (body: Uint8Array): readonly (string | null)[] => {
+  const reader = new ByteReader(body, 'data-row');
+  const count = reader.int16();
+  const values: (string | null)[] = [];
+  for (let index = 0; index < count; index += 1) {
+    const length = reader.int32();
+    values.push(length < 0 ? null : reader.utf8(length));
+  }
+  return values;
+};

package/src/pg-entity-row.ts

@@ -0,0 +1,130 @@
+// Physical Postgres row -> entity-row shaping: snake_case columns become camelCase properties,
+// and a `<p>_minor` / `<p>_currency` column pair folds into one `<p>: Money`-shaped property.
+// The inverse of the camelCasing here is `@ultimat3/entity`'s `column.ts#snake` — not imported
+// (a tier-3 package may not reach across to tier-2), so the round trip is pinned by a test instead.
+
+import { ReplicationProtocolError } from './errors';
+import type { JsonObject, JsonValue } from './json';
+
+/** `org_id` -> `orgId`, `published_at` -> `publishedAt`. The inverse of `@ultimat3/entity`'s `snake()`. */
+export function camel(column: string): string {
+  const [head = '', ...tail] = column.split('_');
+  return head + tail.map(capitalize).join('');
+}
+
+/** A leading, trailing, or doubled underscore produces an empty part; it contributes nothing. */
+function capitalize(part: string): string {
+  return part.charAt(0).toUpperCase() + part.slice(1);
+}
+
+interface MoneyPair {
+  readonly property: string;
+  readonly minorKey: string;
+  readonly currencyKey: string;
+  readonly minor: number;
+  readonly currency: string;
+}
+
+/** `price_minor` / `price_currency` -> `price`; any other column name has no money prefix. */
+function moneyPrefix(column: string): string | null {
+  if (column.endsWith('_minor')) return column.slice(0, -'_minor'.length);
+  if (column.endsWith('_currency')) return column.slice(0, -'_currency'.length);
+  return null;
+}
+
+/**
+ * `Money` is `{ minor: number; currency: string }` everywhere in the framework, so `minor` is
+ * normalised here rather than passed through: `pgoutput` decodes an int8 as text once it leaves
+ * `Number.isSafeInteger` range and a numeric as text always, which would otherwise make one column
+ * a number on one row and a string on the next. A value no JS number holds exactly is not money
+ * this pipeline can carry, and saying so is better than shipping a `minor` the contract forbids.
+ */
+function moneyMinor(column: string, value: number | string): number {
+  const minor =
+    typeof value === 'number' ? value : /^-?\d+$/.test(value) ? Number(value) : Number.NaN;
+  if (Number.isSafeInteger(minor)) return minor;
+  throw new ReplicationProtocolError({
+    stage: 'value',
+    detail: `column "${column}" carries "${value}", which is not a whole number of minor units`,
+    fix: `store ${column} as a bigint inside ±2^53 — Money.minor is a number, never a float or a bigint`,
+  });
+}
+
+/**
+ * `name` is one half of a `<p>_minor` / `<p>_currency` pair, or null if it is not part of one.
+ * Both halves must be present *and* typed like money — a null currency (an unset money value) is
+ * not "half a pair", it simply is not a pair, so both columns fall through as ordinary values.
+ */
+function moneyPairAt(
+  physical: Readonly<Record<string, JsonValue>>,
+  name: string,
+): MoneyPair | null {
+  const prefix = moneyPrefix(name);
+  if (prefix === null) return null;
+
+  const minorKey = `${prefix}_minor`;
+  const currencyKey = `${prefix}_currency`;
+  if (!Object.hasOwn(physical, minorKey) || !Object.hasOwn(physical, currencyKey)) return null;
+
+  const minor = physical[minorKey];
+  const currency = physical[currencyKey];
+  if ((typeof minor === 'number' || typeof minor === 'string') && typeof currency === 'string') {
+    return {
+      property: camel(prefix),
+      minorKey,
+      currencyKey,
+      minor: moneyMinor(minorKey, minor),
+      currency,
+    };
+  }
+  return null;
+}
+
+/**
+ * `camel()` is not injective — `a_b` and `a__b` both give `aB`, `x` and `x_` both give `x` — and
+ * `entityRow` writes by assignment, so without this the second column would silently overwrite the
+ * first and the row would be short one value with nothing to read about it.
+ */
+function claim(taken: Map<string, string>, property: string, column: string): void {
+  const first = taken.get(property);
+  if (first !== undefined) {
+    throw new ReplicationProtocolError({
+      stage: 'value',
+      detail: `columns "${first}" and "${column}" both map to the entity property "${property}"`,
+      fix: `rename one of them — two columns cannot share one property once camelCased`,
+    });
+  }
+  taken.set(property, column);
+}
+
+/**
+ * A physical postgres row -> the row shape the rest of the pipeline is written against.
+ * Two things are not one-to-one and both live here: the column is snake_case while the entity
+ * property is camelCase, and money is one property over the two columns `<p>_minor`/`<p>_currency`.
+ */
+export function entityRow(physical: Readonly<Record<string, JsonValue>>): JsonObject {
+  const row: JsonObject = {};
+  // Column order in is key order out; a folded money property lands wherever its earlier half
+  // (whichever of _minor/_currency the source happened to emit first) would otherwise have sat.
+  const consumed = new Set<string>();
+  // Which column produced each property, so a collision names both sides rather than losing one.
+  const taken = new Map<string, string>();
+
+  for (const name of Object.keys(physical)) {
+    if (consumed.has(name)) continue;
+
+    const money = moneyPairAt(physical, name);
+    if (money !== null) {
+      claim(taken, money.property, `${money.minorKey}/${money.currencyKey}`);
+      row[money.property] = { minor: money.minor, currency: money.currency };
+      consumed.add(money.minorKey);
+      consumed.add(money.currencyKey);
+      continue;
+    }
+
+    const property = camel(name);
+    claim(taken, property, name);
+    row[property] = physical[name] ?? null;
+  }
+  return row;
+}