@ultimat3/db 22.14.0 → 23.0.0

package/src/introspect-catalog.ts

@@ -0,0 +1,154 @@
+// Single responsibility: read the WHOLE schema out of `pg_catalog` — tables and everything that is
+// not a table — into one `CatalogDescription`, sorted in JS. `introspect()` beside it stays the
+// entity-vocabulary reading drift compares to a snapshot; this is the catalog's own spelling, the
+// input of the schema dump, and comparable only to another reading of itself.
+
+import type { CatalogDescription, CatalogType } from './catalog';
+import { by, sequenceOf, tableOf } from './catalog-fold';
+import {
+  domainCheckRows,
+  domainRows,
+  enumRows,
+  extensionRows,
+  functionRows,
+  triggerRows,
+  unrenderedRows,
+  viewRows,
+} from './catalog-objects';
+import {
+  columnRows,
+  constraintRows,
+  indexRows,
+  sequenceRows,
+  tableRows,
+} from './catalog-relations';
+import { type DbClient, db } from './client';
+
+export interface IntrospectCatalogOptions {
+  readonly client?: DbClient | undefined;
+  readonly schema?: string | undefined;
+}
+
+async function typesOf(client: DbClient, schema: string): Promise<readonly CatalogType[]> {
+  const labels = await enumRows(client, schema);
+  const domains = await domainRows(client, schema);
+  const checks = await domainCheckRows(client, schema);
+  const enums = [...new Set(labels.map((row) => row.name))].map(
+    (name): CatalogType => ({
+      kind: 'enum',
+      name,
+      labels: labels
+        .filter((row) => row.name === name)
+        .sort((a, b) => a.position - b.position)
+        .map((row) => row.label),
+    }),
+  );
+  const described = domains.map(
+    (row): CatalogType => ({
+      kind: 'domain',
+      name: row.name,
+      baseType: row.base_type,
+      notNull: row.not_null,
+      default: row.expression,
+      checks: checks
+        .filter((check) => check.domain_name === row.name)
+        .sort(by((check) => check.name))
+        .map((check) => ({ name: check.name, definition: check.definition })),
+    }),
+  );
+  return [...enums, ...described].sort(by((type) => type.name));
+}
+
+/**
+ * Eleven round trips, sequential: a pinned transaction connection answers one statement at a
+ * time, and this runs once per `x db gen`, never per request.
+ */
+export async function introspectCatalog(
+  options: IntrospectCatalogOptions = {},
+): Promise<CatalogDescription> {
+  const client = options.client ?? db();
+  const schema = options.schema ?? 'public';
+  const tables = await tableRows(client, schema);
+  const columns = await columnRows(client, schema);
+  const constraints = await constraintRows(client, schema);
+  const sequences = await sequenceRows(client, schema);
+  const indexes = await indexRows(client, schema);
+  const views = await viewRows(client, schema);
+  const functions = await functionRows(client, schema);
+  const triggers = await triggerRows(client, schema);
+  const unrendered = await unrenderedRows(client, schema);
+  const known = new Set(tables.map((table) => table.name));
+  const materialized = new Set(views.filter((view) => view.kind === 'm').map((view) => view.name));
+
+  return {
+    schema,
+    extensions: (await extensionRows(client))
+      .map((row) => ({ name: row.name }))
+      .sort(by((extension) => extension.name)),
+    types: await typesOf(client, schema),
+    sequences: sequences
+      .filter((sequence) => sequence.ownership === null)
+      .map(sequenceOf)
+      .sort(by((sequence) => sequence.name)),
+    tables: tables
+      .map((table) => tableOf(table, columns, constraints, sequences))
+      .sort(by((table) => table.name)),
+    // An index on a relation this reading does not render (a partition) has nowhere to load.
+    indexes: indexes
+      .filter((index) => known.has(index.table_name) || materialized.has(index.table_name))
+      .map((row) => ({ table: row.table_name, name: row.name, definition: row.definition }))
+      .sort(
+        by(
+          (index) => index.table,
+          (index) => index.name,
+        ),
+      ),
+    foreignKeys: constraints
+      .filter((constraint) => constraint.type === 'f' && known.has(constraint.table_name))
+      .map((row) => ({ table: row.table_name, name: row.name, definition: row.definition }))
+      .sort(
+        by(
+          (key) => key.table,
+          (key) => key.name,
+        ),
+      ),
+    views: views
+      .map((row) => ({
+        name: row.name,
+        materialized: row.kind === 'm',
+        options: row.options,
+        definition: row.definition,
+      }))
+      .sort(by((view) => view.name)),
+    functions: functions
+      .map((row) => ({ name: row.name, arguments: row.arguments, definition: row.definition }))
+      .sort(
+        by(
+          (fn) => fn.name,
+          (fn) => fn.arguments,
+        ),
+      ),
+    triggers: triggers
+      .map((row) => ({
+        table: row.table_name,
+        name: row.name,
+        definition: row.definition,
+        enabled: row.enabled,
+      }))
+      .sort(
+        by(
+          (trigger) => trigger.table,
+          (trigger) => trigger.name,
+        ),
+      ),
+    unrendered: unrendered
+      .map((row) => ({ kind: row.kind, name: row.name, table: row.table_name }))
+      .sort(
+        by(
+          (object) => object.kind,
+          (object) => object.table ?? '',
+          (object) => object.name,
+        ),
+      ),
+  };
+}

package/src/listen.ts

@@ -0,0 +1,62 @@
+// Single responsibility: the LISTEN seam — the one capability a pooled statement cannot carry. A
+// subscription is a property of a SESSION, and a pooled client has none to name: the next
+// statement runs on another connection. So the client holds one connection of its own for it,
+// outside the pool, and this file is the shape both drivers answer with.
+
+import { DbError } from './errors';
+
+/** One live `LISTEN`. Idempotent: a second `unlisten()` is the first one's promise. */
+export interface DbSubscription {
+  unlisten(): Promise<void>;
+}
+
+export interface ListeningClient {
+  /**
+   * `LISTEN <channel>` on a connection this client owns for it — never one out of the pool.
+   *
+   * `onListening` fires every time the subscription is (re-)established: once when this resolves,
+   * and again after the driver has re-dialled a connection that died. A notification sent while
+   * it was down is LOST — Postgres queues nothing for a session that is gone — so a caller that
+   * must not miss one re-reads its source of truth there.
+   *
+   * Not through a transaction-pooling proxy (PgBouncer `pool_mode = transaction`): the `LISTEN`
+   * lands on a server connection the proxy takes back at once, and nothing is ever delivered.
+   * This resolves all the same; only a notification that arrives proves the path.
+   */
+  listen(
+    channel: string,
+    onNotify: (payload: string) => void,
+    onListening?: () => void,
+  ): Promise<DbSubscription>;
+}
+
+export function canListen(client: object): client is ListeningClient {
+  return typeof (client as Partial<ListeningClient>).listen === 'function';
+}
+
+/** What `LISTEN` takes unquoted: a channel is an identifier, and Postgres truncates past 63. */
+const CHANNEL = /^[a-z_][a-z0-9_]{0,62}$/;
+
+/**
+ * Refused before it reaches a driver: PGlite quotes the name and `Bun.SQL` validates it, and two
+ * drivers reading one string two ways is a channel that works under `x dev` and not in production.
+ * `X_SQL_UNSAFE`, the code an identifier that cannot be spliced already answers with
+ * (`identifierUnsafe`): the argument is the fix, never the database's reachability.
+ */
+export function assertListenChannel(channel: string): void {
+  if (CHANNEL.test(channel)) return;
+  throw new DbError({
+    code: 'X_SQL_UNSAFE',
+    cause: 'the LISTEN channel is not a lower-case identifier of at most 63 characters',
+    fix: "pass a channel matching [a-z_][a-z0-9_]*, e.g. client.listen('x_jobs_wake', onNotify)",
+  });
+}
+
+/** The driver has no `listen` — a fake, or a runtime older than the one this package requires. */
+export function listenUnsupported(driver: string): DbError {
+  return new DbError({
+    code: 'X_DB_UNAVAILABLE',
+    cause: `${driver} has no listen(), so this client cannot hold a LISTEN`,
+    fix: 'bun upgrade   # Bun.SQL.listen ships with the Bun this package requires (>= 1.4.0)',
+  });
+}

package/src/object-drift.ts

@@ -0,0 +1,105 @@
+// Single responsibility: the objects a live database holds that replaying the migrations does not
+// create — a trigger, function, view, type or sequence made by hand. `drift.ts` asks the same
+// question of tables and columns against a snapshot; a snapshot records only what entities declare,
+// so everything else is compared here, catalog against catalog, by identity and never by text.
+
+import type { CatalogDescription } from './catalog';
+import { FRAMEWORK_TABLE_PREFIX } from './drift';
+import type { DriftDifference } from './drift-findings';
+import { shellInertIdentifier } from './sql';
+
+interface ObjectIdentity {
+  /** The word `drop` takes: `trigger`, `function`, `view`, `materialized view`, `type`, `sequence`. */
+  readonly kind: string;
+  readonly name: string;
+  /** What tells two objects of one name apart: a function's arguments. */
+  readonly signature: string;
+  readonly table: string | null;
+}
+
+/**
+ * Identity only. Two servers spell one definition differently (`pg_get_viewdef` moved between
+ * majors), and the live database is the operator's Postgres while the expected side is a replay
+ * on the embedded one — a text comparison would report every view in a correct database.
+ *
+ * Tables are `diffSchema`'s (`unexpected-table`). Extensions are left out on purpose: an operator
+ * installs `pg_stat_statements` on the server, and that is not the app's schema.
+ */
+function identities(catalog: CatalogDescription): readonly ObjectIdentity[] {
+  const plain = (kind: string, name: string): ObjectIdentity => ({
+    kind,
+    name,
+    signature: '',
+    table: null,
+  });
+  return [
+    ...catalog.types.map((type) => plain('type', type.name)),
+    ...catalog.sequences.map((sequence) => plain('sequence', sequence.name)),
+    ...catalog.views.map((view) =>
+      plain(view.materialized ? 'materialized view' : 'view', view.name),
+    ),
+    ...catalog.functions.map((fn) => ({ ...plain('function', fn.name), signature: fn.arguments })),
+    ...catalog.triggers.map((trigger) => ({
+      ...plain('trigger', trigger.name),
+      table: trigger.table,
+    })),
+  ];
+}
+
+/** What `shellInertIdentifier` refuses in a name, plus the controls a pasted line must not carry. */
+// biome-ignore lint/suspicious/noControlCharactersInRegex: matching control characters is the point.
+const SIGNATURE_ACTIVE = /[`$\\\u0000-\u001f\u007f]/;
+
+const keyOf = (object: ObjectIdentity): string =>
+  [object.kind, object.table ?? '', object.name, object.signature].join('\u0000');
+
+/**
+ * The `drop` comes FIRST in the fix, and that order is the instruction: a migration that creates
+ * an object the database already holds fails on `already exists`, so the hand-made copy has to go
+ * before the migration that owns it can apply.
+ */
+function unexpectedObject(object: ObjectIdentity): DriftDifference {
+  const name = shellInertIdentifier(object.name);
+  const table = object.table === null ? null : shellInertIdentifier(object.table);
+  const where = object.table === null ? '' : ` on table "${object.table}"`;
+  const signature = object.signature === '' ? '' : `(${object.signature})`;
+  // A function is dropped by its argument list, empty included: `drop function "add";` is
+  // `42725 function name is not unique` while an overload lives beside it. The list is catalog
+  // text (`pg_get_function_identity_arguments`), already quoted for SQL, so it is screened for
+  // what a shell or a pasted line would read and never escaped.
+  const args = object.kind === 'function' ? `(${object.signature})` : '';
+  const spellable =
+    name !== null &&
+    (object.table === null || table !== null) &&
+    !SIGNATURE_ACTIVE.test(object.signature);
+  const drop = `drop ${object.kind} ${name}${args}${table === null ? '' : ` on ${table}`};`;
+  return {
+    kind: 'unexpected-object',
+    table: object.table ?? object.name,
+    column: null,
+    cause: `${object.kind} "${object.name}"${signature}${where} exists in this database and no migration creates it`,
+    fix: spellable
+      ? `run ${drop} inside psql "$DATABASE_URL", then write its create statement into a ` +
+        'migration and run x db migrate — or leave it dropped if nothing owns it'
+      : 'drop it by hand, then write its create statement into a migration and run x db migrate ' +
+        '— its name or arguments carry a backtick, a dollar sign, a quote, a backslash or ' +
+        'whitespace, so ' +
+        'no statement here can spell it',
+  };
+}
+
+/**
+ * Everything `live` holds that `expected` does not, framework bookkeeping excluded on the rule
+ * `appTables()` states. One direction only: an object the migrations create and the database
+ * lacks is a migration that has not run, which the ledger already reports.
+ */
+export function unexpectedObjects(
+  live: CatalogDescription,
+  expected: CatalogDescription,
+): readonly DriftDifference[] {
+  const known = new Set(identities(expected).map(keyOf));
+  return identities(live)
+    .filter((object) => !(object.table ?? object.name).startsWith(FRAMEWORK_TABLE_PREFIX))
+    .filter((object) => !known.has(keyOf(object)))
+    .map(unexpectedObject);
+}

package/src/pglite-extensions.ts

@@ -0,0 +1,112 @@
+// Single responsibility: which PGlite bundle a Postgres extension name resolves to, and which
+// names resolve to none. PGlite links an extension only when it is handed over at boot, so every
+// embedded boot — `x dev`'s database and the schema dump's scratch replay — asks this one linker,
+// and a caller that must choose another engine asks it what is `missing`.
+
+import { renderThrowable, stringField } from '@ultimat3/core';
+import { DbError } from './errors';
+import { PGLITE_PACKAGE } from './pglite-package';
+
+export type PgliteExtensionLoader = (specifier: string) => Promise<unknown>;
+
+export interface LinkedExtensions {
+  /** Keyed by the bundle's own export name — what PGlite's `extensions` option takes. */
+  readonly linked: Readonly<Record<string, unknown>>;
+  /** Postgres names no installed bundle answers for, sorted. */
+  readonly missing: readonly string[];
+}
+
+/**
+ * The name reaches a module specifier, and it is DATA — it is read out of an app's migration
+ * text. Lowercase letters, digits, `_` and `-` only, so no name can leave the package.
+ */
+const EXTENSION_NAME = /^[a-z][a-z0-9_-]*$/;
+
+/** `uuid-ossp` ships as `contrib/uuid_ossp`, exporting `uuid_ossp`. */
+export const pgliteExtensionExport = (name: string): string | undefined =>
+  EXTENSION_NAME.test(name) ? name.replaceAll('-', '_') : undefined;
+
+/** Compiled into every Postgres: `create extension plpgsql` needs no bundle and is never missing. */
+const BUILT_IN: ReadonlySet<string> = new Set(['plpgsql']);
+
+/**
+ * Entry points of the package that are not Postgres extensions. Only the second specifier below
+ * could reach one, and `live` exports an object PGlite would accept as a plugin.
+ */
+const NOT_EXTENSIONS: ReadonlySet<string> = new Set([
+  'template',
+  'live',
+  'worker',
+  'nodefs',
+  'opfs_ahp',
+  'basefs',
+]);
+
+/**
+ * `contrib/<name>` is where 0.5 ships every extension it has. `<name>` at the package root is
+ * where earlier and later lines ship the ones that are not contrib (`vector`); asked second, so
+ * an install that has it is linked without this file learning a version table.
+ */
+const specifiersFor = (exported: string): readonly string[] =>
+  NOT_EXTENSIONS.has(exported)
+    ? [`${PGLITE_PACKAGE}/contrib/${exported}`]
+    : [`${PGLITE_PACKAGE}/contrib/${exported}`, `${PGLITE_PACKAGE}/${exported}`];
+
+const importBundle: PgliteExtensionLoader = (specifier) => import(specifier);
+
+/**
+ * What `import()` answers for a path nothing ships — Bun says `ERR_MODULE_NOT_FOUND` for an
+ * unexported subpath too (measured on 1.4, 2026-10-01). Anything else is a bundle that IS there
+ * and failed to evaluate, and reading that as absent would boot without it and lose the reason.
+ */
+const NOT_SHIPPED: ReadonlySet<string> = new Set([
+  'ERR_MODULE_NOT_FOUND',
+  'MODULE_NOT_FOUND',
+  'ERR_PACKAGE_PATH_NOT_EXPORTED',
+]);
+
+const bundleBroken = (specifier: string, sourceError: unknown): DbError =>
+  new DbError({
+    code: 'X_DB_UNAVAILABLE',
+    cause: `${specifier} is installed and failed to load: ${renderThrowable(sourceError)}`,
+    fix: `bun install --force   # reinstall ${PGLITE_PACKAGE}; its extension bundle does not evaluate`,
+    sourceError,
+  });
+
+async function bundleFor(exported: string, load: PgliteExtensionLoader): Promise<unknown> {
+  for (const specifier of specifiersFor(exported)) {
+    let bundle: unknown;
+    try {
+      bundle = await load(specifier);
+    } catch (error) {
+      // Not at this specifier: deliberately not an error — the answer is `missing`, and the
+      // caller decides what an extension the embedded database cannot link means.
+      if (NOT_SHIPPED.has(stringField(error, 'code') ?? '')) continue;
+      throw bundleBroken(specifier, error);
+    }
+    const extension = (bundle as Readonly<Record<string, unknown>> | null | undefined)?.[exported];
+    if (extension !== undefined) return extension;
+  }
+  return undefined;
+}
+
+/**
+ * Resolve every name. Never boots anything: a bundle is a small module beside a tarball PGlite
+ * reads only when `create extension` runs. Throws only for a bundle that is installed and fails
+ * to evaluate (`X_DB_UNAVAILABLE`) — an absent one is `missing`.
+ */
+export async function linkPgliteExtensions(
+  names: readonly string[],
+  load: PgliteExtensionLoader = importBundle,
+): Promise<LinkedExtensions> {
+  const linked: Record<string, unknown> = {};
+  const missing = new Set<string>();
+  for (const name of names) {
+    if (BUILT_IN.has(name)) continue;
+    const exported = pgliteExtensionExport(name);
+    const extension = exported === undefined ? undefined : await bundleFor(exported, load);
+    if (exported === undefined || extension === undefined) missing.add(name);
+    else linked[exported] = extension;
+  }
+  return { linked, missing: [...missing].sort() };
+}

package/src/pglite-package.ts

@@ -0,0 +1,11 @@
+// Single responsibility: the optional peer's NAME. A leaf, so the modules that build specifiers
+// from it (`pglite.ts`, `pglite-extensions.ts`, `pglite-snapshot.ts`) share one spelling without
+// importing each other.
+
+/**
+ * The optional peer's specifier. Exported because `x doctor` asks whether it RESOLVES — a resolve,
+ * never an import, since loading it boots the WASM build and takes the single-writer lock — and a
+ * diagnostic that spelled the package name a second time is a diagnostic that can name the wrong
+ * one after a rename.
+ */
+export const PGLITE_PACKAGE = '@electric-sql/pglite';

package/src/pglite-snapshot.ts

@@ -0,0 +1,121 @@
+// Single responsibility: the cache that turns an in-memory embedded boot from an `initdb` into a
+// restore — a post-`initdb` data-directory tarball, keyed on what produced it, kept in a file the
+// caller names and in this process's memory. Never trusted: a file is read only through its own
+// checksum, and anything that does not verify is deleted and rebuilt.
+
+// why: Bun has no rename and no recursive remove; the write is a temp name then an atomic rename.
+import { rename, rm } from 'node:fs/promises';
+// why: Bun ships no path joiner.
+import { dirname, join } from 'node:path';
+import { PGLITE_PACKAGE } from './pglite-package';
+
+/** Bumped when the file layout below changes, so an old file is a miss rather than a misread. */
+const SNAPSHOT_FORMAT = 1;
+const MAGIC = `x-pglite-snapshot:${SNAPSHOT_FORMAT}:`;
+/** `MAGIC`, 64 hex characters, a newline — then the tarball. */
+const HEADER_BYTES = MAGIC.length + 64 + 1;
+
+const sha256 = (bytes: Uint8Array): string =>
+  new Bun.CryptoHasher('sha256').update(bytes).digest('hex');
+
+/**
+ * The installed PGlite's version, read off its own `package.json` — the package exports no
+ * version and no `./package.json`, so the entry point is resolved and its manifest read beside
+ * it. `undefined` when either step fails, which a caller reads as "do not cache": a snapshot that
+ * cannot name what produced it must not outlive an upgrade.
+ */
+export async function pgliteVersion(
+  resolve: (specifier: string) => string = (specifier) => import.meta.resolve(specifier),
+): Promise<string | undefined> {
+  try {
+    const entry = Bun.fileURLToPath(resolve(PGLITE_PACKAGE));
+    const manifest: unknown = await Bun.file(join(dirname(entry), '..', 'package.json')).json();
+    const version = (manifest as { readonly version?: unknown } | null)?.version;
+    return typeof version === 'string' && version.length > 0 ? version : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * What a snapshot is a snapshot OF: the PGlite build, and this file's own layout. NOT the linked
+ * extensions — `initdb` never sees them. An extension is files PGlite puts beside the data
+ * directory at boot, so one snapshot serves every extension set, and an app that adds `citext`
+ * does not pay a second `initdb` (`schema-dump.test.ts` restores a plain snapshot with one
+ * linked and creates it).
+ */
+export const snapshotKey = (version: string): string => `${version}-f${SNAPSHOT_FORMAT}`;
+
+export const snapshotFile = (dir: string, key: string): string =>
+  join(dir, `pglite-${key}.snapshot`);
+
+/** One snapshot per key for the life of the process: the second scratch boot never touches disk. */
+const memo = new Map<string, Blob>();
+
+export const rememberSnapshot = (key: string, blob: Blob): void => {
+  memo.set(key, blob);
+};
+
+export const forgetSnapshot = (key: string): void => {
+  memo.delete(key);
+};
+
+/**
+ * The tarball, or `undefined` — for a file that is absent, unreadable, short, mislabelled or whose
+ * bytes do not hash to what its header says. A file that fails any of those is DELETED: it will never
+ * verify, and leaving it costs every later boot the same read.
+ */
+export async function readSnapshot(file: string, key: string): Promise<Blob | undefined> {
+  const held = memo.get(key);
+  if (held !== undefined) return held;
+  const onDisk = Bun.file(file);
+  if (!(await onDisk.exists())) return undefined;
+  let bytes: Uint8Array<ArrayBuffer>;
+  try {
+    bytes = new Uint8Array(await onDisk.arrayBuffer());
+  } catch {
+    // `exists()` and the read are two calls: a racing `discardSnapshot` (ENOENT) or a file this
+    // user cannot read (EACCES) lands between them. A cache that cannot be read is a miss, never
+    // a failed boot — and it is not deleted, since this process could not read it to judge it.
+    return undefined;
+  }
+  const header = new TextDecoder().decode(bytes.subarray(0, HEADER_BYTES));
+  const body = bytes.subarray(HEADER_BYTES);
+  const sound =
+    header.startsWith(MAGIC) &&
+    header.endsWith('\n') &&
+    body.length > 0 &&
+    header.slice(MAGIC.length, -1) === sha256(body);
+  if (!sound) {
+    await discardSnapshot(file, key);
+    return undefined;
+  }
+  const blob = new Blob([body]);
+  memo.set(key, blob);
+  return blob;
+}
+
+/**
+ * Written under a name no other process shares, then renamed over the target. A rename within one
+ * directory is atomic, so a reader sees the old file or the new one and never half of either, and
+ * two writers racing each leave a whole file — the last one wins, and both are correct.
+ *
+ * Best effort: a cache that cannot be written (a read-only checkout, a full disk) costs the next
+ * boot an `initdb`, which is what it cost before there was a cache.
+ */
+export async function writeSnapshot(file: string, key: string, blob: Blob): Promise<void> {
+  memo.set(key, blob);
+  const body = new Uint8Array(await blob.arrayBuffer());
+  const temp = `${file}.${process.pid}.${crypto.randomUUID()}.tmp`;
+  try {
+    await Bun.write(temp, new Blob([`${MAGIC}${sha256(body)}\n`, body]), { createPath: true });
+    await rename(temp, file);
+  } catch {
+    await rm(temp, { force: true }).catch(() => undefined);
+  }
+}
+
+export async function discardSnapshot(file: string, key: string): Promise<void> {
+  memo.delete(key);
+  await rm(file, { force: true }).catch(() => undefined);
+}