@softure-ai/db 0.0.0-stage → 0.1.6

package/src/migrations/stubs.ts

@@ -0,0 +1,139 @@
+// Stubs of the app's own tables for the adoption reference schema (issue #170). Module SQL may
+// reference an app table (`REFERENCES public.app_users (id)`), which the scratch database the
+// reference is built on does not have. `readAppTables` copies, from the live database, every table
+// outside the schemas the migrator owns: its columns (type only: no defaults, NOT NULL, checks or
+// foreign keys, which can point at things the scratch lacks), its primary key, unique constraints
+// and plain unique indexes, plus the enum types of those schemas. Domains, extension types and
+// other objects are not copied; a column of such a type is skipped and reported.
+import type { MigrationSession } from "./session.js";
+
+/** One DDL statement for the scratch database, with the object it creates for reports. */
+export interface StubStatement {
+  /** `schema.table`, `schema.table.column`, `schema.table.constraint` or `schema.type`, unquoted. */
+  readonly object: string;
+  readonly sql: string;
+}
+
+/** The app's tables as statements to replay, in a runnable order. */
+export interface AppCatalog {
+  readonly statements: readonly StubStatement[];
+}
+
+// Names come back quoted by Postgres (`quote_ident`, `format('%I')`), so no DDL below concatenates
+// a raw identifier. Each query takes the excluded schemas as $1.
+const IN_APP_SCHEMA = (alias: string): string =>
+  `${alias}.nspname <> ALL($1::text[]) AND ${alias}.nspname <> 'information_schema' AND ${alias}.nspname NOT LIKE 'pg\\_%'`;
+
+const NOT_IN_EXTENSION = (catalog: string, alias: string): string =>
+  `NOT EXISTS (SELECT 1 FROM pg_depend dep WHERE dep.classid = '${catalog}'::regclass AND dep.objid = ${alias}.oid AND dep.deptype = 'e')`;
+
+const QUERIES = {
+  schemas: `
+    SELECT DISTINCT n.nspname AS name, quote_ident(n.nspname) AS quoted
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE c.relkind IN ('r', 'p') AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1`,
+  enums: `
+    SELECT n.nspname || '.' || t.typname AS object, format('%I.%I', n.nspname, t.typname) AS quoted,
+      (SELECT string_agg(quote_literal(e.enumlabel), ', ' ORDER BY e.enumsortorder) FROM pg_enum e WHERE e.enumtypid = t.oid) AS labels
+    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
+    WHERE t.typtype = 'e' AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_type", "t")}
+    ORDER BY 1`,
+  tables: `
+    SELECT n.nspname || '.' || c.relname AS object, format('%I.%I', n.nspname, c.relname) AS quoted
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE c.relkind IN ('r', 'p') AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1`,
+  columns: `
+    SELECT n.nspname || '.' || c.relname AS table_object, format('%I.%I', n.nspname, c.relname) AS table_quoted,
+      a.attname AS name, quote_ident(a.attname) AS quoted, format_type(a.atttypid, a.atttypmod) AS type
+    FROM pg_attribute a
+      JOIN pg_class c ON c.oid = a.attrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE c.relkind IN ('r', 'p') AND a.attnum > 0 AND NOT a.attisdropped
+      AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1, a.attnum`,
+  keys: `
+    SELECT n.nspname || '.' || c.relname AS table_object, format('%I.%I', n.nspname, c.relname) AS table_quoted,
+      con.conname AS name, quote_ident(con.conname) AS quoted, pg_get_constraintdef(con.oid) AS definition
+    FROM pg_constraint con
+      JOIN pg_class c ON c.oid = con.conrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE con.contype IN ('p', 'u') AND c.relkind IN ('r', 'p')
+      AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1, con.contype, 3`,
+  // A unique index that backs no key constraint. A foreign key records the index it references in
+  // `conindid` too, so only key constraints count.
+  uniqueIndexes: `
+    SELECT n.nspname || '.' || c.relname AS table_object, i.relname AS name, pg_get_indexdef(i.oid) AS definition
+    FROM pg_index x
+      JOIN pg_class i ON i.oid = x.indexrelid
+      JOIN pg_class c ON c.oid = x.indrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE x.indisunique AND x.indpred IS NULL AND x.indexprs IS NULL AND c.relkind IN ('r', 'p')
+      AND NOT EXISTS (SELECT 1 FROM pg_constraint con WHERE con.conindid = x.indexrelid AND con.contype IN ('p', 'u', 'x'))
+      AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1, 2`,
+} as const;
+
+interface ColumnRow {
+  table_object: string;
+  table_quoted: string;
+  name: string;
+  quoted: string;
+  type: string;
+}
+
+interface KeyRow {
+  table_object: string;
+  table_quoted: string;
+  name: string;
+  quoted: string;
+  definition: string;
+}
+
+/**
+ * The app's tables in every schema except `pg_*`, `information_schema` and `excludedSchemas` (the
+ * ledger, the enabled modules' schemas, the app migrator's own). Runs in its own read-only transaction.
+ */
+export async function readAppTables(session: MigrationSession, excludedSchemas: readonly string[]): Promise<AppCatalog> {
+  await session.exec("BEGIN READ ONLY; SET LOCAL search_path = pg_catalog;");
+  try {
+    const params = [[...excludedSchemas]];
+    const schemas = await session.query<{ name: string; quoted: string }>(QUERIES.schemas, params);
+    const enums = await session.query<{ object: string; quoted: string; labels: string }>(QUERIES.enums, params);
+    const tables = await session.query<{ object: string; quoted: string }>(QUERIES.tables, params);
+    const columns = await session.query<ColumnRow>(QUERIES.columns, params);
+    const keys = await session.query<KeyRow>(QUERIES.keys, params);
+    const indexes = await session.query<{ table_object: string; name: string; definition: string }>(QUERIES.uniqueIndexes, params);
+    return {
+      statements: [
+        ...schemas.map((row) => ({ object: row.name, sql: `CREATE SCHEMA IF NOT EXISTS ${row.quoted}` })),
+        ...enums.map((row) => ({ object: row.object, sql: `CREATE TYPE ${row.quoted} AS ENUM (${row.labels})` })),
+        ...tables.map((row) => ({ object: row.object, sql: `CREATE TABLE ${row.quoted} ()` })),
+        ...columns.map((row) => ({ object: `${row.table_object}.${row.name}`, sql: `ALTER TABLE ${row.table_quoted} ADD COLUMN ${row.quoted} ${row.type}` })),
+        ...keys.map((row) => ({ object: `${row.table_object}.${row.name}`, sql: `ALTER TABLE ${row.table_quoted} ADD CONSTRAINT ${row.quoted} ${row.definition}` })),
+        ...indexes.map((row) => ({ object: `${row.table_object}.${row.name}`, sql: row.definition })),
+      ],
+    };
+  } finally {
+    await session.exec("ROLLBACK");
+  }
+}
+
+/**
+ * Replays the catalog on the scratch database, each statement on its own: a statement that fails
+ * (a type the scratch lacks, a key on a skipped column) is skipped, so a module that never
+ * references that table still builds its reference. Returns the skipped objects with the reason.
+ */
+export async function createAppStubs(session: MigrationSession, catalog: AppCatalog): Promise<string[]> {
+  const skipped: string[] = [];
+  for (const statement of catalog.statements) {
+    try {
+      await session.exec(statement.sql);
+    } catch (error) {
+      skipped.push(`${statement.object}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+  return skipped;
+}

package/src/shared.ts

@@ -0,0 +1,36 @@
+// One database handle per URL for the whole server process, shared by every module's Next.js
+// adapter and the app. Package code cannot import the app's own handle, and `next dev` re-evaluates
+// modules on every change, so the handles live on `globalThis` under a `Symbol.for` key: each
+// reload, and each copy of this file in separate server bundles, finds the same pool.
+import { createDatabase, type CreateDatabaseOptions, type DatabaseHandle } from "./client.js";
+
+const HANDLES_KEY = Symbol.for("@softure-ai/db/shared-handles");
+
+type HandleHost = Record<typeof HANDLES_KEY, Map<string, Promise<DatabaseHandle>> | undefined>;
+
+const host = globalThis as unknown as HandleHost;
+
+/**
+ * The process-wide handle for `url`, opened on first use. `options` apply only to that first call.
+ * A failed open is not kept, so the next call tries again.
+ */
+export function getSharedDatabase(url: string, options: CreateDatabaseOptions = {}): Promise<DatabaseHandle> {
+  const handles = (host[HANDLES_KEY] ??= new Map<string, Promise<DatabaseHandle>>());
+  let handle = handles.get(url);
+  if (handle === undefined) {
+    handle = createDatabase(url, options);
+    handles.set(url, handle);
+    handle.catch(() => handles.delete(url));
+  }
+  return handle;
+}
+
+/** Closes and forgets every shared handle. For tests and graceful shutdown. */
+export async function closeSharedDatabases(): Promise<void> {
+  const handles = host[HANDLES_KEY];
+  if (handles === undefined) return;
+  const opened = [...handles.values()];
+  handles.clear();
+  const results = await Promise.allSettled(opened);
+  await Promise.all(results.flatMap((result) => (result.status === "fulfilled" ? [result.value.close()] : [])));
+}

package/src/testing.ts

@@ -0,0 +1,80 @@
+// Test databases for module unit tests (docs/02-module-standard.md §10): a real Postgres (PGlite)
+// with the real migrations of the given modules. Migrating costs about two seconds, so each
+// module set is migrated once per test worker into a template, and every call starts from a copy
+// of the template's data folder (FIRE_TRACKER `src/db/test-db.ts`). An app with its own migrations
+// passes the same `app` hooks (and baseline) as to `migrate`, so module tables can reference app
+// tables and a module the app's history already creates is adopted in the template.
+import { PGlite } from "@electric-sql/pglite";
+import type { AnySoftureModule } from "@softure-ai/core";
+import { createPgliteHandle, type PgliteClientDatabase } from "./client.js";
+import { migrate, type AppMigrationHook, type AppMigrations } from "./migrations/migrator.js";
+import { describeProblem } from "./migrations/problems.js";
+
+export interface TestDatabase {
+  readonly db: PgliteClientDatabase;
+  readonly client: PGlite;
+  readonly close: () => Promise<void>;
+}
+
+export interface CreateTestDatabaseOptions {
+  /**
+   * The app's own migrations, run around the module files as `migrate` runs them. The template is
+   * cached per hook object: define the hooks once, at module level, not inline per call.
+   */
+  readonly app?: AppMigrations;
+}
+
+const templates = new Map<string, Promise<Blob | File>>();
+const hookIds = new WeakMap<AppMigrationHook, number>();
+let nextHookId = 1;
+
+/**
+ * A fresh, migrated, in-memory database per call. List the module under test and the modules it
+ * depends on. Throws when a migration (a module file or an app hook) fails: that is a bug in the
+ * module package or the app's migrations.
+ */
+export async function createTestDatabase(modules: readonly AnySoftureModule[] = [], options: CreateTestDatabaseOptions = {}): Promise<TestDatabase> {
+  const client = new PGlite({ loadDataDir: await getTemplate(modules, options.app ?? {}) });
+  const handle = await createPgliteHandle(client);
+  return { db: handle.db, client, close: handle.close };
+}
+
+function getTemplate(modules: readonly AnySoftureModule[], app: AppMigrations): Promise<Blob | File> {
+  const moduleKey = modules.map((module) => `${module.id}@${module.migrations?.dir.href ?? "-"}`).join("|");
+  const baselineKey = Object.entries(app.baseline ?? {}).sort(([left], [right]) => left.localeCompare(right)).map(([id, through]) => `${id}:${through}`).join(",");
+  const key = `${moduleKey}#before:${identifyHook(app.before)}#after:${identifyHook(app.after)}#baseline:${baselineKey}`;
+  let template = templates.get(key);
+  if (template === undefined) {
+    template = buildTemplate(modules, app);
+    templates.set(key, template);
+    // A failed build is not cached: the next call reports the problem again.
+    template.catch(() => templates.delete(key));
+  }
+  return template;
+}
+
+// Hooks are functions, so the key holds an id per hook object rather than its source text.
+function identifyHook(hook: AppMigrationHook | undefined): string {
+  if (hook === undefined) {
+    return "-";
+  }
+  let id = hookIds.get(hook);
+  if (id === undefined) {
+    id = nextHookId++;
+    hookIds.set(hook, id);
+  }
+  return String(id);
+}
+
+async function buildTemplate(modules: readonly AnySoftureModule[], app: AppMigrations): Promise<Blob | File> {
+  const handle = await createPgliteHandle(new PGlite());
+  try {
+    const result = await migrate(handle, { modules, app });
+    if (!result.ok) {
+      throw new Error(`createTestDatabase: migrations failed:\n${result.problems.map(describeProblem).join("\n")}`);
+    }
+    return await handle.client.dumpDataDir("none");
+  } finally {
+    await handle.close();
+  }
+}