@softure-ai/db 0.1.5 → 0.1.7

package/src/migrations/stubs.ts

@@ -0,0 +1,139 @@
+// Stubs of the app's own tables for the adoption reference schema (issue #170). Module SQL may
+// reference an app table (`REFERENCES public.app_users (id)`), which the scratch database the
+// reference is built on does not have. `readAppTables` copies, from the live database, every table
+// outside the schemas the migrator owns: its columns (type only: no defaults, NOT NULL, checks or
+// foreign keys, which can point at things the scratch lacks), its primary key, unique constraints
+// and plain unique indexes, plus the enum types of those schemas. Domains, extension types and
+// other objects are not copied; a column of such a type is skipped and reported.
+import type { MigrationSession } from "./session.js";
+
+/** One DDL statement for the scratch database, with the object it creates for reports. */
+export interface StubStatement {
+  /** `schema.table`, `schema.table.column`, `schema.table.constraint` or `schema.type`, unquoted. */
+  readonly object: string;
+  readonly sql: string;
+}
+
+/** The app's tables as statements to replay, in a runnable order. */
+export interface AppCatalog {
+  readonly statements: readonly StubStatement[];
+}
+
+// Names come back quoted by Postgres (`quote_ident`, `format('%I')`), so no DDL below concatenates
+// a raw identifier. Each query takes the excluded schemas as $1.
+const IN_APP_SCHEMA = (alias: string): string =>
+  `${alias}.nspname <> ALL($1::text[]) AND ${alias}.nspname <> 'information_schema' AND ${alias}.nspname NOT LIKE 'pg\\_%'`;
+
+const NOT_IN_EXTENSION = (catalog: string, alias: string): string =>
+  `NOT EXISTS (SELECT 1 FROM pg_depend dep WHERE dep.classid = '${catalog}'::regclass AND dep.objid = ${alias}.oid AND dep.deptype = 'e')`;
+
+const QUERIES = {
+  schemas: `
+    SELECT DISTINCT n.nspname AS name, quote_ident(n.nspname) AS quoted
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE c.relkind IN ('r', 'p') AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1`,
+  enums: `
+    SELECT n.nspname || '.' || t.typname AS object, format('%I.%I', n.nspname, t.typname) AS quoted,
+      (SELECT string_agg(quote_literal(e.enumlabel), ', ' ORDER BY e.enumsortorder) FROM pg_enum e WHERE e.enumtypid = t.oid) AS labels
+    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
+    WHERE t.typtype = 'e' AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_type", "t")}
+    ORDER BY 1`,
+  tables: `
+    SELECT n.nspname || '.' || c.relname AS object, format('%I.%I', n.nspname, c.relname) AS quoted
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE c.relkind IN ('r', 'p') AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1`,
+  columns: `
+    SELECT n.nspname || '.' || c.relname AS table_object, format('%I.%I', n.nspname, c.relname) AS table_quoted,
+      a.attname AS name, quote_ident(a.attname) AS quoted, format_type(a.atttypid, a.atttypmod) AS type
+    FROM pg_attribute a
+      JOIN pg_class c ON c.oid = a.attrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE c.relkind IN ('r', 'p') AND a.attnum > 0 AND NOT a.attisdropped
+      AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1, a.attnum`,
+  keys: `
+    SELECT n.nspname || '.' || c.relname AS table_object, format('%I.%I', n.nspname, c.relname) AS table_quoted,
+      con.conname AS name, quote_ident(con.conname) AS quoted, pg_get_constraintdef(con.oid) AS definition
+    FROM pg_constraint con
+      JOIN pg_class c ON c.oid = con.conrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE con.contype IN ('p', 'u') AND c.relkind IN ('r', 'p')
+      AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1, con.contype, 3`,
+  // A unique index that backs no key constraint. A foreign key records the index it references in
+  // `conindid` too, so only key constraints count.
+  uniqueIndexes: `
+    SELECT n.nspname || '.' || c.relname AS table_object, i.relname AS name, pg_get_indexdef(i.oid) AS definition
+    FROM pg_index x
+      JOIN pg_class i ON i.oid = x.indexrelid
+      JOIN pg_class c ON c.oid = x.indrelid
+      JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE x.indisunique AND x.indpred IS NULL AND x.indexprs IS NULL AND c.relkind IN ('r', 'p')
+      AND NOT EXISTS (SELECT 1 FROM pg_constraint con WHERE con.conindid = x.indexrelid AND con.contype IN ('p', 'u', 'x'))
+      AND ${IN_APP_SCHEMA("n")} AND ${NOT_IN_EXTENSION("pg_class", "c")}
+    ORDER BY 1, 2`,
+} as const;
+
+interface ColumnRow {
+  table_object: string;
+  table_quoted: string;
+  name: string;
+  quoted: string;
+  type: string;
+}
+
+interface KeyRow {
+  table_object: string;
+  table_quoted: string;
+  name: string;
+  quoted: string;
+  definition: string;
+}
+
+/**
+ * The app's tables in every schema except `pg_*`, `information_schema` and `excludedSchemas` (the
+ * ledger, the enabled modules' schemas, the app migrator's own). Runs in its own read-only transaction.
+ */
+export async function readAppTables(session: MigrationSession, excludedSchemas: readonly string[]): Promise<AppCatalog> {
+  await session.exec("BEGIN READ ONLY; SET LOCAL search_path = pg_catalog;");
+  try {
+    const params = [[...excludedSchemas]];
+    const schemas = await session.query<{ name: string; quoted: string }>(QUERIES.schemas, params);
+    const enums = await session.query<{ object: string; quoted: string; labels: string }>(QUERIES.enums, params);
+    const tables = await session.query<{ object: string; quoted: string }>(QUERIES.tables, params);
+    const columns = await session.query<ColumnRow>(QUERIES.columns, params);
+    const keys = await session.query<KeyRow>(QUERIES.keys, params);
+    const indexes = await session.query<{ table_object: string; name: string; definition: string }>(QUERIES.uniqueIndexes, params);
+    return {
+      statements: [
+        ...schemas.map((row) => ({ object: row.name, sql: `CREATE SCHEMA IF NOT EXISTS ${row.quoted}` })),
+        ...enums.map((row) => ({ object: row.object, sql: `CREATE TYPE ${row.quoted} AS ENUM (${row.labels})` })),
+        ...tables.map((row) => ({ object: row.object, sql: `CREATE TABLE ${row.quoted} ()` })),
+        ...columns.map((row) => ({ object: `${row.table_object}.${row.name}`, sql: `ALTER TABLE ${row.table_quoted} ADD COLUMN ${row.quoted} ${row.type}` })),
+        ...keys.map((row) => ({ object: `${row.table_object}.${row.name}`, sql: `ALTER TABLE ${row.table_quoted} ADD CONSTRAINT ${row.quoted} ${row.definition}` })),
+        ...indexes.map((row) => ({ object: `${row.table_object}.${row.name}`, sql: row.definition })),
+      ],
+    };
+  } finally {
+    await session.exec("ROLLBACK");
+  }
+}
+
+/**
+ * Replays the catalog on the scratch database, each statement on its own: a statement that fails
+ * (a type the scratch lacks, a key on a skipped column) is skipped, so a module that never
+ * references that table still builds its reference. Returns the skipped objects with the reason.
+ */
+export async function createAppStubs(session: MigrationSession, catalog: AppCatalog): Promise<string[]> {
+  const skipped: string[] = [];
+  for (const statement of catalog.statements) {
+    try {
+      await session.exec(statement.sql);
+    } catch (error) {
+      skipped.push(`${statement.object}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+  return skipped;
+}

package/src/process.ts

@@ -0,0 +1,110 @@
+// A process-wide database for an app whose code imports one module-level `db` (drizzle queries everywhere)
+// instead of awaiting a handle in every function. `db` is a synchronous stand-in for the shared handle's
+// drizzle instance: each entry point (the server's startup, a script's main) opens it first, and a query
+// before that throws instead of silently opening a second pool.
+import type { Database, DatabaseHandle, DatabaseSchema, CreateDatabaseOptions } from "./client.js";
+import { closeSharedDatabase, getSharedDatabase } from "./shared.js";
+
+const NOT_OPEN_MESSAGE = "Database is not open: call open() or withDatabase() at the process entry before querying";
+
+export interface ProcessDatabaseOptions<TSchema extends DatabaseSchema> extends CreateDatabaseOptions {
+  /** The app's drizzle schema: `db` is built with it, so `db.query.<table>` is typed and works. */
+  readonly schema?: TSchema;
+}
+
+export interface ProcessDatabase<TSchema extends DatabaseSchema = DatabaseSchema> {
+  /** The open database. Usable from module scope; a query before `open()` resolves throws "Database is not open". */
+  readonly db: Database<TSchema>;
+  /** Opens the process-wide handle for the URL (`getSharedDatabase`), once; later calls return the same handle. */
+  readonly open: () => Promise<DatabaseHandle>;
+  /** Closes the handle and forgets it, so `db` refuses queries again. Does nothing when not open. */
+  readonly close: () => Promise<void>;
+  /** For a script: opens, runs `main` with `db`, and closes whether `main` resolves or throws. */
+  readonly withDatabase: <T>(main: (db: Database<TSchema>) => Promise<T>) => Promise<T>;
+  readonly isOpen: () => boolean;
+}
+
+/**
+ * A module-level database for the process. `url` may be a function, read on `open()`, so a script can load its
+ * environment after importing the module that creates it. The handle is the one `getSharedDatabase(url)` gives the
+ * modules, so the app and the modules share one pool (db README §3, "One handle per process").
+ */
+export function createProcessDatabase<TSchema extends DatabaseSchema = DatabaseSchema>(
+  url: string | (() => string),
+  options: ProcessDatabaseOptions<TSchema> = {},
+): ProcessDatabase<TSchema> {
+  const { schema, ...databaseOptions } = options;
+  let opening: Promise<DatabaseHandle> | undefined;
+  let opened: { readonly url: string; readonly db: Database<TSchema> } | undefined;
+
+  const open = (): Promise<DatabaseHandle> => {
+    opening ??= openHandle().catch((error: unknown) => {
+      opening = undefined;
+      throw error;
+    });
+    return opening;
+  };
+
+  async function openHandle(): Promise<DatabaseHandle> {
+    const resolvedUrl = typeof url === "function" ? url() : url;
+    const handle = await getSharedDatabase(resolvedUrl, databaseOptions);
+    opened = { url: resolvedUrl, db: await buildDatabase(handle, schema) };
+    return handle;
+  }
+
+  const close = async (): Promise<void> => {
+    if (opening === undefined) return;
+    const pending = opening;
+    opening = undefined;
+    await pending.catch(() => undefined);
+    const current = opened;
+    opened = undefined;
+    if (current !== undefined) await closeSharedDatabase(current.url);
+  };
+
+  const withDatabase = async <T>(main: (db: Database<TSchema>) => Promise<T>): Promise<T> => {
+    await open();
+    try {
+      return await main(db);
+    } finally {
+      await close();
+    }
+  };
+
+  const db = createDatabaseProxy(() => opened?.db);
+
+  return { db, open, close, withDatabase, isOpen: () => opened !== undefined };
+}
+
+/** The handle's drizzle instance, or a new one over the same pool or client when the app passes its schema. */
+async function buildDatabase<TSchema extends DatabaseSchema>(handle: DatabaseHandle, schema: TSchema | undefined): Promise<Database<TSchema>> {
+  // The handle's own instance is typed with the default schema; without the app's schema the two types agree.
+  if (schema === undefined) return handle.db as unknown as Database<TSchema>;
+  if (handle.kind === "postgres") {
+    const { drizzle } = await import("drizzle-orm/node-postgres");
+    return drizzle({ client: handle.pool, schema });
+  }
+  const { drizzle } = await import("drizzle-orm/pglite");
+  return drizzle({ client: handle.client, schema });
+}
+
+/** A stand-in that forwards every property to the open database, or throws when none is open. */
+function createDatabaseProxy<TSchema extends DatabaseSchema>(getDatabase: () => Database<TSchema> | undefined): Database<TSchema> {
+  return new Proxy({} as Database<TSchema>, {
+    get(_target, property) {
+      const database = getDatabase();
+      if (database === undefined) {
+        // `await db` and `Promise.resolve(db)` look for `then`: a closed database is just not a thenable.
+        if (property === "then") return undefined;
+        throw new Error(NOT_OPEN_MESSAGE);
+      }
+      const value: unknown = Reflect.get(database, property, database);
+      return typeof value === "function" ? (value as (...args: unknown[]) => unknown).bind(database) : value;
+    },
+    has(_target, property) {
+      const database = getDatabase();
+      if (database === undefined) throw new Error(NOT_OPEN_MESSAGE);
+      return Reflect.has(database, property);
+    },
+  });
+}

package/src/shared.ts

@@ -25,6 +25,15 @@ export function getSharedDatabase(url: string, options: CreateDatabaseOptions =
   return handle;
 }
 
+/** Closes and forgets the shared handle for `url`, if one is open. For a script's end. */
+export async function closeSharedDatabase(url: string): Promise<void> {
+  const handle = host[HANDLES_KEY]?.get(url);
+  if (handle === undefined) return;
+  host[HANDLES_KEY]?.delete(url);
+  const opened = await handle.catch(() => undefined);
+  await opened?.close();
+}
+
 /** Closes and forgets every shared handle. For tests and graceful shutdown. */
 export async function closeSharedDatabases(): Promise<void> {
   const handles = host[HANDLES_KEY];

package/src/testing.ts

@@ -1,11 +1,13 @@
 // Test databases for module unit tests (docs/02-module-standard.md §10): a real Postgres (PGlite)
 // with the real migrations of the given modules. Migrating costs about two seconds, so each
 // module set is migrated once per test worker into a template, and every call starts from a copy
-// of the template's data folder (FIRE_TRACKER `src/db/test-db.ts`).
+// of the template's data folder (FIRE_TRACKER `src/db/test-db.ts`). An app with its own migrations
+// passes the same `app` hooks (and baseline) as to `migrate`, so module tables can reference app
+// tables and a module the app's history already creates is adopted in the template.
 import { PGlite } from "@electric-sql/pglite";
 import type { AnySoftureModule } from "@softure-ai/core";
 import { createPgliteHandle, type PgliteClientDatabase } from "./client.js";
-import { migrate } from "./migrations/migrator.js";
+import { migrate, type AppMigrationHook, type AppMigrations } from "./migrations/migrator.js";
 import { describeProblem } from "./migrations/problems.js";
 
 export interface TestDatabase {
@@ -14,23 +16,36 @@ export interface TestDatabase {
   readonly close: () => Promise<void>;
 }
 
+export interface CreateTestDatabaseOptions {
+  /**
+   * The app's own migrations, run around the module files as `migrate` runs them. The template is
+   * cached per hook object: define the hooks once, at module level, not inline per call.
+   */
+  readonly app?: AppMigrations;
+}
+
 const templates = new Map<string, Promise<Blob | File>>();
+const hookIds = new WeakMap<AppMigrationHook, number>();
+let nextHookId = 1;
 
 /**
  * A fresh, migrated, in-memory database per call. List the module under test and the modules it
- * depends on. Throws when a migration is invalid or fails: that is a bug in the module package.
+ * depends on. Throws when a migration (a module file or an app hook) fails: that is a bug in the
+ * module package or the app's migrations.
  */
-export async function createTestDatabase(modules: readonly AnySoftureModule[] = []): Promise<TestDatabase> {
-  const client = new PGlite({ loadDataDir: await getTemplate(modules) });
+export async function createTestDatabase(modules: readonly AnySoftureModule[] = [], options: CreateTestDatabaseOptions = {}): Promise<TestDatabase> {
+  const client = new PGlite({ loadDataDir: await getTemplate(modules, options.app ?? {}) });
   const handle = await createPgliteHandle(client);
   return { db: handle.db, client, close: handle.close };
 }
 
-function getTemplate(modules: readonly AnySoftureModule[]): Promise<Blob | File> {
-  const key = modules.map((module) => `${module.id}@${module.migrations?.dir.href ?? "-"}`).join("|");
+function getTemplate(modules: readonly AnySoftureModule[], app: AppMigrations): Promise<Blob | File> {
+  const moduleKey = modules.map((module) => `${module.id}@${module.migrations?.dir.href ?? "-"}`).join("|");
+  const baselineKey = Object.entries(app.baseline ?? {}).sort(([left], [right]) => left.localeCompare(right)).map(([id, through]) => `${id}:${through}`).join(",");
+  const key = `${moduleKey}#before:${identifyHook(app.before)}#after:${identifyHook(app.after)}#baseline:${baselineKey}`;
   let template = templates.get(key);
   if (template === undefined) {
-    template = buildTemplate(modules);
+    template = buildTemplate(modules, app);
     templates.set(key, template);
     // A failed build is not cached: the next call reports the problem again.
     template.catch(() => templates.delete(key));
@@ -38,10 +53,23 @@ function getTemplate(modules: readonly AnySoftureModule[]): Promise<Blob | File>
   return template;
 }
 
-async function buildTemplate(modules: readonly AnySoftureModule[]): Promise<Blob | File> {
+// Hooks are functions, so the key holds an id per hook object rather than its source text.
+function identifyHook(hook: AppMigrationHook | undefined): string {
+  if (hook === undefined) {
+    return "-";
+  }
+  let id = hookIds.get(hook);
+  if (id === undefined) {
+    id = nextHookId++;
+    hookIds.set(hook, id);
+  }
+  return String(id);
+}
+
+async function buildTemplate(modules: readonly AnySoftureModule[], app: AppMigrations): Promise<Blob | File> {
   const handle = await createPgliteHandle(new PGlite());
   try {
-    const result = await migrate(handle, { modules });
+    const result = await migrate(handle, { modules, app });
     if (!result.ok) {
       throw new Error(`createTestDatabase: migrations failed:\n${result.problems.map(describeProblem).join("\n")}`);
     }