@bevel-software/platform-core-backend 0.20.0 → 0.22.0

package/src/modules/database/__tests__/migrations-unqualified.test.ts

@@ -0,0 +1,32 @@
+import { describe, it, expect } from 'vitest';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { coreMigrationsDir } from '../../../assets.js';
+
+/**
+ * A knowledge base hosted beside others lives in a schema of its own, found
+ * through `search_path`. A migration that spells `public.` runs against the
+ * wrong schema there — creating the table in `public`, or guarding on a
+ * constraint that lives elsewhere — and the tenant that runs it is the first
+ * to find out. So no migration may name a schema: this reads every file the
+ * runner would, so the rule cannot be forgotten by the next one.
+ */
+describe('core migrations', () => {
+  it('never qualify a table with a schema', async () => {
+    const dir = coreMigrationsDir();
+    const files = (await fs.readdir(dir)).filter((name) => name.endsWith('.sql')).sort();
+    expect(files.length).toBeGreaterThan(0);
+    const offenders: string[] = [];
+    for (const name of files) {
+      const text = await fs.readFile(path.join(dir, name), 'utf8');
+      text.split('\n').forEach((line, i) => {
+        // Bare (`public.users`) and quoted (`"public"."users"`) alike: the
+        // quoted spelling is what drizzle generates, and it is what a live
+        // two-tenant run found still pointing every tenant's foreign keys at
+        // a `public.users` that does not exist there.
+        if (/\bpublic"?\./.test(line)) offenders.push(`${name}:${i + 1}: ${line.trim()}`);
+      });
+    }
+    expect(offenders).toEqual([]);
+  });
+});

package/src/modules/database/advisory-lock.ts

@@ -1,3 +1,4 @@
+import { crc32 } from 'node:zlib';
 import pg from 'pg';
 import type { Database } from './connection.js';
 
@@ -84,6 +85,23 @@ export const AdvisoryLock = {
 
 export type AdvisoryLockId = (typeof AdvisoryLock)[keyof typeof AdvisoryLock];
 
+/**
+ * The second integer of a lock: the lock id, folded with the knowledge base
+ * it belongs to. A process serving several knowledge bases from one database
+ * runs several commit workers and several migration runs, and each must
+ * exclude only its own kind: the migrations lock of tenant A has nothing to
+ * say to tenant B's. An empty `tenantKey` is the single-tenant deployment
+ * and keeps the ids that were always used, so a redeploy's outgoing process
+ * on the older build still contends for the same lock.
+ *
+ * `crc32` for a stable 32-bit spread of the key; XOR keeps every value
+ * inside `int4`. Two tenants whose keys collide serialise against each other
+ * and nothing worse: an over-locking, never an under-locking.
+ */
+export function advisoryLockKey(lock: AdvisoryLockId, tenantKey = ''): number {
+  return tenantKey ? lock ^ crc32(tenantKey) : lock;
+}
+
 /** How long to wait for the lock before giving up and failing the boot. */
 const DEFAULT_WAIT_MS = 60_000;
 
@@ -138,9 +156,10 @@ export async function withAdvisoryLock<T>(
   db: Database,
   lock: AdvisoryLockId,
   fn: () => Promise<T>,
-  opts: { waitMs?: number } = {},
+  opts: { waitMs?: number; tenantKey?: string } = {},
 ): Promise<T> {
   const waitMs = opts.waitMs ?? DEFAULT_WAIT_MS;
+  const key = advisoryLockKey(lock, opts.tenantKey);
   const client = await db.$client.connect();
   // A statement that did not answer in time is STILL RUNNING on this
   // connection — `bounded` gives up waiting, it cannot take the statement
@@ -170,7 +189,7 @@ export async function withAdvisoryLock<T>(
       // parameter. `true` makes the setting transaction-local, so the
       // `rollback` below reverts it.
       await onWire(client.query("select set_config('lock_timeout', $1, true)", [String(waitMs)]), 'setting lock_timeout');
-      await client.query('select pg_advisory_xact_lock($1, $2)', [LOCK_NAMESPACE, lock]);
+      await client.query('select pg_advisory_xact_lock($1, $2)', [LOCK_NAMESPACE, key]);
     } catch (err) {
       if (isLockTimeout(err)) {
         throw new Error(
@@ -215,6 +234,8 @@ export interface LeaseClient {
 }
 
 export interface AdvisoryLeaseOptions {
+  /** The knowledge base this lease belongs to — see {@link advisoryLockKey}. Empty: the single-tenant ids. */
+  tenantKey?: string;
   /** How the lease opens its connection. Default: a `pg.Client` on the pool's connection string. */
   connect?: () => Promise<LeaseClient>;
   /** Bound on the connect and on each query, so an unreachable or half-dead database fails rather than waits. Default 10s. */
@@ -290,7 +311,7 @@ export class AdvisoryLease {
     try {
       const { rows } = await client.query('select pg_try_advisory_lock($1, $2) as held', [
         LOCK_NAMESPACE,
-        this.lock,
+        advisoryLockKey(this.lock, this.opts.tenantKey),
       ]);
       if (rows[0]?.held !== true) {
         await client.end().catch(() => undefined);
@@ -315,7 +336,10 @@ export class AdvisoryLease {
     this.heldFlag = false;
     if (!client) return;
     try {
-      await client.query('select pg_advisory_unlock($1, $2)', [LOCK_NAMESPACE, this.lock]);
+      await client.query('select pg_advisory_unlock($1, $2)', [
+        LOCK_NAMESPACE,
+        advisoryLockKey(this.lock, this.opts.tenantKey),
+      ]);
     } catch {
       // Ending the session below releases the lock regardless.
     } finally {

package/src/modules/database/connection.ts

@@ -1,23 +1,153 @@
+import { sql } from 'drizzle-orm';
 import { drizzle } from 'drizzle-orm/node-postgres';
 import pg from 'pg';
 import * as schema from './schema.js';
 
-let db: ReturnType<typeof drizzle<typeof schema>> | null = null;
+/**
+ * One database, one schema per knowledge base.
+ *
+ * A single-tenant deployment keeps `public`, exactly as before this option
+ * existed: no `search_path` is set for it, so an operator's own default
+ * (`"$user", public`) is honoured, and the migration ledger stays where
+ * drizzle put it. A knowledge base hosted beside others gets a schema of its
+ * own: every connection of its pool starts with `search_path` pointing at it,
+ * so the unqualified table names the schema module and the migrations use
+ * resolve there and nowhere else. Isolation is then a Postgres object, and a
+ * tenant leaves as `pg_dump -n <schema>`.
+ */
+export const DEFAULT_DB_SCHEMA = 'public';
 
-export function getDb(databaseUrl: string) {
+/**
+ * A schema name this module will put into `search_path`: a plain lowercase
+ * identifier, at most Postgres's 63 characters. Validated rather than quoted,
+ * because the name also travels into `CREATE SCHEMA` and a `pg_dump`
+ * invocation, and a name that needs quoting in one of them is a name that
+ * will be spelled wrong in another.
+ */
+export function assertSchemaName(name: string): string {
+  if (!/^[a-z][a-z0-9_]{0,62}$/.test(name)) {
+    throw new Error(
+      `Database schema name must be a lowercase identifier ([a-z][a-z0-9_]*, at most 63 characters); got "${name}"`,
+    );
+  }
+  // Names Postgres or drizzle already own. `pg_*` would fail at CREATE
+  // SCHEMA anyway; `information_schema` would not, and `drizzle` is where a
+  // deployment on the default schema keeps its migration ledger, so a
+  // knowledge base placed there would share that ledger's home.
+  if (name.startsWith('pg_') || name === 'information_schema' || name === 'drizzle') {
+    throw new Error(`Database schema name "${name}" is reserved`);
+  }
+  return name;
+}
+
+/**
+ * Prove that this handle's connections really search `schema` first, by
+ * asking the server. The schema is set as a startup parameter, and a
+ * transaction- or statement-mode pooler in front of the database (PgBouncer
+ * in those modes, RDS Proxy) either refuses that parameter or resets it
+ * between transactions; every unqualified table name would then resolve to
+ * `public`, and tenants would read each other's rows without a single
+ * error. So a graph refuses to come up on a handle that lost its schema,
+ * which is the failure an operator can see. Only session pooling is
+ * supported in front of a process that serves several knowledge bases.
+ */
+export async function assertSearchPath(db: Pick<Database, 'execute'>, schema: string): Promise<void> {
+  const result = await db.execute(sql`select current_schema() as schema`);
+  const current = (result.rows[0] as { schema?: string } | undefined)?.schema ?? null;
+  if (current !== schema) {
+    throw new Error(
+      `Database connections search "${current ?? '(none)'}" first, not the configured schema "${schema}". ` +
+        'A transaction-mode pooler in front of Postgres drops the search_path startup parameter; ' +
+        'use session pooling, or connect directly.',
+    );
+  }
+}
+
+export interface DbOptions {
+  /** The schema the pool's connections search first. Default {@link DEFAULT_DB_SCHEMA}. */
+  schema?: string;
+  /**
+   * Connections the pool may hold. `pg`'s default (10) suits a process that
+   * serves one knowledge base; a process serving many keeps each pool small,
+   * since the database's `max_connections` is shared by all of them.
+   */
+  max?: number;
+  /** How long an idle connection is kept before the pool closes it. Default: `pg`'s. */
+  idleTimeoutMillis?: number;
+}
+
+/**
+ * A bounded handshake: `pg` waits forever by default for a NEW connection to
+ * come up, so a database that accepts the socket and never finishes the
+ * handshake would hold a boot (the migration lock is the first thing that
+ * asks) with nothing to say and nothing for the restart policy to see. Half
+ * a minute is generous for a handshake and short enough for a failed boot to
+ * be visible. It bounds only that: a checkout queued behind a saturated pool,
+ * or a query on a connection that stopped answering, is the caller's to
+ * bound — the advisory lock does.
+ */
+const CONNECT_TIMEOUT_MS = 30_000;
+
+/**
+ * A NEW pool on `databaseUrl`, searching `opts.schema` first. Not cached:
+ * the caller owns it and ends it with {@link closeDb}. The composition root
+ * of a process that serves one knowledge base goes through {@link getDb}
+ * instead, which caches.
+ */
+export function createDb(databaseUrl: string, opts: DbOptions = {}) {
+  const dbSchema = assertSchemaName(opts.schema ?? DEFAULT_DB_SCHEMA);
+  const pool = new pg.Pool({
+    connectionString: databaseUrl,
+    connectionTimeoutMillis: CONNECT_TIMEOUT_MS,
+    max: opts.max,
+    idleTimeoutMillis: opts.idleTimeoutMillis,
+    // A startup parameter, applied by the server to every connection this
+    // pool opens — not a `SET` run after connecting, which a pooled
+    // connection could outlive. The default schema sets nothing, so an
+    // operator's own `search_path` stays in force there.
+    ...(dbSchema === DEFAULT_DB_SCHEMA ? {} : { options: `-c search_path=${dbSchema}` }),
+  });
+  const db = drizzle(pool, { schema });
+  schemas.set(db, dbSchema);
+  return db;
+}
+
+export type Database = ReturnType<typeof createDb>;
+
+/** The schema a database handle searches first — for the callers that name it in DDL. */
+export function dbSchemaOf(db: Pick<Database, '$client'>): string {
+  return schemas.get(db as Database) ?? DEFAULT_DB_SCHEMA;
+}
+
+const schemas = new WeakMap<Database, string>();
+const cache = new Map<string, Database>();
+
+const cacheKey = (databaseUrl: string, dbSchema: string) => `${dbSchema}\u0000${databaseUrl}`;
+
+/**
+ * The process's pool on `databaseUrl` for `opts.schema`, created on first
+ * use and shared afterwards. One per (url, schema): two handles on the same
+ * schema would be two pools competing for the same connection budget.
+ */
+export function getDb(databaseUrl: string, opts: DbOptions = {}): Database {
+  const dbSchema = assertSchemaName(opts.schema ?? DEFAULT_DB_SCHEMA);
+  const key = cacheKey(databaseUrl, dbSchema);
+  let db = cache.get(key);
   if (!db) {
-    // A bounded handshake: `pg` waits forever by default for a NEW connection
-    // to come up, so a database that accepts the socket and never finishes
-    // the handshake would hold a boot (the migration lock is the first thing
-    // that asks) with nothing to say and nothing for the restart policy to
-    // see. Half a minute is generous for a handshake and short enough for a
-    // failed boot to be visible. It bounds only that: a checkout queued
-    // behind a saturated pool, or a query on a connection that stopped
-    // answering, is the caller's to bound — the advisory lock does.
-    const pool = new pg.Pool({ connectionString: databaseUrl, connectionTimeoutMillis: 30_000 });
-    db = drizzle(pool, { schema });
+    db = createDb(databaseUrl, opts);
+    cache.set(key, db);
   }
   return db;
 }
 
-export type Database = ReturnType<typeof getDb>;
+/**
+ * End a handle's pool and forget it, so a later {@link getDb} for the same
+ * schema opens a fresh one rather than handing out a pool that was ended.
+ * The last step of stopping a knowledge base's graph.
+ */
+export async function closeDb(db: Pick<Database, '$client'>): Promise<void> {
+  for (const [key, cached] of cache) {
+    if (cached === db) cache.delete(key);
+  }
+  await db.$client.end();
+}

package/src/modules/database/migrate.ts

@@ -2,7 +2,7 @@ import { migrate } from 'drizzle-orm/node-postgres/migrator';
 import { logger } from '../../shared/logging.js';
 
 const log = logger('database');
-import type { Database } from './connection.js';
+import { DEFAULT_DB_SCHEMA, dbSchemaOf, type Database } from './connection.js';
 import { AdvisoryLock, withAdvisoryLock } from './advisory-lock.js';
 
 /*
@@ -38,13 +38,40 @@ import { AdvisoryLock, withAdvisoryLock } from './advisory-lock.js';
  * container takes to exit. See `advisory-lock.ts` for the mechanism.
  */
 
+/**
+ * Where a handle's migration ledger lives, and which lock guards its runs.
+ *
+ * The default schema keeps drizzle's own `drizzle` schema for the ledger,
+ * where every existing deployment already has one: naming `public` here
+ * would make the next boot read an empty ledger and re-apply plain DDL that
+ * is already applied. A knowledge base on a schema of its own keeps its
+ * ledger IN that schema, so the schema is the whole of the tenant: what
+ * `pg_dump -n` carries out includes what has been applied to it, and the
+ * dedicated deployment it is restored into resumes from there rather than
+ * from nothing. The lock is keyed the same way, so two tenants of one
+ * database migrate side by side while two processes of one tenant still
+ * take turns.
+ */
+function ledgerOptions(db: Database): { migrationsSchema?: string; tenantKey: string } {
+  const schema = dbSchemaOf(db);
+  return schema === DEFAULT_DB_SCHEMA
+    ? { tenantKey: '' }
+    : { migrationsSchema: schema, tenantKey: schema };
+}
+
 /** Apply the CORE migration history from `folder`, tracked in `__drizzle_migrations_core`. */
 export async function runCoreMigrations(db: Database, folder: string): Promise<void> {
-  await withAdvisoryLock(db, AdvisoryLock.Migrations, async () => {
-    log.info('Running core database migrations...');
-    await migrate(db, { migrationsFolder: folder, migrationsTable: '__drizzle_migrations_core' });
-    log.info('Core migrations complete.');
-  });
+  const { migrationsSchema, tenantKey } = ledgerOptions(db);
+  await withAdvisoryLock(
+    db,
+    AdvisoryLock.Migrations,
+    async () => {
+      log.info('Running core database migrations...');
+      await migrate(db, { migrationsFolder: folder, migrationsTable: '__drizzle_migrations_core', migrationsSchema });
+      log.info('Core migrations complete.');
+    },
+    { tenantKey },
+  );
 }
 
 /**
@@ -53,12 +80,19 @@ export async function runCoreMigrations(db: Database, folder: string): Promise<v
  * enterprise tables FK into core tables.
  */
 export async function runEnterpriseMigrations(db: Database, folder: string): Promise<void> {
-  await withAdvisoryLock(db, AdvisoryLock.Migrations, async () => {
-    log.info('Running enterprise database migrations...');
-    await migrate(db, {
-      migrationsFolder: folder,
-      migrationsTable: '__drizzle_migrations_enterprise',
-    });
-    log.info('Enterprise migrations complete.');
-  });
+  const { migrationsSchema, tenantKey } = ledgerOptions(db);
+  await withAdvisoryLock(
+    db,
+    AdvisoryLock.Migrations,
+    async () => {
+      log.info('Running enterprise database migrations...');
+      await migrate(db, {
+        migrationsFolder: folder,
+        migrationsTable: '__drizzle_migrations_enterprise',
+        migrationsSchema,
+      });
+      log.info('Enterprise migrations complete.');
+    },
+    { tenantKey },
+  );
 }

package/src/modules/diff/__tests__/diff.service.seed-atomicity.test.ts

@@ -7,6 +7,7 @@ import { WorkspaceService } from '../../workspace/workspace.service.js';
 import { workspaceIdForBranch } from '../../../shared/workspace-id.js';
 import { WorkspaceMutex } from '../../kb-fs/mutex.js';
 import { DiffService } from '../diff.service.js';
+import { testKbContext } from '../../../__tests__/kb-context.js';
 
 /**
  * Atomic-seed regression suite.
@@ -55,7 +56,7 @@ async function setup(): Promise<Fixture> {
   const workspaceService = new WorkspaceService(
     workspacesRoot,
     'https://github.com/Bevel-Software/knowledge-base.git',
-    'knowledge-base',
+    testKbContext(),
     new NodeFs(),
   );
   const mutex = new WorkspaceMutex();

package/src/modules/diff/__tests__/diff.service.test.ts

@@ -7,6 +7,7 @@ import { WorkspaceService } from '../../workspace/workspace.service.js';
 import { workspaceIdForBranch } from '../../../shared/workspace-id.js';
 import { WorkspaceMutex } from '../../kb-fs/mutex.js';
 import { DiffService } from '../diff.service.js';
+import { testKbContext } from '../../../__tests__/kb-context.js';
 
 async function mkTmpRoot(prefix: string): Promise<string> {
   return fs.mkdtemp(path.join(os.tmpdir(), prefix));
@@ -41,7 +42,7 @@ async function setup(): Promise<Fixture> {
   const workspaceService = new WorkspaceService(
     workspacesRoot,
     'https://github.com/Bevel-Software/knowledge-base.git',
-    'knowledge-base',
+    testKbContext(),
     new NodeFs(),
   );
   const mutex = new WorkspaceMutex();