@cosmicdrift/kumiko-testing 0.385.0 → 0.386.0

package/bin/kumiko-testing.ts

@@ -8,6 +8,7 @@ import {
   buildIntegrationTestArgs,
   listIntegrationTestFiles,
   resolveRequestedIntegrationFiles,
+  runWithTestDbTemplates,
   selectIntegrationFiles,
 } from "../src/integration-runner";
 
@@ -99,8 +100,15 @@ async function runIntegration(args: readonly string[]): Promise<number> {
     ...(values.timings !== undefined && { timings: values.timings }),
     ...(values["update-timings"] === true && { updateTimings: true }),
   });
-  const proc = Bun.spawn(["bun", ...testArgs], { stdio: ["inherit", "inherit", "inherit"] });
-  return proc.exited;
+  return runWithTestDbTemplates(
+    // Bun.spawn defaults to the env from process start; pass the live one so the
+    // template run id set by runWithTestDbTemplates reaches the test process.
+    () =>
+      Bun.spawn(["bun", ...testArgs], {
+        stdio: ["inherit", "inherit", "inherit"],
+        env: process.env,
+      }).exited,
+  );
 }
 
 const [command, ...rest] = process.argv.slice(2);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-testing",
-  "version": "0.385.0",
+  "version": "0.386.0",
   "description": "Test template for Kumiko apps: per-flow seedTenant, app test stack, canonical test preloads, bunfig generator and integration runner.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -99,9 +99,9 @@
     "kumiko-testing": "./bin/kumiko-testing.ts"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-framework": "0.385.0",
-    "@cosmicdrift/kumiko-guards": "0.385.0",
-    "@cosmicdrift/kumiko-types": "0.385.0",
+    "@cosmicdrift/kumiko-framework": "0.386.0",
+    "@cosmicdrift/kumiko-guards": "0.386.0",
+    "@cosmicdrift/kumiko-types": "0.386.0",
     "zod": "^4.4.3"
   },
   "peerDependencies": {
@@ -125,8 +125,8 @@
     }
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-bundled-features": "0.385.0",
-    "@cosmicdrift/kumiko-dev-server": "0.385.0",
+    "@cosmicdrift/kumiko-bundled-features": "0.386.0",
+    "@cosmicdrift/kumiko-dev-server": "0.386.0",
     "@happy-dom/global-registrator": "^20.11.2",
     "@playwright/test": "^1.62.1",
     "@testing-library/dom": "^10.4.1",

package/src/changes.json

@@ -1,4 +1,16 @@
 [
+  {
+    "version": "0.386.0",
+    "type": "fix",
+    "title": "Integration test stacks clone a migrated template database",
+    "detail": "kumiko-testing integration and the framework integration runner set a run id; setupTestStack then builds one template per schema fingerprint under an advisory lock and clones it, and the runner drops the templates at the end. Plain bun test runs keep the in-place DDL path. setupAppTestStack skips pushEntityProjectionTables on fresh databases."
+  },
+  {
+    "version": "0.386.0",
+    "type": "fix",
+    "title": "createTestRedis waits for a ready connection",
+    "detail": "The test Redis client connected in the background, so the first rate-limit check in a test file could see status \"connect\" and fail with \"rate-limit redis connection not ready\". createTestRedis now connects explicitly and returns after the connection is ready."
+  },
   {
     "version": "0.385.0",
     "type": "fix",

package/src/integration-runner.ts

@@ -1,6 +1,9 @@
+import { randomBytes } from "node:crypto";
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { BUNFIG_FILES, TEST_TIMEOUT_MS } from "./bunfig";
+import { withEnv } from "./helpers/with-env";
+import { SERVICE_ENV_DEFAULTS } from "./preload/service-env-defaults-values";
 
 export type IntegrationRunOptions = {
   readonly files: readonly string[];
@@ -85,3 +88,21 @@ export function buildIntegrationTestArgs(opts: IntegrationRunOptions): string[]
     ...opts.files,
   ];
 }
+
+// Test stacks of this run clone migrated template databases instead of repeating
+// the DDL per stack; the templates live exactly as long as the run.
+export async function runWithTestDbTemplates(run: () => Promise<number>): Promise<number> {
+  const { TEST_DB_TEMPLATE_RUN_ENV, dropTestDbTemplates } = await import("./stack/db.ts");
+  const runId = randomBytes(4).toString("hex");
+  try {
+    return await withEnv({ [TEST_DB_TEMPLATE_RUN_ENV]: runId }, run);
+  } finally {
+    const baseUrl = process.env["TEST_DATABASE_URL"] ?? SERVICE_ENV_DEFAULTS.TEST_DATABASE_URL;
+    try {
+      await dropTestDbTemplates(baseUrl, runId);
+    } catch (error) {
+      // biome-ignore lint/suspicious/noConsole: runner output is the interface
+      console.warn(`kumiko-testing: could not drop the test DB templates of run ${runId}:`, error);
+    }
+  }
+}

package/src/stack/db.ts

@@ -3,10 +3,13 @@
 
 import { DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX } from "@cosmicdrift/kumiko-framework/db";
 import {
+  acquireSessionAdvisoryLock,
   createDatabase,
   createDbConnection,
   databaseExists,
   dropDatabaseIfExists,
+  listDatabaseNamesWithPrefix,
+  renameDatabase,
 } from "@cosmicdrift/kumiko-framework/internal/stack-support";
 import { ensureTemporalPolyfill } from "@cosmicdrift/kumiko-framework/time";
 import { generateId } from "@cosmicdrift/kumiko-framework/utils";
@@ -39,14 +42,32 @@ export type CreateTestDbOptions = {
   readonly persistent?: boolean;
   readonly poolMax?: number;
   readonly outsideTransactionPoolMax?: number;
+  /** Clone this already-migrated database instead of starting empty. */
+  readonly templateDbName?: string;
 };
 
+// Set by the integration runner for one run; templates named after it are dropped when the run ends.
+export const TEST_DB_TEMPLATE_RUN_ENV = "KUMIKO_TEST_DB_TEMPLATE_RUN";
+
+// Inside the kumiko_test_ namespace, so clean-test-dbs also sweeps templates of killed runs.
+export function testDbTemplatePrefix(runId: string): string {
+  return `kumiko_test_tpl_${runId}_`;
+}
+
+function adminUrlOf(url: string): string {
+  return url.replace(/\/[^/]+$/, "/postgres");
+}
+
+function databaseUrlOf(url: string, dbName: string): string {
+  return url.replace(/\/[^/]+$/, `/${dbName}`);
+}
+
 export async function createTestDb(arg?: string | CreateTestDbOptions): Promise<TestDb> {
   await ensureTemporalPolyfill();
   const opts: CreateTestDbOptions = typeof arg === "string" ? { baseUrl: arg } : (arg ?? {});
   const url = opts.baseUrl ?? requireEnv("TEST_DATABASE_URL");
   const dbName = opts.dbName ?? `kumiko_test_${generateId().slice(-8)}`;
-  const adminUrl = url.replace(/\/[^/]+$/, "/postgres");
+  const adminUrl = adminUrlOf(url);
 
   const admin = createDbConnection({ url: adminUrl, maxConnections: 1 });
   try {
@@ -55,13 +76,13 @@ export async function createTestDb(arg?: string | CreateTestDbOptions): Promise<
         await createDatabase(admin.db, dbName);
       }
     } else {
-      await createDatabase(admin.db, dbName);
+      await createDatabase(admin.db, dbName, opts.templateDbName);
     }
   } finally {
     await admin.close();
   }
 
-  const testUrl = url.replace(/\/[^/]+$/, `/${dbName}`);
+  const testUrl = databaseUrlOf(url, dbName);
   const poolMax = opts.poolMax ?? DEFAULT_TEST_DB_POOL_MAX;
   const conn = createDbConnection({ url: testUrl, maxConnections: poolMax });
   const outsideConn = createDbConnection({
@@ -69,8 +90,10 @@ export async function createTestDb(arg?: string | CreateTestDbOptions): Promise<
     maxConnections: opts.outsideTransactionPoolMax ?? DEFAULT_OUTSIDE_TRANSACTION_POOL_MAX,
   });
 
-  const { createEventsTable } = await import("@cosmicdrift/kumiko-framework/event-store");
-  await createEventsTable(conn.db);
+  if (opts.templateDbName === undefined) {
+    const { createEventsTable } = await import("@cosmicdrift/kumiko-framework/event-store");
+    await createEventsTable(conn.db);
+  }
 
   return {
     db: conn.db,
@@ -93,4 +116,58 @@ export async function createTestDb(arg?: string | CreateTestDbOptions): Promise<
   };
 }
 
+const testDbTemplates = new Map<string, Promise<void>>();
+
+// One build per template and process; other workers of the run wait on the
+// advisory lock and reuse the finished database.
+export function ensureTestDbTemplate(
+  templateDbName: string,
+  buildSchema: (db: DbConnection) => Promise<void>,
+): Promise<void> {
+  const known = testDbTemplates.get(templateDbName);
+  if (known) return known;
+  const pending = buildTestDbTemplate(templateDbName, buildSchema);
+  testDbTemplates.set(templateDbName, pending);
+  pending.catch(() => testDbTemplates.delete(templateDbName));
+  return pending;
+}
+
+async function buildTestDbTemplate(
+  templateDbName: string,
+  buildSchema: (db: DbConnection) => Promise<void>,
+): Promise<void> {
+  const url = requireEnv("TEST_DATABASE_URL");
+  // One connection, so the session that takes the advisory lock is the one closed below.
+  const admin = createDbConnection({ url: adminUrlOf(url), maxConnections: 1 });
+  try {
+    await acquireSessionAdvisoryLock(admin.db, templateDbName);
+    // skip: another worker of this run already built it while we waited on the lock
+    if (await databaseExists(admin.db, templateDbName)) return;
+    // Built under a side name and renamed once complete, so a clone never sees a half-built schema.
+    const buildDbName = `${templateDbName}_build`;
+    await dropDatabaseIfExists(admin.db, buildDbName);
+    await createDatabase(admin.db, buildDbName);
+    const conn = createDbConnection({ url: databaseUrlOf(url, buildDbName), maxConnections: 1 });
+    try {
+      await buildSchema(conn.db);
+    } finally {
+      await conn.close();
+    }
+    await renameDatabase(admin.db, buildDbName, templateDbName);
+  } finally {
+    await admin.close();
+  }
+}
+
+export async function dropTestDbTemplates(baseUrl: string, runId: string): Promise<void> {
+  const admin = createDbConnection({ url: adminUrlOf(baseUrl), maxConnections: 1 });
+  try {
+    for (const name of await listDatabaseNamesWithPrefix(admin.db, testDbTemplatePrefix(runId))) {
+      await dropDatabaseIfExists(admin.db, name);
+    }
+  } finally {
+    await admin.close();
+  }
+}
+
 export { requireEnv };

package/src/stack/redis.ts

@@ -39,7 +39,9 @@ export async function createTestRedis(opts: CreateTestRedisOptions = {}): Promis
   // round-robin). Collisions at birthday-paradox rates are gone — the
   // prefix space is unbounded. See Track B.3 in docs/plans/tests-refactor.
   const prefix = opts.borrowKeyPrefix ?? `kt:${generateId().slice(-8)}:`;
-  const redis = new Redis(redisUrl, { keyPrefix: prefix });
+  // Callers that check redis.status (rate-limit resolver) must not see "connect" on their first call.
+  const redis = new Redis(redisUrl, { keyPrefix: prefix, lazyConnect: true });
+  await redis.connect();
 
   async function deleteKeysMatching(pattern: string): Promise<void> {
     // Open a prefix-less client for the scan — ioredis' keyPrefix is applied

package/src/stack/test-stack.ts

@@ -1,6 +1,8 @@
 // @runtime dev
+import { createHash } from "node:crypto";
 import type { AuthRoutesConfig, JwtHelper } from "@cosmicdrift/kumiko-framework/api";
 import { buildServer, createSseBroker, type SseBroker } from "@cosmicdrift/kumiko-framework/api";
+import type { DbConnection, EntityTableMeta } from "@cosmicdrift/kumiko-framework/db";
 import { createRegistry, validateBoot } from "@cosmicdrift/kumiko-framework/engine";
 import type {
   AppContext,
@@ -50,7 +52,12 @@ import type { SearchAdapter } from "@cosmicdrift/kumiko-framework/search";
 import { createInMemorySearchAdapter } from "@cosmicdrift/kumiko-framework/search";
 import { propertyOf } from "@cosmicdrift/kumiko-types/type-guards";
 import type { Hono } from "hono";
-import { createTestDb } from "./db.ts";
+import {
+  createTestDb,
+  ensureTestDbTemplate,
+  TEST_DB_TEMPLATE_RUN_ENV,
+  testDbTemplatePrefix,
+} from "./db.ts";
 import { createJobFailureTracker, drainJobs } from "./drain-jobs.ts";
 import { createEventCollector, type EventCollector } from "./event-collector.ts";
 import { createTestRedis, queueNamePrefixForTestRedis, type TestRedis } from "./redis.ts";
@@ -304,6 +311,87 @@ function bindMfaTokenSecrets(features: readonly FeatureDefinition[], jwtSecret:
   }
 }
 
+// Same table list as `kumiko schema generate` (collectTableMetas) — divergence
+// was the #255 prod-crash and the #3102 missing r.entity() tables.
+async function collectStackTableMetas(
+  options: TestStackOptions,
+): Promise<readonly EntityTableMeta[]> {
+  const { collectTableMetas, enumerateFeatureTableSources, extractTableInfo } = await import(
+    "@cosmicdrift/kumiko-framework/db"
+  );
+  const metas = collectTableMetas(options.features);
+  if (options.entityTables !== false) return metas;
+  const nonEntityNames = new Set(
+    options.features.flatMap((f) =>
+      enumerateFeatureTableSources(f).map((s) => extractTableInfo(s.table).name),
+    ),
+  );
+  return metas.filter((m) => nonEntityNames.has(m.tableName));
+}
+
+async function pushStackSchema(
+  db: DbConnection,
+  metas: readonly EntityTableMeta[],
+  withFileTables: boolean,
+): Promise<void> {
+  // Every ES-entity writes events via createEventStoreExecutor in the
+  // feature's write handlers. Auto-create the events table so every
+  // setupTestStack call is ready for writes without needing a manual
+  // createEventsTable().
+  await createEventsTable(db);
+  // Archive-stream metadata — needed by ctx.appendEvent's archive guard and
+  // loadAggregate's default-skip. Idempotent, so production boot running
+  // the same call is fine.
+  await createArchivedStreamsTable(db);
+
+  // Framework state for projection rebuild/status + event-consumer cursors.
+  // Idempotent — production boot flows run the same calls.
+  const { createProjectionStateTable, createEventConsumerStateTable } = await import(
+    "@cosmicdrift/kumiko-framework/pipeline"
+  );
+  await createProjectionStateTable(db);
+  await createEventConsumerStateTable(db);
+
+  // Files support: when a provider is registered, the fileRefs table must
+  // exist before the first upload. Skipped when no provider — the table
+  // stays off tenant test DBs that never touch files.
+  if (withFileTables) {
+    const { fileRefsTable, fileUploadLedgerTable } = await import(
+      "@cosmicdrift/kumiko-framework/files"
+    );
+    await unsafePushTables(db, { fileRefsTable, fileUploadLedgerTable });
+  }
+
+  const { tableExists } = await import("@cosmicdrift/kumiko-framework/db");
+  const missing: Record<string, unknown> = {};
+  for (const meta of metas) {
+    if (await tableExists(db, `public.${meta.tableName}`)) continue;
+    missing[meta.tableName] = meta;
+  }
+  if (Object.keys(missing).length > 0) {
+    await unsafePushTables(db, missing);
+  }
+}
+
+// Only inside an integration-runner run: the framework tables above are not part
+// of the fingerprint, which holds because one run never sees two framework versions.
+async function resolveTestDbTemplateName(
+  options: TestStackOptions,
+  metas: readonly EntityTableMeta[],
+  withFileTables: boolean,
+): Promise<string | undefined> {
+  const runId = process.env[TEST_DB_TEMPLATE_RUN_ENV];
+  if (runId === undefined || options.persistentDb === true || options.dbName !== undefined) {
+    return undefined;
+  }
+  const { renderTablesDdl } = await import("@cosmicdrift/kumiko-framework/db");
+  const fingerprint = createHash("sha256")
+    .update(JSON.stringify({ withFileTables, ddl: renderTablesDdl(metas) }))
+    .digest("hex")
+    .slice(0, 16);
+  return `${testDbTemplatePrefix(runId)}${fingerprint}`;
+}
+
 export async function setupTestStack(options: TestStackOptions): Promise<TestStack> {
   const jwtSecret = options.jwtSecret ?? DEFAULT_JWT_SECRET;
   const enabledHooks = options.systemHooks ?? ["sse", "search"];
@@ -325,6 +413,13 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
   if (options.validateBoot === "full") validateBoot(options.features);
   bindMfaTokenSecrets(options.features, jwtSecret);
 
+  const stackTableMetas = await collectStackTableMetas(options);
+  const withFileTables = options.files !== undefined;
+  const buildStackSchema = (db: DbConnection): Promise<void> =>
+    pushStackSchema(db, stackTableMetas, withFileTables);
+  const templateDbName = await resolveTestDbTemplateName(options, stackTableMetas, withFileTables);
+  if (templateDbName !== undefined) await ensureTestDbTemplate(templateDbName, buildStackSchema);
+
   // Forward db-name/persistent-flag through to createTestDb. The
   // defaults (undefined dbName, persistent:false) keep the legacy
   // test contract: fresh kumiko_test_<random> DB per setup, dropped
@@ -337,6 +432,7 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
       ...(options.outsideTransactionPoolMax !== undefined && {
         outsideTransactionPoolMax: options.outsideTransactionPoolMax,
       }),
+      ...(templateDbName !== undefined && { templateDbName }),
     }),
     createTestRedis(
       options.sharedRedisWith && { borrowKeyPrefix: options.sharedRedisWith.redis.keyPrefix },
@@ -347,70 +443,22 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
   // DB/Redis — the caller never gets the stack object back to call
   // cleanup() itself (same "leaked ephemeral DB" failure class as the later
   // try/catch below, but for setup that runs before that try starts).
-  try {
-    // Every ES-entity writes events via createEventStoreExecutor in the
-    // feature's write handlers. Auto-create the events table so every
-    // setupTestStack call is ready for writes without needing a manual
-    // createEventsTable().
-    await createEventsTable(testDb.db);
-    // Archive-stream metadata — needed by ctx.appendEvent's archive guard and
-    // loadAggregate's default-skip. Idempotent, so production boot running
-    // the same call is fine.
-    await createArchivedStreamsTable(testDb.db);
-
-    // Framework state for projection rebuild/status + event-consumer cursors.
-    // Idempotent — production boot flows run the same calls.
-    const { createProjectionStateTable, createEventConsumerStateTable } = await import(
-      "@cosmicdrift/kumiko-framework/pipeline"
-    );
-    await createProjectionStateTable(testDb.db);
-    await createEventConsumerStateTable(testDb.db);
-
-    // Files support: when a provider is registered, the fileRefs table must
-    // exist before the first upload. Skipped when no provider — the table
-    // stays off tenant test DBs that never touch files.
-    if (options.files) {
-      const { fileRefsTable, fileUploadLedgerTable } = await import(
-        "@cosmicdrift/kumiko-framework/files"
-      );
-      await unsafePushTables(testDb.db, { fileRefsTable, fileUploadLedgerTable });
-    }
-
-    // Same table list as `kumiko schema generate` (collectTableMetas) — divergence
-    // was the #255 prod-crash and the #3102 missing r.entity() tables.
-    const { collectTableMetas } = await import("@cosmicdrift/kumiko-framework/db");
-    const { tableExists } = await import("@cosmicdrift/kumiko-framework/db");
-    let metas = collectTableMetas(options.features);
-    if (options.entityTables === false) {
-      const { enumerateFeatureTableSources } = await import("@cosmicdrift/kumiko-framework/db");
-      const { extractTableInfo } = await import("@cosmicdrift/kumiko-framework/db");
-      const nonEntityNames = new Set(
-        options.features.flatMap((f) =>
-          enumerateFeatureTableSources(f).map((s) => extractTableInfo(s.table).name),
-        ),
-      );
-      metas = metas.filter((m) => nonEntityNames.has(m.tableName));
-    }
-    const missing: Record<string, unknown> = {};
-    for (const meta of metas) {
-      if (await tableExists(testDb.db, `public.${meta.tableName}`)) continue;
-      missing[meta.tableName] = meta;
-    }
-    if (Object.keys(missing).length > 0) {
-      await unsafePushTables(testDb.db, missing);
-    }
-  } catch (error) {
+  if (templateDbName === undefined) {
     try {
-      await testDb.cleanup();
-    } catch {
-      // ignore — `error` is the one that matters
-    }
-    try {
-      await testRedis.cleanup();
-    } catch {
-      // ignore — `error` is the one that matters
+      await buildStackSchema(testDb.db);
+    } catch (error) {
+      try {
+        await testDb.cleanup();
+      } catch {
+        // ignore — `error` is the one that matters
+      }
+      try {
+        await testRedis.cleanup();
+      } catch {
+        // ignore — `error` is the one that matters
+      }
+      throw error;
     }
-    throw error;
   }
 
   // Redis only where two stacks can actually disagree (avoids two extra