@elench/testkit 0.1.13 → 0.1.15

package/lib/database.mjs

@@ -0,0 +1,656 @@
+import crypto from "crypto";
+import fs from "fs";
+import path from "path";
+import { execa } from "execa";
+import { requireNeonApiKey, resolveServiceCwd } from "./config.mjs";
+import { runScript } from "./exec.mjs";
+
+const LOCAL_IMAGE = "postgres:16";
+const LOCAL_USER = "testkit";
+const LOCAL_PASSWORD = "testkit";
+const LOCAL_ADMIN_DB = "postgres";
+const LOCAL_READY_TIMEOUT_MS = 60_000;
+const LOCAL_POLL_INTERVAL_MS = 1_000;
+
+export async function prepareDatabaseRuntime(config, hooks = {}) {
+  const db = config.testkit.database;
+  if (!db) return;
+
+  fs.mkdirSync(config.stateDir, { recursive: true });
+  if (db.provider === "neon") {
+    await prepareNeonDatabase(config, hooks);
+    return;
+  }
+  if (db.provider === "local") {
+    await prepareLocalDatabase(config, hooks);
+    return;
+  }
+
+  throw new Error(`Unsupported database provider "${db.provider}"`);
+}
+
+export async function destroyRuntimeDatabase({ productDir, stateDir }) {
+  const backend = readStateValue(path.join(stateDir, "database_backend"));
+  if (!backend || backend === "neon") {
+    await destroyNeonDatabase(stateDir);
+    return;
+  }
+
+  if (backend === "local") {
+    await destroyLocalRuntimeDatabase(productDir, stateDir);
+    return;
+  }
+}
+
+export async function destroyServiceDatabaseCache(productDir, serviceName) {
+  const cacheDir = getLocalServiceCacheDir(productDir, serviceName);
+  if (!fs.existsSync(cacheDir)) return;
+
+  const backend = readStateValue(path.join(cacheDir, "database_backend"));
+  if (backend !== "local") {
+    fs.rmSync(cacheDir, { recursive: true, force: true });
+    return;
+  }
+
+  const lockDir = getLocalLocksDir(productDir);
+  fs.mkdirSync(lockDir, { recursive: true });
+  await withLock(path.join(lockDir, `template-${serviceName}.lock`), async () => {
+    const infra = await loadExistingLocalContainer(productDir);
+    if (!infra) {
+      fs.rmSync(cacheDir, { recursive: true, force: true });
+      return;
+    }
+    const templateDbName = readStateValue(path.join(cacheDir, "template_database_name"));
+    if (templateDbName) {
+      await dropDatabaseIfExists(infra, templateDbName);
+    }
+    fs.rmSync(cacheDir, { recursive: true, force: true });
+  });
+}
+
+export async function cleanupOrphanedLocalInfrastructure(productDir) {
+  const infraDir = getLocalInfraDir(productDir);
+  if (!fs.existsSync(infraDir)) return;
+
+  if (hasRemainingLocalArtifacts(productDir)) return;
+
+  const containerName = readStateValue(path.join(infraDir, "container_name"));
+  if (containerName) {
+    await stopAndRemoveContainer(containerName);
+  }
+  fs.rmSync(infraDir, { recursive: true, force: true });
+}
+
+export function isDatabaseStateDir(dir) {
+  if (!fs.existsSync(dir)) return false;
+  return (
+    fs.existsSync(path.join(dir, "database_backend")) ||
+    fs.existsSync(path.join(dir, "neon_branch_id")) ||
+    fs.existsSync(path.join(dir, "neon_project_id"))
+  );
+}
+
+export function showServiceDatabaseStatus(productDir, serviceName) {
+  const cacheDir = getLocalServiceCacheDir(productDir, serviceName);
+  const infraDir = getLocalInfraDir(productDir);
+  if (!fs.existsSync(cacheDir) && !fs.existsSync(infraDir)) return false;
+
+  if (fs.existsSync(cacheDir)) {
+    console.log("  database-cache/");
+    printStateDir(cacheDir, "    ");
+  }
+
+  if (fs.existsSync(infraDir)) {
+    console.log("  database-infra/");
+    printStateDir(infraDir, "    ");
+  }
+
+  return true;
+}
+
+async function prepareNeonDatabase(config, hooks) {
+  const db = config.testkit.database;
+  requireNeonApiKey();
+
+  removeLocalRuntimeState(config.stateDir);
+
+  await runScript("neon-up.sh", {
+    NEON_PROJECT_ID: db.projectId,
+    NEON_DB_NAME: db.dbName,
+    NEON_BRANCH_NAME: db.branchName,
+    NEON_RESET: db.reset === false ? "false" : "true",
+    STATE_DIR: config.stateDir,
+  });
+
+  fs.writeFileSync(path.join(config.stateDir, "database_backend"), "neon");
+  fs.writeFileSync(path.join(config.stateDir, "neon_project_id"), db.projectId);
+
+  const databaseUrl = readStateValue(path.join(config.stateDir, "database_url"));
+  if (hooks.runMigrate) {
+    await hooks.runMigrate(databaseUrl);
+  }
+  if (hooks.runSeed) {
+    await hooks.runSeed(databaseUrl);
+  }
+}
+
+async function destroyNeonDatabase(stateDir) {
+  const projectId = readStateValue(path.join(stateDir, "neon_project_id"));
+  if (!projectId) return;
+
+  await runScript("neon-down.sh", {
+    NEON_PROJECT_ID: projectId,
+    STATE_DIR: stateDir,
+  });
+}
+
+async function prepareLocalDatabase(config, hooks) {
+  const db = config.testkit.database;
+  const productDir = config.productDir;
+  const serviceName = config.name;
+  const lockDir = getLocalLocksDir(productDir);
+  const cacheDir = getLocalServiceCacheDir(productDir, serviceName);
+  fs.mkdirSync(lockDir, { recursive: true });
+  fs.mkdirSync(cacheDir, { recursive: true });
+
+  const templateFingerprint = await computeTemplateFingerprint(config);
+  const infra = await withLock(path.join(lockDir, "container.lock"), () =>
+    ensureLocalContainer(productDir, db)
+  );
+
+  await withLock(path.join(lockDir, `template-${serviceName}.lock`), async () => {
+    await ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint, hooks);
+  });
+
+  await withLock(path.join(lockDir, `runtime-${serviceName}-${hashString(config.stateDir, 10)}.lock`), async () => {
+    await ensureWorkerClone(config, infra, cacheDir, templateFingerprint);
+  });
+}
+
+async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint, hooks) {
+  const serviceName = config.name;
+  const existingFingerprint = readStateValue(path.join(cacheDir, "template_fingerprint"));
+  const existingDbName = readStateValue(path.join(cacheDir, "template_database_name"));
+  const desiredDbName = buildTemplateDatabaseName(serviceName, templateFingerprint);
+
+  if (
+    existingFingerprint === templateFingerprint &&
+    existingDbName &&
+    (await databaseExists(infra, existingDbName))
+  ) {
+    writeLocalCacheState(cacheDir, infra, existingDbName, templateFingerprint);
+    return;
+  }
+
+  if (existingDbName && existingDbName !== desiredDbName) {
+    await dropDatabaseIfExists(infra, existingDbName);
+  }
+  if (await databaseExists(infra, desiredDbName)) {
+    await dropDatabaseIfExists(infra, desiredDbName);
+  }
+
+  await createEmptyDatabase(infra, desiredDbName);
+  const templateUrl = buildDatabaseUrl(infra, desiredDbName);
+  if (hooks.runMigrate) {
+    await hooks.runMigrate(templateUrl);
+  }
+  if (hooks.runSeed) {
+    await hooks.runSeed(templateUrl);
+  }
+
+  writeLocalCacheState(cacheDir, infra, desiredDbName, templateFingerprint);
+}
+
+async function ensureWorkerClone(config, infra, cacheDir, templateFingerprint) {
+  const serviceName = config.name;
+  const templateDbName = readStateValue(path.join(cacheDir, "template_database_name"));
+  if (!templateDbName) {
+    throw new Error(`Missing template database for service "${serviceName}"`);
+  }
+
+  const desiredDbName = buildWorkerDatabaseName(serviceName, config.stateDir, templateFingerprint);
+  const existingDbName = readStateValue(path.join(config.stateDir, "local_database_name"));
+  const existingFingerprint = readStateValue(path.join(config.stateDir, "local_template_fingerprint"));
+  const needsReset =
+    config.testkit.database.reset !== false ||
+    existingFingerprint !== templateFingerprint ||
+    existingDbName !== desiredDbName ||
+    !(await databaseExists(infra, desiredDbName));
+
+  if (existingDbName && existingDbName !== desiredDbName) {
+    await dropDatabaseIfExists(infra, existingDbName);
+  }
+
+  if (needsReset) {
+    await dropDatabaseIfExists(infra, desiredDbName);
+    await cloneDatabaseFromTemplate(infra, desiredDbName, templateDbName);
+  }
+
+  fs.mkdirSync(config.stateDir, { recursive: true });
+  removeNeonRuntimeState(config.stateDir);
+  fs.writeFileSync(path.join(config.stateDir, "database_backend"), "local");
+  fs.writeFileSync(path.join(config.stateDir, "database_url"), buildDatabaseUrl(infra, desiredDbName));
+  fs.writeFileSync(path.join(config.stateDir, "local_database_name"), desiredDbName);
+  fs.writeFileSync(path.join(config.stateDir, "local_template_fingerprint"), templateFingerprint);
+  fs.writeFileSync(path.join(config.stateDir, "local_template_database_name"), templateDbName);
+  fs.writeFileSync(path.join(config.stateDir, "local_container_name"), infra.containerName);
+}
+
+async function destroyLocalRuntimeDatabase(productDir, stateDir) {
+  const dbName = readStateValue(path.join(stateDir, "local_database_name"));
+  if (!dbName) return;
+
+  const infra = await loadExistingLocalContainer(productDir);
+  if (!infra) return;
+  await dropDatabaseIfExists(infra, dbName);
+}
+
+async function ensureLocalContainer(productDir, database = {}) {
+  const infraDir = getLocalInfraDir(productDir);
+  fs.mkdirSync(infraDir, { recursive: true });
+
+  const containerName =
+    readStateValue(path.join(infraDir, "container_name")) || buildContainerName(productDir);
+  const image = database.image || readStateValue(path.join(infraDir, "image")) || LOCAL_IMAGE;
+  const user = database.user || readStateValue(path.join(infraDir, "user")) || LOCAL_USER;
+  const password =
+    database.password || readStateValue(path.join(infraDir, "password")) || LOCAL_PASSWORD;
+
+  let inspect = await inspectContainer(containerName);
+  if (inspect && inspect.Config?.Image !== image) {
+    await stopAndRemoveContainer(containerName);
+    inspect = null;
+  }
+
+  if (!inspect) {
+    await execa("docker", [
+      "run",
+      "-d",
+      "--name",
+      containerName,
+      "-e",
+      `POSTGRES_USER=${user}`,
+      "-e",
+      `POSTGRES_PASSWORD=${password}`,
+      "-e",
+      `POSTGRES_DB=${LOCAL_ADMIN_DB}`,
+      "-p",
+      "127.0.0.1::5432",
+      image,
+    ]);
+    inspect = await inspectContainer(containerName);
+  } else if (!inspect.State?.Running) {
+    await execa("docker", ["start", containerName]);
+    inspect = await inspectContainer(containerName);
+  }
+
+  const hostPort = inspect?.NetworkSettings?.Ports?.["5432/tcp"]?.[0]?.HostPort;
+  if (!hostPort) {
+    throw new Error(`Could not determine published port for local database container ${containerName}`);
+  }
+
+  const infra = {
+    containerName,
+    containerId: inspect.Id,
+    image,
+    user,
+    password,
+    host: "127.0.0.1",
+    port: Number(hostPort),
+  };
+
+  await waitForLocalContainerReady(infra);
+  writeLocalInfraState(infraDir, infra);
+  return infra;
+}
+
+async function loadExistingLocalContainer(productDir) {
+  const infraDir = getLocalInfraDir(productDir);
+  const containerName = readStateValue(path.join(infraDir, "container_name"));
+  if (!containerName) return null;
+
+  const inspect = await inspectContainer(containerName);
+  if (!inspect) return null;
+
+  if (!inspect.State?.Running) {
+    await execa("docker", ["start", containerName]);
+  }
+
+  const nextInspect = await inspectContainer(containerName);
+  const hostPort = nextInspect?.NetworkSettings?.Ports?.["5432/tcp"]?.[0]?.HostPort;
+  if (!hostPort) return null;
+
+  const infra = {
+    containerName,
+    containerId: nextInspect.Id,
+    image: nextInspect.Config?.Image || readStateValue(path.join(infraDir, "image")) || LOCAL_IMAGE,
+    user: readStateValue(path.join(infraDir, "user")) || LOCAL_USER,
+    password: readStateValue(path.join(infraDir, "password")) || LOCAL_PASSWORD,
+    host: "127.0.0.1",
+    port: Number(hostPort),
+  };
+  await waitForLocalContainerReady(infra);
+  return infra;
+}
+
+async function waitForLocalContainerReady(infra) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < LOCAL_READY_TIMEOUT_MS) {
+    try {
+      await execa("docker", [
+        "exec",
+        infra.containerName,
+        "pg_isready",
+        "-U",
+        infra.user,
+        "-d",
+        LOCAL_ADMIN_DB,
+      ]);
+      return;
+    } catch {
+      await sleep(LOCAL_POLL_INTERVAL_MS);
+    }
+  }
+
+  throw new Error(`Timed out waiting for local database container ${infra.containerName}`);
+}
+
+async function inspectContainer(containerName) {
+  try {
+    const { stdout } = await execa("docker", ["inspect", containerName]);
+    const parsed = JSON.parse(stdout);
+    return parsed[0] || null;
+  } catch {
+    return null;
+  }
+}
+
+async function stopAndRemoveContainer(containerName) {
+  try {
+    await execa("docker", ["rm", "-f", containerName]);
+  } catch {
+    // Already gone.
+  }
+}
+
+async function databaseExists(infra, dbName) {
+  const result = await runAdminQuery(infra, [
+    "-tAc",
+    `SELECT 1 FROM pg_database WHERE datname = '${escapeSqlLiteral(dbName)}'`,
+  ]);
+  return result.trim() === "1";
+}
+
+async function createEmptyDatabase(infra, dbName) {
+  await runAdminQuery(infra, ["-c", `CREATE DATABASE "${escapeIdentifier(dbName)}"`]);
+}
+
+async function cloneDatabaseFromTemplate(infra, dbName, templateDbName) {
+  await runAdminQuery(infra, [
+    "-c",
+    `CREATE DATABASE "${escapeIdentifier(dbName)}" TEMPLATE "${escapeIdentifier(templateDbName)}"`,
+  ]);
+}
+
+async function dropDatabaseIfExists(infra, dbName) {
+  await runAdminQuery(infra, [
+    "-c",
+    [
+      `SELECT pg_terminate_backend(pid)`,
+      `FROM pg_stat_activity`,
+      `WHERE datname = '${escapeSqlLiteral(dbName)}' AND pid <> pg_backend_pid();`,
+    ].join(" "),
+  ]);
+  await runAdminQuery(infra, [
+    "-c",
+    `DROP DATABASE IF EXISTS "${escapeIdentifier(dbName)}"`,
+  ]);
+}
+
+async function runAdminQuery(infra, args) {
+  const commandArgs = [
+    "exec",
+    "-e",
+    `PGPASSWORD=${infra.password}`,
+    infra.containerName,
+    "psql",
+    "-v",
+    "ON_ERROR_STOP=1",
+    "-U",
+    infra.user,
+    "-d",
+    LOCAL_ADMIN_DB,
+    ...args,
+  ];
+  const { stdout } = await execa("docker", commandArgs);
+  return stdout;
+}
+
+async function computeTemplateFingerprint(config) {
+  const hash = crypto.createHash("sha256");
+  const db = config.testkit.database;
+  hash.update(JSON.stringify({
+    provider: db.provider,
+    selectedBackend: db.selectedBackend,
+    image: db.image || LOCAL_IMAGE,
+    user: db.user || LOCAL_USER,
+    migrate: config.testkit.migrate || null,
+    seed: config.testkit.seed || null,
+  }));
+
+  const rootEnvPath = path.join(config.productDir, ".env");
+  appendFileToHash(hash, config.productDir, rootEnvPath);
+  for (const envFile of config.testkit.envFiles || []) {
+    appendFileToHash(hash, config.productDir, resolveServiceCwd(config.productDir, envFile));
+  }
+  for (const input of db.template.inputs || []) {
+    appendInputToHash(hash, config.productDir, input);
+  }
+
+  return hash.digest("hex");
+}
+
+function appendInputToHash(hash, productDir, input) {
+  const absPath = resolveServiceCwd(productDir, input);
+  if (!fs.existsSync(absPath)) {
+    hash.update(`missing:${path.relative(productDir, absPath)}`);
+    return;
+  }
+
+  const stat = fs.statSync(absPath);
+  if (stat.isDirectory()) {
+    hash.update(`dir:${path.relative(productDir, absPath)}`);
+    for (const entry of fs.readdirSync(absPath).sort()) {
+      if (entry === ".git" || entry === "node_modules" || entry === ".testkit") continue;
+      appendInputToHash(hash, productDir, path.join(input, entry));
+    }
+    return;
+  }
+
+  appendFileToHash(hash, productDir, absPath);
+}
+
+function appendFileToHash(hash, productDir, absPath) {
+  if (!fs.existsSync(absPath)) {
+    hash.update(`missing:${path.relative(productDir, absPath)}`);
+    return;
+  }
+  const stat = fs.statSync(absPath);
+  if (!stat.isFile()) return;
+
+  hash.update(`file:${path.relative(productDir, absPath)}:${stat.size}:${stat.mtimeMs}`);
+  hash.update(fs.readFileSync(absPath));
+}
+
+function buildDatabaseUrl(infra, dbName) {
+  return `postgresql://${encodeURIComponent(infra.user)}:${encodeURIComponent(infra.password)}@${infra.host}:${infra.port}/${dbName}?sslmode=disable`;
+}
+
+function buildContainerName(productDir) {
+  return limitIdentifier(
+    `testkit_pg_${slugSegment(path.basename(productDir))}_${hashString(productDir, 10)}`,
+    63
+  );
+}
+
+function buildTemplateDatabaseName(serviceName, fingerprint) {
+  return limitIdentifier(
+    `tk_tpl_${slugSegment(serviceName)}_${fingerprint.slice(0, 16)}`,
+    63
+  );
+}
+
+function buildWorkerDatabaseName(serviceName, stateDir, fingerprint) {
+  return limitIdentifier(
+    `tk_${slugSegment(serviceName)}_${hashString(stateDir, 10)}_${fingerprint.slice(0, 12)}`,
+    63
+  );
+}
+
+function writeLocalInfraState(infraDir, infra) {
+  fs.writeFileSync(path.join(infraDir, "database_backend"), "local");
+  fs.writeFileSync(path.join(infraDir, "container_name"), infra.containerName);
+  fs.writeFileSync(path.join(infraDir, "container_id"), infra.containerId);
+  fs.writeFileSync(path.join(infraDir, "image"), infra.image);
+  fs.writeFileSync(path.join(infraDir, "user"), infra.user);
+  fs.writeFileSync(path.join(infraDir, "password"), infra.password);
+  fs.writeFileSync(path.join(infraDir, "host"), infra.host);
+  fs.writeFileSync(path.join(infraDir, "port"), String(infra.port));
+}
+
+function writeLocalCacheState(cacheDir, infra, templateDbName, fingerprint) {
+  fs.writeFileSync(path.join(cacheDir, "database_backend"), "local");
+  fs.writeFileSync(path.join(cacheDir, "template_database_name"), templateDbName);
+  fs.writeFileSync(path.join(cacheDir, "template_fingerprint"), fingerprint);
+  fs.writeFileSync(path.join(cacheDir, "container_name"), infra.containerName);
+}
+
+function getLocalInfraDir(productDir) {
+  return path.join(productDir, ".testkit", "_infra", "local-postgres");
+}
+
+function getLocalLocksDir(productDir) {
+  return path.join(getLocalInfraDir(productDir), "locks");
+}
+
+function getLocalServiceCacheDir(productDir, serviceName) {
+  return path.join(productDir, ".testkit", "_dbcache", serviceName);
+}
+
+function hasRemainingLocalArtifacts(productDir) {
+  const root = path.join(productDir, ".testkit");
+  if (!fs.existsSync(root)) return false;
+
+  let found = false;
+  visitDirs(root, (dir) => {
+    if (found) return;
+    if (dir.includes(`${path.sep}.testkit${path.sep}_infra`)) return;
+    if (readStateValue(path.join(dir, "database_backend")) === "local") {
+      found = true;
+    }
+  });
+
+  if (found) return true;
+
+  const cacheRoot = path.join(productDir, ".testkit", "_dbcache");
+  if (!fs.existsSync(cacheRoot)) return false;
+  return fs.readdirSync(cacheRoot).some((entry) => {
+    const entryPath = path.join(cacheRoot, entry);
+    return fs.statSync(entryPath).isDirectory();
+  });
+}
+
+function visitDirs(root, visitor) {
+  if (!fs.existsSync(root)) return;
+  for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const dir = path.join(root, entry.name);
+    visitor(dir);
+    visitDirs(dir, visitor);
+  }
+}
+
+function printStateDir(dir, indent) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const filePath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      console.log(`${indent}${entry.name}/`);
+      printStateDir(filePath, `${indent}  `);
+      continue;
+    }
+    const value =
+      entry.name === "password" ? "********" : fs.readFileSync(filePath, "utf8").trim();
+    console.log(`${indent}${entry.name}: ${value}`);
+  }
+}
+
+async function withLock(lockPath, fn) {
+  const lockDir = `${lockPath}.dir`;
+  const timeoutMs = 60_000;
+  const startedAt = Date.now();
+
+  while (true) {
+    try {
+      fs.mkdirSync(lockDir, { recursive: false });
+      break;
+    } catch (error) {
+      if (error.code !== "EEXIST") throw error;
+      if (Date.now() - startedAt > timeoutMs) {
+        throw new Error(`Timed out waiting for lock ${path.basename(lockPath)}`);
+      }
+      await sleep(200);
+    }
+  }
+
+  try {
+    return await fn();
+  } finally {
+    fs.rmSync(lockDir, { recursive: true, force: true });
+  }
+}
+
+function hashString(value, length = 12) {
+  return crypto.createHash("sha256").update(value).digest("hex").slice(0, length);
+}
+
+function slugSegment(value) {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") || "db";
+}
+
+function limitIdentifier(value, maxLength) {
+  return value.length <= maxLength ? value : value.slice(0, maxLength);
+}
+
+function escapeIdentifier(value) {
+  return String(value).replace(/"/g, "\"\"");
+}
+
+function escapeSqlLiteral(value) {
+  return String(value).replace(/'/g, "''");
+}
+
+function readStateValue(filePath) {
+  if (!fs.existsSync(filePath)) return null;
+  return fs.readFileSync(filePath, "utf8").trim();
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function removeLocalRuntimeState(stateDir) {
+  for (const file of [
+    "local_database_name",
+    "local_template_fingerprint",
+    "local_template_database_name",
+    "local_container_name",
+  ]) {
+    fs.rmSync(path.join(stateDir, file), { force: true });
+  }
+}
+
+function removeNeonRuntimeState(stateDir) {
+  for (const file of ["neon_project_id", "neon_branch_id"]) {
+    fs.rmSync(path.join(stateDir, file), { force: true });
+  }
+}