@elench/testkit 0.1.144 → 0.1.146

package/README.md

@@ -61,6 +61,7 @@ npx @elench/testkit --type int --write-status
 npx @elench/testkit status
 npx @elench/testkit destroy
 npx @elench/testkit cleanup
+npx @elench/testkit cleanup --resources --global --dry-run
 
 # Local production environment
 npx @elench/testkit local up
@@ -367,6 +368,15 @@ environment state. Testkit only supports local environments here; it does not
 copy production data and it refuses managed runtime database URLs that are not
 loopback PostgreSQL URLs.
 
+Testkit-owned Docker resources are labelled at creation time so they can be
+found again even if product-local `.testkit` state is deleted. `testkit destroy`
+removes labelled resources for the current product after service state is
+destroyed. `testkit cleanup --resources` cleans stale resources for the current
+product; add `--global` to clean labelled resources whose product directory no
+longer exists, and add `--include-legacy` only when you intentionally want to
+remove old unlabelled `testkit_pg_*` containers from pre-ownership Testkit
+versions. Use `--dry-run` first when cleaning globally.
+
 `database.template` is the database-side equivalent for reusable template DB
 state. When `database.sourceSchema` is configured, Testkit treats the configured
 source database as the schema source of truth. A normal `testkit run` resolves a

package/lib/cli/commands/cleanup.mjs

@@ -19,6 +19,18 @@ export default class CleanupCommand extends Command {
       multiple: true,
       options: ["runtime", "bundles", "assistant", "all"],
     }),
+    resources: Flags.boolean({
+      description: "Clean Testkit-owned Docker resources that are stale or orphaned",
+      default: false,
+    }),
+    global: Flags.boolean({
+      description: "Find orphaned Testkit-owned resources across all products",
+      default: false,
+    }),
+    "include-legacy": Flags.boolean({
+      description: "Include old unlabelled testkit_pg_* Docker containers in resource cleanup",
+      default: false,
+    }),
   };
 
   async run() {

package/lib/cli/operations/cleanup/operation.mjs

@@ -9,5 +9,8 @@ export async function executeCleanupOperation(flags = {}) {
     serviceName: flags.service || null,
     dryRun: flags["dry-run"],
     cache: flags.cache || [],
+    resources: flags.resources,
+    globalResources: flags.global,
+    includeLegacyResources: flags["include-legacy"],
   });
 }

package/lib/cli/operations/db/schema/refresh/operation.mjs

@@ -3,7 +3,11 @@ import os from "os";
 import path from "path";
 import { loadManagedConfigs, resolveTargetConfig, collectRequiredConfigs, topologicallySortConfigs } from "../../../../../app/configs.mjs";
 import { resolveProductDir } from "../../../../../config/index.mjs";
-import { prepareDatabaseRuntime } from "../../../../../database/index.mjs";
+import {
+  cleanupOrphanedLocalInfrastructure,
+  destroyRuntimeDatabase,
+  prepareDatabaseRuntime,
+} from "../../../../../database/index.mjs";
 import { forceRefreshSourceSchemaCache } from "../../../../../database/schema-source.mjs";
 import { createRunReporter } from "../../../../renderers/run/text-reporter.mjs";
 import { createRunLogRegistry } from "../../../../../runner/logs.mjs";
@@ -28,10 +32,12 @@ export async function executeDatabaseSchemaRefreshOperation(options = {}) {
   const reporter = createRunReporter({ outputMode: options.debug ? "debug" : "compact" });
   const logRegistry = createRunLogRegistry(productDir);
   const setupRegistry = createSetupOperationRegistry({ logRegistry });
+  const preparedStateDirs = [];
   try {
     for (const config of topologicallySortConfigs(resolvedConfigs)) {
       if (config.name === resolvedTarget.name) break;
       if (config.testkit.database) {
+        preparedStateDirs.push(config.stateDir);
         await prepareDatabaseRuntime(config, { reporter, logRegistry, setupRegistry });
       }
     }
@@ -54,6 +60,10 @@ export async function executeDatabaseSchemaRefreshOperation(options = {}) {
       reusedExistingRefresh: Boolean(state.refreshInfo?.reusedExistingRefresh),
     };
   } finally {
+    for (const stateDir of preparedStateDirs.reverse()) {
+      await destroyRuntimeDatabase({ productDir, stateDir }).catch(() => {});
+    }
+    await cleanupOrphanedLocalInfrastructure(productDir).catch(() => {});
     logRegistry.closeAll();
     fs.rmSync(runtimeRoot, { recursive: true, force: true });
   }

package/lib/cli/operations/db/schema/verify/operation.mjs

@@ -3,7 +3,11 @@ import os from "os";
 import path from "path";
 import { loadManagedConfigs, resolveTargetConfig, collectRequiredConfigs, topologicallySortConfigs } from "../../../../../app/configs.mjs";
 import { resolveProductDir } from "../../../../../config/index.mjs";
-import { prepareDatabaseRuntime } from "../../../../../database/index.mjs";
+import {
+  cleanupOrphanedLocalInfrastructure,
+  destroyRuntimeDatabase,
+  prepareDatabaseRuntime,
+} from "../../../../../database/index.mjs";
 import { createRunReporter } from "../../../../renderers/run/text-reporter.mjs";
 import { createRunLogRegistry } from "../../../../../runner/logs.mjs";
 import { createSetupOperationRegistry } from "../../../../../runner/setup-operations.mjs";
@@ -27,9 +31,11 @@ export async function executeDatabaseSchemaVerifyOperation(options = {}) {
   const reporter = createRunReporter({ outputMode: options.debug ? "debug" : "compact" });
   const logRegistry = createRunLogRegistry(productDir);
   const setupRegistry = createSetupOperationRegistry({ logRegistry });
+  const preparedStateDirs = [];
   try {
     for (const config of topologicallySortConfigs(resolvedConfigs)) {
       if (config.testkit.database) {
+        preparedStateDirs.push(config.stateDir);
         await prepareDatabaseRuntime(config, {
           reporter,
           logRegistry,
@@ -44,6 +50,10 @@ export async function executeDatabaseSchemaVerifyOperation(options = {}) {
       service: target.name,
     };
   } finally {
+    for (const stateDir of preparedStateDirs.reverse()) {
+      await destroyRuntimeDatabase({ productDir, stateDir }).catch(() => {});
+    }
+    await cleanupOrphanedLocalInfrastructure(productDir).catch(() => {});
     logRegistry.closeAll();
     fs.rmSync(runtimeRoot, { recursive: true, force: true });
   }

package/lib/cli/operations/destroy/operation.mjs

@@ -1,12 +1,17 @@
 import * as runner from "../../../runner/index.mjs";
 import { loadManagedConfigs } from "../../../app/configs.mjs";
+import { destroyOwnedLocalDatabaseResources } from "../../../database/index.mjs";
 
 export async function executeDestroyOperation(flags = {}) {
-  const { configs } = await loadManagedConfigs({ dir: flags.dir, service: flags.service });
+  const { allConfigs, configs } = await loadManagedConfigs({ dir: flags.dir, service: flags.service });
   const results = [];
   for (const config of configs) {
     await runner.destroy(config);
     results.push({ name: config.name, destroyed: true });
   }
+  const productDir = allConfigs[0]?.productDir || process.cwd();
+  if (!flags.service) {
+    await destroyOwnedLocalDatabaseResources(productDir);
+  }
   return results;
 }

package/lib/database/admin.mjs

@@ -0,0 +1,227 @@
+import { execa } from "execa";
+import {
+  LOCAL_ADMIN_DB,
+  LOCAL_ADMIN_QUERY_RETRY_INTERVAL_MS,
+  LOCAL_ADMIN_QUERY_RETRY_MS,
+  LOCAL_DROP_DATABASE_POLL_INTERVAL_MS,
+  LOCAL_DROP_DATABASE_TIMEOUT_MS,
+  LOCAL_POLL_INTERVAL_MS,
+  LOCAL_READY_TIMEOUT_MS,
+} from "./constants.mjs";
+import {
+  buildDatabaseUrl,
+  escapeIdentifier,
+  escapeSqlLiteral,
+} from "./naming.mjs";
+
+export { buildDatabaseUrl };
+
+export async function waitForResourcePostgresReady(infra) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < LOCAL_READY_TIMEOUT_MS) {
+    try {
+      await runAdminQuery(infra, ["-tAc", "SELECT 1"]);
+      return;
+    } catch (error) {
+      if (!isTransientAdminQueryConnectionError(error)) throw error;
+      await sleep(LOCAL_POLL_INTERVAL_MS);
+    }
+  }
+
+  throw new Error(`Timed out waiting for Postgres resource "${infra.resourceName}" at ${infra.host}:${infra.port}`);
+}
+
+export async function databaseExists(infra, dbName) {
+  const result = await runAdminQuery(infra, [
+    "-tAc",
+    `SELECT 1 FROM pg_database WHERE datname = '${escapeSqlLiteral(dbName)}'`,
+  ]);
+  return result.trim() === "1";
+}
+
+export async function createEmptyDatabase(infra, dbName) {
+  await runAdminQuery(infra, ["-c", `CREATE DATABASE "${escapeIdentifier(dbName)}"`]);
+}
+
+export async function cloneDatabaseFromTemplate(infra, dbName, templateDbName) {
+  await runAdminQuery(infra, [
+    "-c",
+    `CREATE DATABASE "${escapeIdentifier(dbName)}" TEMPLATE "${escapeIdentifier(templateDbName)}"`,
+  ]);
+}
+
+export async function dropDatabaseIfExists(infra, dbName) {
+  await dropDatabaseWithForceOrDrain(infra, dbName);
+}
+
+export async function dropDatabaseWithForceOrDrain(infra, dbName, hooks = {}) {
+  const runAdminQueryFn = hooks.runAdminQuery || runAdminQuery;
+  const databaseExistsFn = hooks.databaseExists || databaseExists;
+  const sleepFn = hooks.sleep || sleep;
+
+  try {
+    await runAdminQueryFn(infra, [
+      "-c",
+      `DROP DATABASE IF EXISTS "${escapeIdentifier(dbName)}" WITH (FORCE)`,
+    ]);
+    return;
+  } catch (error) {
+    if (!isUnsupportedForceDropError(error)) {
+      throw error;
+    }
+  }
+
+  if (!(await databaseExistsFn(infra, dbName))) {
+    return;
+  }
+
+  let restoreConnections = false;
+  try {
+    await runAdminQueryFn(infra, [
+      "-c",
+      `ALTER DATABASE "${escapeIdentifier(dbName)}" WITH ALLOW_CONNECTIONS false`,
+    ]);
+    restoreConnections = true;
+    await waitForDatabaseConnectionsToDrain(infra, dbName, {
+      runAdminQuery: runAdminQueryFn,
+      sleep: sleepFn,
+    });
+    await runAdminQueryFn(infra, [
+      "-c",
+      `DROP DATABASE IF EXISTS "${escapeIdentifier(dbName)}"`,
+    ]);
+    restoreConnections = false;
+  } finally {
+    if (restoreConnections && (await databaseExistsFn(infra, dbName))) {
+      await runAdminQueryFn(infra, [
+        "-c",
+        `ALTER DATABASE "${escapeIdentifier(dbName)}" WITH ALLOW_CONNECTIONS true`,
+      ]).catch(() => {
+        // Best-effort restoration for failed fallback drops.
+      });
+    }
+  }
+}
+
+export async function waitForDatabaseConnectionsToDrain(infra, dbName, hooks = {}) {
+  const runAdminQueryFn = hooks.runAdminQuery || runAdminQuery;
+  const sleepFn = hooks.sleep || sleep;
+  const now = hooks.now || Date.now;
+  const timeoutMs = hooks.timeoutMs ?? LOCAL_DROP_DATABASE_TIMEOUT_MS;
+  const deadline = now() + timeoutMs;
+
+  while (true) {
+    await terminateDatabaseConnections(infra, dbName, runAdminQueryFn);
+    const remainingConnections = await countDatabaseConnections(infra, dbName, runAdminQueryFn);
+    if (remainingConnections === 0) {
+      return;
+    }
+    if (now() >= deadline) {
+      throw new Error(
+        `Timed out waiting for database "${dbName}" connections to close (${remainingConnections} remaining)`
+      );
+    }
+    await sleepFn(LOCAL_DROP_DATABASE_POLL_INTERVAL_MS);
+  }
+}
+
+export async function runAdminQuery(infra, args) {
+  const command = infra.containerName ? "docker" : "psql";
+  const commandArgs = infra.containerName
+    ? [
+        "exec",
+        "-e",
+        `PGPASSWORD=${infra.password}`,
+        infra.containerName,
+        "psql",
+        "-v",
+        "ON_ERROR_STOP=1",
+        "-U",
+        infra.user,
+        "-d",
+        infra.adminDatabase || LOCAL_ADMIN_DB,
+        ...args,
+      ]
+    : [
+        "-v",
+        "ON_ERROR_STOP=1",
+        "-h",
+        infra.host,
+        "-p",
+        String(infra.port),
+        "-U",
+        infra.user,
+        "-d",
+        infra.adminDatabase || LOCAL_ADMIN_DB,
+        ...args,
+      ];
+  const commandOptions = infra.containerName
+    ? {}
+    : {
+        env: {
+          ...process.env,
+          PGPASSWORD: infra.password,
+          ...(infra.sslMode ? { PGSSLMODE: infra.sslMode } : {}),
+        },
+      };
+  const startedAt = Date.now();
+  while (true) {
+    try {
+      const { stdout } = await execa(command, commandArgs, commandOptions);
+      return stdout;
+    } catch (error) {
+      if (
+        Date.now() - startedAt >= LOCAL_ADMIN_QUERY_RETRY_MS ||
+        !isTransientAdminQueryConnectionError(error)
+      ) {
+        throw error;
+      }
+      await sleep(LOCAL_ADMIN_QUERY_RETRY_INTERVAL_MS);
+    }
+  }
+}
+
+export function isTransientAdminQueryConnectionError(error) {
+  const text = `${error?.shortMessage || ""}\n${error?.stderr || ""}\n${error?.stdout || ""}\n${error?.message || ""}`;
+  return (
+    text.includes('connection to server on socket "/var/run/postgresql/.s.PGSQL.5432" failed') ||
+    text.includes("No such file or directory") ||
+    text.includes("the database system is starting up") ||
+    text.includes("could not connect to server")
+  );
+}
+
+async function terminateDatabaseConnections(infra, dbName, runAdminQueryFn) {
+  await runAdminQueryFn(infra, [
+    "-c",
+    [
+      "SELECT pg_terminate_backend(pid)",
+      "FROM pg_stat_activity",
+      `WHERE datname = '${escapeSqlLiteral(dbName)}' AND pid <> pg_backend_pid();`,
+    ].join(" "),
+  ]);
+}
+
+async function countDatabaseConnections(infra, dbName, runAdminQueryFn) {
+  const result = await runAdminQueryFn(infra, [
+    "-tAc",
+    [
+      "SELECT COUNT(*)",
+      "FROM pg_stat_activity",
+      `WHERE datname = '${escapeSqlLiteral(dbName)}' AND pid <> pg_backend_pid();`,
+    ].join(" "),
+  ]);
+  return Number.parseInt(result.trim(), 10) || 0;
+}
+
+function isUnsupportedForceDropError(error) {
+  const text = `${error?.shortMessage || ""}\n${error?.stderr || ""}\n${error?.stdout || ""}\n${error?.message || ""}`;
+  return (
+    text.includes('syntax error at or near "WITH"') ||
+    text.includes('option "force"')
+  );
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/lib/database/cleanup.mjs

@@ -0,0 +1,201 @@
+import fs from "fs";
+import path from "path";
+import {
+  TESTKIT_PRODUCT_DIR_LABEL,
+  TESTKIT_PRODUCT_ID_LABEL,
+  TESTKIT_RESOURCE_KIND_LABEL,
+  TESTKIT_SCOPE_LABEL,
+  buildProductIdentity,
+  dockerContainerSummary,
+  listLegacyTestkitPostgresContainers,
+  listManagedDockerContainers,
+  removeDockerContainer,
+  stopDockerContainer,
+} from "../ownership/docker.mjs";
+import { buildContainerName } from "./naming.mjs";
+import {
+  hasRemainingLocalArtifacts,
+  readStateValue,
+  visitDirs,
+} from "./state.mjs";
+
+export async function cleanupLocalPostgresDockerResources(options = {}) {
+  const product = buildProductIdentity(options.productDir || process.cwd());
+  const managed = await listManagedDockerContainers();
+  const targets = [];
+  const kept = [];
+  const stopped = [];
+
+  for (const container of managed) {
+    if (!isManagedLocalPostgresContainer(container)) continue;
+    const classification = classifyManagedLocalPostgresContainer(container, product, options);
+    if (classification.action === "remove") {
+      targets.push({ ...container, action: "remove", reason: classification.reason, legacy: false });
+    } else if (classification.action === "stop") {
+      targets.push({ ...container, action: "stop", reason: classification.reason, legacy: false });
+    } else if (classification.reason) {
+      kept.push({ ...container, reason: classification.reason, legacy: false });
+    }
+  }
+
+  if (options.includeLegacy) {
+    const currentLegacyName = buildContainerName(product.dir);
+    for (const container of await listLegacyTestkitPostgresContainers()) {
+      if (!options.global && container.name !== currentLegacyName) continue;
+      targets.push({ ...container, action: "remove", reason: "legacy-unlabelled", legacy: true });
+    }
+  }
+
+  if (!options.dryRun) {
+    for (const container of targets) {
+      if (container.action === "stop") {
+        await stopDockerContainer(container.name || container.id);
+        stopped.push(container);
+      } else {
+        await removeDockerContainer(container.name || container.id);
+      }
+    }
+  }
+
+  return {
+    removed: options.dryRun ? [] : targets.filter((container) => container.action !== "stop"),
+    stopped: options.dryRun ? [] : stopped,
+    targets,
+    kept,
+  };
+}
+
+export function formatDatabaseResourceCleanupLine(entry, dryRun = false) {
+  const isStop = entry.action === "stop";
+  const action = dryRun
+    ? isStop ? "Would stop" : "Would remove"
+    : isStop ? "Stopped" : "Removed";
+  const legacy = entry.legacy ? " legacy" : "";
+  return `${action}${legacy} database resource ${dockerContainerSummary(entry)} reason=${entry.reason}`;
+}
+
+function classifyManagedLocalPostgresContainer(container, product, options) {
+  const labels = container.labels || {};
+  const containerProductId = labels[TESTKIT_PRODUCT_ID_LABEL] || "";
+  const containerProductDir = labels[TESTKIT_PRODUCT_DIR_LABEL] || "";
+  const currentProduct = containerProductId === product.id;
+
+  if (options.force && currentProduct) {
+    return { action: "remove", reason: "destroy-current-product" };
+  }
+
+  if (!options.global && !currentProduct) {
+    return { action: "keep", reason: "different-product" };
+  }
+
+  if (currentProduct) {
+    if (!hasRemainingLocalArtifacts(product.dir, readStateValue)) {
+      return { action: "remove", reason: "current-product-no-local-artifacts" };
+    }
+    if (!localArtifactsReferenceContainer(product.dir, container.name)) {
+      return { action: "remove", reason: "current-product-unreferenced" };
+    }
+    if (shouldStopIdleLocalPostgresContainer(container, product.dir, options)) {
+      return { action: "stop", reason: "current-product-idle" };
+    }
+    return { action: "keep", reason: "current-product-referenced" };
+  }
+
+  if (options.global && containerProductDir && !fs.existsSync(containerProductDir)) {
+    return { action: "remove", reason: "product-dir-missing" };
+  }
+
+  if (options.global && containerProductDir && !hasRemainingLocalArtifacts(containerProductDir, readStateValue)) {
+    return { action: "remove", reason: "product-has-no-local-artifacts" };
+  }
+
+  return { action: "keep", reason: options.global ? "other-product-retained" : "different-product" };
+}
+
+function shouldStopIdleLocalPostgresContainer(container, productDir, options) {
+  return Boolean(
+    options.stopIdle &&
+    container.running &&
+    productDir &&
+    fs.existsSync(productDir) &&
+    !hasActiveProductRuntime(productDir)
+  );
+}
+
+function isManagedLocalPostgresContainer(container) {
+  const labels = container.labels || {};
+  return (
+    labels[TESTKIT_RESOURCE_KIND_LABEL] === "postgres-container" &&
+    labels[TESTKIT_SCOPE_LABEL] === "local-postgres"
+  );
+}
+
+function localArtifactsReferenceContainer(productDir, containerName) {
+  if (!containerName) return false;
+  const root = path.join(productDir, ".testkit");
+  let referenced = false;
+  visitDirs(root, (dir) => {
+    if (referenced) return;
+    for (const fileName of ["container_name", "local_container_name"]) {
+      if (readStateValue(path.join(dir, fileName)) === containerName) {
+        referenced = true;
+        return;
+      }
+    }
+  });
+  return referenced;
+}
+
+function hasActiveProductRuntime(productDir) {
+  return hasActiveRunManifest(productDir) || hasActiveLocalEnvironmentManifest(productDir);
+}
+
+function hasActiveRunManifest(productDir) {
+  const runsDir = path.join(productDir, ".testkit", "_runs");
+  for (const manifest of readManifestFiles(runsDir)) {
+    if (isPidRunning(manifest.pid)) return true;
+  }
+  return false;
+}
+
+function hasActiveLocalEnvironmentManifest(productDir) {
+  const environmentsDir = path.join(productDir, ".testkit", "environments");
+  if (!fs.existsSync(environmentsDir)) return false;
+  for (const entry of fs.readdirSync(environmentsDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const manifest = readJsonFile(path.join(environmentsDir, entry.name, "manifest.json"));
+    if (!manifest) continue;
+    if (manifest.driver === "kiln" && manifest.status === "running") return true;
+    if ((manifest.services || []).some((service) => isPidRunning(service.processGroupId || service.pid))) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function readManifestFiles(dir) {
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir, { withFileTypes: true })
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+    .map((entry) => readJsonFile(path.join(dir, entry.name)))
+    .filter(Boolean);
+}
+
+function readJsonFile(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function isPidRunning(pid) {
+  const numericPid = Number(pid);
+  if (!Number.isInteger(numericPid) || numericPid <= 0) return false;
+  try {
+    process.kill(numericPid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}

package/lib/database/constants.mjs

@@ -0,0 +1,10 @@
+export const LOCAL_IMAGE = "pgvector/pgvector:pg16";
+export const LOCAL_USER = "testkit";
+export const LOCAL_PASSWORD = "testkit";
+export const LOCAL_ADMIN_DB = "postgres";
+export const LOCAL_READY_TIMEOUT_MS = 60_000;
+export const LOCAL_POLL_INTERVAL_MS = 1_000;
+export const LOCAL_ADMIN_QUERY_RETRY_MS = 15_000;
+export const LOCAL_ADMIN_QUERY_RETRY_INTERVAL_MS = 250;
+export const LOCAL_DROP_DATABASE_TIMEOUT_MS = 15_000;
+export const LOCAL_DROP_DATABASE_POLL_INTERVAL_MS = 250;