@elench/testkit 0.1.66 → 0.1.67

package/README.md

@@ -87,16 +87,13 @@ Create `testkit.setup.ts` at repo root:
 
 ```ts
 import {
-  commandStep,
   defineTestkitSetup,
-  localDatabase,
-  nextService,
+  defineTestkitFile,
+  nextApp,
   nodeToolchain,
-  schemaSql,
+  nodeApp,
   seedCommand,
-  seededDatabaseTemplate,
-  service,
-  tsxService,
+  templateDatabase,
   verifyModule,
 } from "@elench/testkit/setup";
 
@@ -121,77 +118,53 @@ export default defineTestkitSetup({
     }),
   },
   services: {
-    api: service({
-      ...tsxService({
-        cwd: ".",
-        entry: "src/index.ts",
-        port: 3004,
-        readyPath: "/health",
-      }),
+    api: nodeApp({
+      cwd: ".",
+      entry: "src/index.ts",
+      port: 3004,
       envFiles: [".env.testkit"],
-      database: localDatabase({
-        template: seededDatabaseTemplate({
-          inputs: ["db/schema.sql", "scripts/seed.ts"],
-          schema: schemaSql("db/schema.sql"),
-          seed: seedCommand("npm run db:seed"),
-          verify: verifyModule("src/testkit/verify-seed.ts#verifySeed"),
-        }),
+      database: templateDatabase({
+        inputs: ["db/schema.sql", "scripts/seed.ts"],
+        schema: "db/schema.sql",
+        seed: seedCommand("npm run db:seed"),
+        verify: verifyModule("src/testkit/verify-seed.ts#verifySeed"),
       }),
       runtime: {
         instances: 1,
         maxConcurrentTasks: 4,
       },
-      requirements: {
-        files: [
-          {
-            path: "src/api/routes/__testkit__/crawls-worker-concurrency.int.testkit.ts",
-            locks: ["global-worker-loop"],
-          },
-        ],
-      },
     }),
-    frontend: service({
-      ...nextService({
-        cwd: "frontend",
-        port: 3000,
-        start: "./node_modules/.bin/next start --port {port}",
-        env: {
-          NEXT_DIST_DIR: "{prepareDir}/dist",
-          NEXT_PUBLIC_API_URL: "{baseUrl:api}",
-        },
-      }),
+    frontend: nextApp({
+      cwd: "frontend",
+      mode: "start",
+      port: 3000,
       dependsOn: ["api"],
       envFiles: ["frontend/.env.testkit"],
+      env: {
+        NEXT_DIST_DIR: "{prepareDir}/dist",
+        NEXT_PUBLIC_API_URL: "{baseUrl:api}",
+      },
       runtime: {
         instances: 1,
         maxConcurrentTasks: 2,
         toolchain: "frontendNode",
-        prepare: {
-          inputs: ["frontend/src", "frontend/public", "frontend/package.json"],
-          steps: [commandStep("npm run build", { cwd: "frontend" })],
-        },
-      },
-    }),
-    billing: service({
-      skip: {
-        files: [
-          {
-            path: "__testkit__/invoices/invoices.int.testkit.ts",
-            reason: "Billing is still stubbed locally",
-          },
-        ],
-        suites: [
-          {
-            selector: "pw:lifecycle",
-            reason: "End-to-end billing lifecycle is not implemented yet",
-          },
-        ],
       },
     }),
   },
 });
 ```
 
+File-local execution metadata now lives next to the test when possible:
+
+```ts
+import { defineTestkitFile } from "@elench/testkit/setup";
+
+export const testkit = defineTestkitFile({
+  skip: "Billing is still stubbed locally",
+  locks: ["global-worker-loop"],
+});
+```
+
 `testkit.setup.ts` is optional for simple repos, but it is the primary escape hatch
 for:
 
@@ -232,7 +205,7 @@ For most repos, prefer the intent-focused helpers:
 - `seedModule(...)`
 - `verifyCommand(...)`
 - `verifyModule(...)`
-- `seededDatabaseTemplate(...)`
+- `templateDatabase(...)`
 
 Use raw `commandStep(...)`, `sqlFileStep(...)`, and `moduleStep(...)` arrays when
 you need lower-level control over the exact stage layout.
@@ -264,7 +237,9 @@ toolchains: {
   }),
 },
 services: {
-  frontend: service({
+  frontend: nextApp({
+    cwd: "frontend",
+    port: 3000,
     runtime: {
       toolchain: "frontendNode",
     },
@@ -473,7 +448,7 @@ Git metadata.
 ## Local Databases
 
 `@elench/testkit` provisions Docker-managed local Postgres automatically for
-services that define `database: localDatabase(...)`.
+services that define `database: postgresDatabase(...)` or `database: templateDatabase(...)`.
 
 - template databases are cached
 - runtime databases are cloned from templates when binding is `per-runtime`

package/lib/app/doctor.mjs

@@ -0,0 +1,139 @@
+import fs from "fs";
+import path from "path";
+import ts from "typescript";
+import { discoverTests } from "../discovery/index.mjs";
+import { loadConfigContext } from "../config/index.mjs";
+import { runTestkitTypecheck } from "./typecheck.mjs";
+
+export async function runDoctor(options = {}) {
+  const checks = [];
+  const productDir = options.dir ? path.resolve(process.cwd(), options.dir) : process.cwd();
+
+  await loadConfigContext({ dir: productDir });
+  checks.push({
+    code: "config-load",
+    level: "pass",
+    message: "Loaded testkit setup and service config",
+  });
+
+  const discovery = await discoverTests({ dir: productDir, diagnostics: "report" });
+  const discoveryErrors = discovery.diagnostics.filter((entry) => entry.severity === "error");
+  checks.push({
+    code: "discovery",
+    level: discoveryErrors.length === 0 ? "pass" : "fail",
+    message:
+      discoveryErrors.length === 0
+        ? `Discovered ${discovery.files.length} testkit files without diagnostics`
+        : `Discovery reported ${discoveryErrors.length} error diagnostics`,
+    details: discoveryErrors,
+  });
+
+  const playwrightViolations = findPlaywrightRuntimeImportViolations(productDir);
+  checks.push({
+    code: "playwright-runtime-imports",
+    level: playwrightViolations.length === 0 ? "pass" : "fail",
+    message:
+      playwrightViolations.length === 0
+        ? "No runtime @playwright/test imports found in testkit Playwright suites"
+        : `Found ${playwrightViolations.length} Playwright runtime import violation(s)`,
+    details: playwrightViolations,
+  });
+
+  const hasBrowserOrNextWork = discovery.files.some((entry) => entry.selectionType === "pw");
+  if (hasBrowserOrNextWork) {
+    const nodeCount = discovery.coverageGraph?.nodes?.length || 0;
+    checks.push({
+      code: "coverage-graph",
+      level: nodeCount > 0 ? "pass" : "warning",
+      message:
+        nodeCount > 0
+          ? `Coverage graph contains ${nodeCount} node(s)`
+          : "Coverage graph is empty even though browser-facing suites were discovered",
+    });
+  }
+
+  if (options.typecheck !== false) {
+    try {
+      const typecheck = await runTestkitTypecheck({ dir: productDir });
+      checks.push({
+        code: "typecheck",
+        level: "pass",
+        message: `Typechecked ${typecheck.results.length} generated program(s)`,
+      });
+    } catch (error) {
+      checks.push({
+        code: "typecheck",
+        level: "fail",
+        message: error?.result?.stderr || error?.result?.stdout || error.message,
+      });
+    }
+  }
+
+  const failed = checks.some((check) => check.level === "fail");
+  return {
+    ok: !failed,
+    productDir,
+    checks,
+  };
+}
+
+function findPlaywrightRuntimeImportViolations(productDir) {
+  const violations = [];
+  for (const absolutePath of collectFiles(productDir)) {
+    const sourceText = fs.readFileSync(absolutePath, "utf8");
+    const sourceFile = ts.createSourceFile(absolutePath, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
+
+    for (const statement of sourceFile.statements) {
+      if (!ts.isImportDeclaration(statement)) continue;
+      if (!ts.isStringLiteral(statement.moduleSpecifier)) continue;
+      if (statement.moduleSpecifier.text !== "@playwright/test") continue;
+      const clause = statement.importClause;
+      if (clause?.isTypeOnly) continue;
+      if (!clause) {
+        violations.push(relativeViolation(productDir, absolutePath, sourceFile, statement));
+        continue;
+      }
+      if (clause.name) {
+        violations.push(relativeViolation(productDir, absolutePath, sourceFile, statement));
+        continue;
+      }
+      if (!clause.namedBindings) {
+        violations.push(relativeViolation(productDir, absolutePath, sourceFile, statement));
+        continue;
+      }
+      if (ts.isNamespaceImport(clause.namedBindings)) {
+        violations.push(relativeViolation(productDir, absolutePath, sourceFile, statement));
+        continue;
+      }
+      if (clause.namedBindings.elements.some((entry) => !entry.isTypeOnly)) {
+        violations.push(relativeViolation(productDir, absolutePath, sourceFile, statement));
+      }
+    }
+  }
+  return violations;
+}
+
+function collectFiles(rootDir, out = []) {
+  if (!fs.existsSync(rootDir)) return out;
+  for (const entry of fs.readdirSync(rootDir, { withFileTypes: true })) {
+    const absolutePath = path.join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name === "node_modules" || entry.name === ".git" || entry.name === ".testkit") continue;
+      collectFiles(absolutePath, out);
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith(".pw.testkit.ts")) {
+      out.push(absolutePath);
+    }
+  }
+  return out.sort((left, right) => left.localeCompare(right));
+}
+
+function relativeViolation(productDir, absolutePath, sourceFile, statement) {
+  const position = sourceFile.getLineAndCharacterOfPosition(statement.getStart(sourceFile));
+  return {
+    file: path.relative(productDir, absolutePath).split(path.sep).join("/"),
+    line: position.line + 1,
+    snippet: statement.getText(sourceFile),
+  };
+}

package/lib/app/typecheck.mjs

@@ -0,0 +1,203 @@
+import fs from "fs";
+import path from "path";
+import { spawn } from "child_process";
+import { fileURLToPath } from "url";
+import { loadConfigContext } from "../config/index.mjs";
+import { detectNextApp } from "../config/runtime.mjs";
+
+const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+export async function runTestkitTypecheck(options = {}) {
+  const context = await loadConfigContext({ dir: options.dir });
+  const productDir = context.productDir;
+  const tempDir = path.join(productDir, ".testkit", "_typecheck");
+  fs.mkdirSync(tempDir, { recursive: true });
+
+  const checks = [];
+  const rootTsconfig = writeRootTypecheckConfig({
+    productDir,
+    setupFile: context.setupFile,
+    outputDir: tempDir,
+  });
+  checks.push({
+    label: "repo",
+    cwd: productDir,
+    tsconfigPath: rootTsconfig,
+  });
+
+  for (const config of context.configs) {
+    const cwd = config.testkit.local?.cwd;
+    if (!cwd) continue;
+    const absoluteCwd = path.resolve(productDir, cwd);
+    if (!detectNextApp(absoluteCwd)) continue;
+
+    const serviceTsconfig = writeNextServiceTypecheckConfig({
+      productDir,
+      cwd,
+      outputDir: tempDir,
+      serviceName: config.name,
+    });
+    checks.push({
+      label: config.name,
+      cwd: productDir,
+      tsconfigPath: serviceTsconfig,
+    });
+  }
+
+  const results = [];
+  for (const check of checks) {
+    const result = await runTscCheck(check);
+    results.push(result);
+    if (!result.ok) {
+      const error = new Error(`testkit typecheck failed for ${check.label}`);
+      error.result = result;
+      error.results = results;
+      throw error;
+    }
+  }
+
+  return {
+    ok: true,
+    productDir,
+    results,
+  };
+}
+
+function writeRootTypecheckConfig({ productDir, setupFile, outputDir }) {
+  const tsconfigPath = path.join(outputDir, "repo.tsconfig.json");
+  const extendsPath = findExistingTsconfig(productDir) || null;
+  const config = {
+    ...(extendsPath ? { extends: relativeJsonPath(tsconfigPath, extendsPath) } : {}),
+    compilerOptions: {
+      baseUrl: ".",
+      module: "ESNext",
+      moduleResolution: "Bundler",
+      noEmit: true,
+      paths: packageTypePaths(tsconfigPath),
+      rootDir: relativeJsonPath(tsconfigPath, productDir),
+      target: "ES2022",
+    },
+    include: [
+      setupFile ? relativeJsonPath(tsconfigPath, setupFile) : "testkit.setup.ts",
+      "**/*.int.testkit.ts",
+      "**/*.e2e.testkit.ts",
+      "**/*.scenario.testkit.ts",
+      "**/*.dal.testkit.ts",
+      "**/*.load.testkit.ts",
+      "**/__testkit__/helpers/**/*.ts",
+    ],
+    exclude: ["node_modules", "dist", ".testkit", "**/*.pw.testkit.ts"],
+  };
+  fs.writeFileSync(tsconfigPath, `${JSON.stringify(config, null, 2)}\n`);
+  return tsconfigPath;
+}
+
+function writeNextServiceTypecheckConfig({ productDir, cwd, outputDir, serviceName }) {
+  const serviceDir = path.resolve(productDir, cwd);
+  const tsconfigPath = path.join(outputDir, `${serviceName}.tsconfig.json`);
+  const extendsPath =
+    findExistingTsconfig(serviceDir) ||
+    findExistingTsconfig(productDir) ||
+    null;
+  const config = {
+    ...(extendsPath ? { extends: relativeJsonPath(tsconfigPath, extendsPath) } : {}),
+    compilerOptions: {
+      baseUrl: ".",
+      module: "ESNext",
+      moduleResolution: "Bundler",
+      noEmit: true,
+      paths: packageTypePaths(tsconfigPath),
+      target: "ES2022",
+    },
+    include: [
+      serviceExists(serviceDir, "next-env.d.ts") ? relativeJsonPath(tsconfigPath, path.join(serviceDir, "next-env.d.ts")) : undefined,
+      relativeGlob(tsconfigPath, serviceDir, "src/**/*.pw.testkit.ts"),
+      relativeGlob(tsconfigPath, serviceDir, "tests/**/*.ts"),
+      relativeGlob(tsconfigPath, serviceDir, ".next/types/**/*.ts"),
+      relativeGlob(tsconfigPath, serviceDir, ".next/dev/types/**/*.ts"),
+      relativeGlob(tsconfigPath, serviceDir, ".next-testkit/**/dist/types/**/*.ts"),
+      relativeGlob(tsconfigPath, serviceDir, ".next-testkit/**/dist/dev/types/**/*.ts"),
+    ].filter(Boolean),
+    exclude: ["node_modules", ".next/cache", ".next/dev"],
+  };
+  fs.writeFileSync(tsconfigPath, `${JSON.stringify(config, null, 2)}\n`);
+  return tsconfigPath;
+}
+
+function runTscCheck({ label, cwd, tsconfigPath }) {
+  const binary = resolveTscBinary(cwd);
+  return new Promise((resolve, reject) => {
+    const child = spawn(binary, ["--noEmit", "-p", tsconfigPath], {
+      cwd,
+      env: process.env,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => {
+      stdout += String(chunk);
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += String(chunk);
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      resolve({
+        label,
+        tsconfigPath,
+        ok: code === 0,
+        exitCode: code,
+        stdout,
+        stderr,
+      });
+    });
+  });
+}
+
+function resolveTscBinary(productDir) {
+  let current = productDir;
+  const binaryName = process.platform === "win32" ? "tsc.cmd" : "tsc";
+
+  while (true) {
+    const candidate = path.join(current, "node_modules", ".bin", binaryName);
+    if (fs.existsSync(candidate)) {
+      return candidate;
+    }
+    const parent = path.dirname(current);
+    if (parent === current) break;
+    current = parent;
+  }
+  const packageCandidate = path.join(PACKAGE_ROOT, "node_modules", ".bin", binaryName);
+  if (fs.existsSync(packageCandidate)) {
+    return packageCandidate;
+  }
+  throw new Error(`TypeScript compiler not found from ${productDir} upward`);
+}
+
+function findExistingTsconfig(rootDir) {
+  for (const candidate of ["tsconfig.json", "tsconfig.build.json"]) {
+    const absolute = path.join(rootDir, candidate);
+    if (fs.existsSync(absolute)) return absolute;
+  }
+  return null;
+}
+
+function relativeJsonPath(fromFile, targetPath) {
+  return path.relative(path.dirname(fromFile), targetPath).split(path.sep).join("/");
+}
+
+function relativeGlob(fromFile, rootDir, pattern) {
+  return path.join(relativeJsonPath(fromFile, rootDir), pattern).split(path.sep).join("/");
+}
+
+function serviceExists(rootDir, relativePath) {
+  return fs.existsSync(path.join(rootDir, relativePath));
+}
+
+function packageTypePaths(tsconfigPath) {
+  return {
+    "@elench/testkit": [relativeJsonPath(tsconfigPath, path.join(PACKAGE_ROOT, "lib", "index.d.ts"))],
+    "@elench/testkit/setup": [relativeJsonPath(tsconfigPath, path.join(PACKAGE_ROOT, "lib", "setup", "index.d.ts"))],
+    "@elench/testkit/runtime": [relativeJsonPath(tsconfigPath, path.join(PACKAGE_ROOT, "lib", "runtime", "index.d.ts"))],
+  };
+}

package/lib/cli/commands/doctor.mjs

@@ -0,0 +1,39 @@
+import { Command, Flags } from "@oclif/core";
+import { runDoctor } from "../../app/doctor.mjs";
+
+export default class DoctorCommand extends Command {
+  static summary = "Run built-in setup, discovery, and hygiene checks";
+
+  static enableJsonFlag = true;
+
+  static flags = {
+    dir: Flags.string({
+      description: "Product directory",
+    }),
+    typecheck: Flags.boolean({
+      description: "Include generated TypeScript checks",
+      default: true,
+      allowNo: true,
+    }),
+  };
+
+  async run() {
+    const { flags } = await this.parse(DoctorCommand);
+    const result = await runDoctor({ dir: flags.dir, typecheck: flags.typecheck });
+
+    if (!this.jsonEnabled()) {
+      this.log(`testkit doctor ${result.ok ? "passed" : "failed"} for ${result.productDir}`);
+      for (const check of result.checks) {
+        this.log(`${check.level.toUpperCase()} ${check.code} ${check.message}`);
+      }
+    }
+
+    if (!result.ok) {
+      const error = new Error("testkit doctor failed");
+      error.result = result;
+      throw error;
+    }
+
+    return result;
+  }
+}

package/lib/cli/commands/typecheck.mjs

@@ -0,0 +1,28 @@
+import { Command, Flags } from "@oclif/core";
+import { runTestkitTypecheck } from "../../app/typecheck.mjs";
+
+export default class TypecheckCommand extends Command {
+  static summary = "Typecheck testkit setup, helpers, and suites";
+
+  static enableJsonFlag = true;
+
+  static flags = {
+    dir: Flags.string({
+      description: "Product directory",
+    }),
+  };
+
+  async run() {
+    const { flags } = await this.parse(TypecheckCommand);
+    const result = await runTestkitTypecheck({ dir: flags.dir });
+
+    if (!this.jsonEnabled()) {
+      this.log(`Typechecked ${result.results.length} testkit program(s) in ${result.productDir}`);
+      for (const entry of result.results) {
+        this.log(`PASS ${entry.label} ${entry.tsconfigPath}`);
+      }
+    }
+
+    return result;
+  }
+}

package/lib/cli/entrypoint.mjs

@@ -9,6 +9,8 @@ export function normalizeCliArgs(argv) {
     "artifacts",
     "watch",
     "discover",
+    "typecheck",
+    "doctor",
     "browser",
     "known-failures",
     "db",

package/lib/config/index.mjs

@@ -3,6 +3,7 @@ import path from "path";
 import { discoverProject } from "./discovery.mjs";
 import { loadTestkitSetup } from "./setup-loader.mjs";
 import { normalizeToolchainRegistry } from "../toolchains/index.mjs";
+import { loadTestFileMetadataMap } from "../discovery/file-metadata.mjs";
 import { mergeDiscoveryConfigs } from "../discovery/path-policy.mjs";
 import { normalizeDatabaseConfig } from "./database.mjs";
 import { normalizeRepoDiscoveryConfig, normalizeServiceDiscoveryConfig } from "./discovery-config.mjs";
@@ -37,6 +38,7 @@ export async function loadConfigContext(opts = {}) {
     ...(opts.discoveryOptions || {}),
     discovery: discoveryConfig,
   });
+  const fileMetadataByPath = loadTestFileMetadataMap(productDir, discovery.suitesByService);
   const serviceNames = new Set([
     ...Object.keys(explicitServices),
     ...Object.keys(discovery.suitesByService),
@@ -57,6 +59,7 @@ export async function loadConfigContext(opts = {}) {
         explicitService: explicitServices[name] || {},
         discoveredService: discovery.services[name] || null,
         suites: discovery.suitesByService[name] || {},
+        fileMetadataByPath,
       })
     );
 
@@ -104,6 +107,7 @@ function normalizeServiceConfig({
   explicitService,
   discoveredService,
   suites,
+  fileMetadataByPath,
 }) {
   const local = normalizeLocalConfig(name, explicitService, discoveredService, productDir);
   const envFiles = inferEnvFiles(productDir, explicitService, local);
@@ -125,6 +129,12 @@ function normalizeServiceConfig({
   const runtime = normalizeRuntimeConfig(explicitService.runtime, name, toolchains);
   const skip = normalizeSkipConfig(explicitService.skip, { name, suites });
   const requirements = normalizeServiceRequirements(explicitService.requirements, { name, suites });
+  const serviceFilePaths = new Set(
+    Object.values(suites || {}).flatMap((suiteList) => suiteList.flatMap((suite) => suite.files || []))
+  );
+  const serviceFileMetadata = new Map(
+    [...fileMetadataByPath.entries()].filter(([filePath]) => serviceFilePaths.has(filePath))
+  );
 
   validateServiceConfig({
     name,
@@ -157,6 +167,7 @@ function normalizeServiceConfig({
       skip,
       runtime,
       browser,
+      fileMetadataByPath: serviceFileMetadata,
       local,
     },
   };

package/lib/config/runtime.mjs

@@ -14,6 +14,7 @@ import {
   normalizeOptionalString,
   parseModuleSpecifier,
 } from "../shared/configured-steps.mjs";
+import { buildConfigToPrepare, normalizeBuildConfig } from "../shared/build-config.mjs";
 import { normalizeKnownFailureIssueValidationConfig } from "../known-failures/github.mjs";
 import { normalizeRuntimeToolchain } from "../toolchains/index.mjs";
 import { resolveServiceCwd } from "./paths.mjs";
@@ -77,6 +78,7 @@ export function inferLocalRuntime(productDir, cwd) {
 export function normalizeRuntimeConfig(value, serviceName, toolchains) {
   if (!value) {
     return {
+      build: null,
       instances: 1,
       maxConcurrentTasks: Number.POSITIVE_INFINITY,
       prepare: {
@@ -87,13 +89,21 @@ export function normalizeRuntimeConfig(value, serviceName, toolchains) {
     };
   }
 
+  const build = normalizeBuildConfig(value.build, `Service "${serviceName}" runtime.build`);
+  const prepare = normalizeRuntimePrepareConfig(value.prepare, serviceName);
+  const buildPrepare = buildConfigToPrepare(build);
+
   return {
+    build,
     instances: normalizeRuntimeInstances(value.instances ?? 1, `Service "${serviceName}" runtime.instances`),
     maxConcurrentTasks: normalizeRuntimeMaxConcurrentTasks(
       value.maxConcurrentTasks,
       `Service "${serviceName}" runtime.maxConcurrentTasks`
     ),
-    prepare: normalizeRuntimePrepareConfig(value.prepare, serviceName),
+    prepare: {
+      inputs: [...new Set([...(buildPrepare.inputs || []), ...(prepare.inputs || [])])],
+      steps: [...(buildPrepare.steps || []), ...(prepare.steps || [])],
+    },
     toolchain: normalizeRuntimeToolchain(
       value.toolchain,
       `Service "${serviceName}" runtime.toolchain`,