@elench/testkit 0.1.16 → 0.1.18

package/lib/config/model.mjs

@@ -0,0 +1,422 @@
+export const VALID_FRAMEWORKS = new Set(["k6", "playwright"]);
+export const VALID_DB_PROVIDERS = new Set(["local"]);
+
+export function parseDotenvString(source) {
+  const env = {};
+  for (const line of String(source).split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eq = trimmed.indexOf("=");
+    if (eq === -1) continue;
+    const key = trimmed.slice(0, eq).trim();
+    let val = trimmed.slice(eq + 1).trim();
+    if (
+      (val.startsWith("'") && val.endsWith("'")) ||
+      (val.startsWith("\"") && val.endsWith("\""))
+    ) {
+      val = val.slice(1, -1);
+    }
+    env[key] = val;
+  }
+  return env;
+}
+
+export function validateRunnerManifest(raw, manifestName = "runner.manifest.json", manifestPath = manifestName) {
+  if (!isObject(raw.services)) {
+    throw new Error(`${manifestName} must have a "services" object (${manifestPath})`);
+  }
+
+  for (const [serviceName, service] of Object.entries(raw.services)) {
+    if (!isObject(service)) {
+      throw new Error(`Service "${serviceName}" in ${manifestName} must be an object`);
+    }
+    if (!isObject(service.suites)) {
+      throw new Error(`Service "${serviceName}" in ${manifestName} must define suites`);
+    }
+
+    for (const [suiteType, suites] of Object.entries(service.suites)) {
+      if (!Array.isArray(suites)) {
+        throw new Error(
+          `Service "${serviceName}" suite type "${suiteType}" must be an array`
+        );
+      }
+
+      const seenNames = new Set();
+      for (const suite of suites) {
+        if (!isObject(suite)) {
+          throw new Error(
+            `Service "${serviceName}" suite type "${suiteType}" contains a non-object suite`
+          );
+        }
+        if (typeof suite.name !== "string" || !suite.name.length) {
+          throw new Error(
+            `Service "${serviceName}" suite type "${suiteType}" has a suite with no name`
+          );
+        }
+        if (seenNames.has(suite.name)) {
+          throw new Error(
+            `Service "${serviceName}" suite type "${suiteType}" has duplicate suite name "${suite.name}"`
+          );
+        }
+        seenNames.add(suite.name);
+
+        if (!Array.isArray(suite.files) || suite.files.length === 0) {
+          throw new Error(
+            `Service "${serviceName}" suite "${suite.name}" must define one or more files`
+          );
+        }
+        for (const file of suite.files) {
+          if (typeof file !== "string" || !file.length) {
+            throw new Error(
+              `Service "${serviceName}" suite "${suite.name}" contains an invalid file entry`
+            );
+          }
+        }
+
+        const framework = suite.framework || "k6";
+        if (!VALID_FRAMEWORKS.has(framework)) {
+          throw new Error(
+            `Service "${serviceName}" suite "${suite.name}" uses unsupported framework "${framework}"`
+          );
+        }
+
+        if (suite.testkit !== undefined) {
+          if (!isObject(suite.testkit)) {
+            throw new Error(
+              `Service "${serviceName}" suite "${suite.name}" testkit config must be an object`
+            );
+          }
+          if (
+            suite.testkit.maxFileConcurrency !== undefined &&
+            (!Number.isInteger(suite.testkit.maxFileConcurrency) ||
+              suite.testkit.maxFileConcurrency <= 0)
+          ) {
+            throw new Error(
+              `Service "${serviceName}" suite "${suite.name}" testkit.maxFileConcurrency must be a positive integer`
+            );
+          }
+          if (
+            suite.testkit.weight !== undefined &&
+            (!Number.isInteger(suite.testkit.weight) || suite.testkit.weight <= 0)
+          ) {
+            throw new Error(
+              `Service "${serviceName}" suite "${suite.name}" testkit.weight must be a positive integer`
+            );
+          }
+        }
+      }
+    }
+  }
+}
+
+export function validateConfigCoverage(
+  runner,
+  config,
+  configName = "testkit.config.json",
+  manifestName = "runner.manifest.json"
+) {
+  for (const serviceName of Object.keys(config.services || {})) {
+    if (!runner.services[serviceName]) {
+      throw new Error(
+        `Service "${serviceName}" exists in ${configName} but not in ${manifestName}`
+      );
+    }
+
+    for (const depName of config.services[serviceName].dependsOn || []) {
+      if (!config.services[depName]) {
+        throw new Error(
+          `Service "${serviceName}" depends on "${depName}", but ${depName} is missing from ${configName}`
+        );
+      }
+      if (!runner.services[depName]) {
+        throw new Error(
+          `Service "${serviceName}" depends on "${depName}", but ${depName} is missing from ${manifestName}`
+        );
+      }
+    }
+
+    const databaseFrom = config.services[serviceName].databaseFrom;
+    if (databaseFrom) {
+      if (!config.services[databaseFrom]) {
+        throw new Error(
+          `Service "${serviceName}" databaseFrom "${databaseFrom}" is missing from ${configName}`
+        );
+      }
+      if (!runner.services[databaseFrom]) {
+        throw new Error(
+          `Service "${serviceName}" databaseFrom "${databaseFrom}" is missing from ${manifestName}`
+        );
+      }
+    }
+  }
+}
+
+export function validateServiceConfig(name, service, configPath) {
+  if (!isObject(service)) {
+    throw new Error(`Service "${name}" in ${configPath} must be an object`);
+  }
+
+  if (service.dependsOn !== undefined) {
+    if (!Array.isArray(service.dependsOn) || service.dependsOn.some((v) => typeof v !== "string")) {
+      throw new Error(`Service "${name}" dependsOn must be an array of service names`);
+    }
+  }
+
+  if (service.databaseFrom !== undefined && typeof service.databaseFrom !== "string") {
+    throw new Error(`Service "${name}" databaseFrom must be a string`);
+  }
+
+  if (service.database !== undefined && service.databaseFrom !== undefined) {
+    throw new Error(`Service "${name}" cannot define both database and databaseFrom`);
+  }
+
+  if (service.envFile !== undefined && typeof service.envFile !== "string") {
+    throw new Error(`Service "${name}" envFile must be a string`);
+  }
+
+  if (
+    service.envFiles !== undefined &&
+    (!Array.isArray(service.envFiles) || service.envFiles.some((v) => typeof v !== "string"))
+  ) {
+    throw new Error(`Service "${name}" envFiles must be an array of strings`);
+  }
+
+  if (service.database !== undefined) {
+    if (!isObject(service.database)) {
+      throw new Error(`Service "${name}" database must be an object`);
+    }
+
+    if (service.database.provider !== undefined) {
+      validateDatabaseProviderConfig(name, service.database, `Service "${name}" database`);
+    } else if (service.database.backends !== undefined) {
+      throw new Error(
+        `Service "${name}" database.backends is no longer supported. Define a single local database provider instead.`
+      );
+    } else {
+      throw new Error(`Service "${name}" database must define provider`);
+    }
+  }
+
+  if (service.migrate !== undefined) {
+    validateLifecycleConfig(name, service.migrate, "migrate");
+  }
+
+  if (service.seed !== undefined) {
+    validateLifecycleConfig(name, service.seed, "seed");
+  }
+
+  if (service.local !== undefined) {
+    if (!isObject(service.local)) {
+      throw new Error(`Service "${name}" local must be an object`);
+    }
+    requireString(service.local, "start", `Service "${name}" local.start`);
+    requireString(service.local, "baseUrl", `Service "${name}" local.baseUrl`);
+    requireString(service.local, "readyUrl", `Service "${name}" local.readyUrl`);
+    if (
+      service.local.port !== undefined &&
+      (!Number.isInteger(service.local.port) || service.local.port <= 0)
+    ) {
+      throw new Error(`Service "${name}" local.port must be a positive integer`);
+    }
+    if (service.local.cwd !== undefined && typeof service.local.cwd !== "string") {
+      throw new Error(`Service "${name}" local.cwd must be a string`);
+    }
+    if (
+      service.local.readyTimeoutMs !== undefined &&
+      (!Number.isInteger(service.local.readyTimeoutMs) || service.local.readyTimeoutMs <= 0)
+    ) {
+      throw new Error(`Service "${name}" local.readyTimeoutMs must be a positive integer`);
+    }
+    if (service.local.env !== undefined && !isObject(service.local.env)) {
+      throw new Error(`Service "${name}" local.env must be an object`);
+    }
+  }
+}
+
+export function validateTelemetryConfig(
+  telemetry,
+  configPath,
+  configName = "testkit.config.json"
+) {
+  if (!isObject(telemetry)) {
+    throw new Error(`${configName} telemetry must be an object (${configPath})`);
+  }
+  if (telemetry.enabled !== undefined && typeof telemetry.enabled !== "boolean") {
+    throw new Error(`${configName} telemetry.enabled must be a boolean (${configPath})`);
+  }
+  if (telemetry.endpoint !== undefined) {
+    requireString(telemetry, "endpoint", `${configName} telemetry.endpoint`);
+    validateHttpUrl(telemetry.endpoint, `${configName} telemetry.endpoint`);
+  }
+  if (telemetry.tokenEnv !== undefined) {
+    requireString(telemetry, "tokenEnv", `${configName} telemetry.tokenEnv`);
+  }
+  if (
+    telemetry.timeoutMs !== undefined &&
+    (!Number.isInteger(telemetry.timeoutMs) || telemetry.timeoutMs <= 0)
+  ) {
+    throw new Error(`${configName} telemetry.timeoutMs must be a positive integer (${configPath})`);
+  }
+  if (telemetry.enabled === true) {
+    requireString(telemetry, "endpoint", `${configName} telemetry.endpoint`);
+    requireString(telemetry, "tokenEnv", `${configName} telemetry.tokenEnv`);
+  }
+}
+
+export function getServiceEnvFiles(serviceConfig) {
+  const files = [];
+  if (serviceConfig.envFile) files.push(serviceConfig.envFile);
+  if (Array.isArray(serviceConfig.envFiles)) {
+    files.push(...serviceConfig.envFiles);
+  }
+  return files;
+}
+
+export function resolveSelectedDatabase(name, serviceConfig) {
+  if (!serviceConfig.database) return undefined;
+
+  const database = serviceConfig.database;
+  if (database.provider !== "local") {
+    throw new Error(
+      `Service "${name}" database.provider must be "local". Received "${database.provider}"`
+    );
+  }
+
+  return {
+    ...database,
+    provider: "local",
+    selectedBackend: "local",
+    reset: database.reset !== false,
+    template: normalizeTemplateConfig(database.template),
+  };
+}
+
+export function validateDatabaseProviderConfig(name, db, label) {
+  if (!VALID_DB_PROVIDERS.has(db.provider)) {
+    throw new Error(
+      `${label}.provider must be one of: ${[...VALID_DB_PROVIDERS].join(", ")}`
+    );
+  }
+
+  if (db.reset !== undefined && typeof db.reset !== "boolean") {
+    throw new Error(`${label}.reset must be a boolean`);
+  }
+  if (db.template !== undefined) {
+    validateTemplateConfig(name, db.template, `${label}.template`);
+  }
+  if (db.image !== undefined && typeof db.image !== "string") {
+    throw new Error(`${label}.image must be a string`);
+  }
+  if (db.user !== undefined && typeof db.user !== "string") {
+    throw new Error(`${label}.user must be a string`);
+  }
+  if (db.password !== undefined && typeof db.password !== "string") {
+    throw new Error(`${label}.password must be a string`);
+  }
+}
+
+export function validateTemplateConfig(name, template, label) {
+  if (!isObject(template)) {
+    throw new Error(`${label} must be an object`);
+  }
+  if (
+    template.inputs !== undefined &&
+    (!Array.isArray(template.inputs) || template.inputs.some((value) => typeof value !== "string"))
+  ) {
+    throw new Error(`${label}.inputs must be an array of strings`);
+  }
+}
+
+export function normalizeTemplateConfig(template) {
+  return {
+    inputs: Array.isArray(template?.inputs) ? [...template.inputs] : [],
+  };
+}
+
+export function validateLifecycleConfig(name, value, label) {
+  if (!isObject(value)) {
+    throw new Error(`Service "${name}" ${label} must be an object`);
+  }
+
+  if (value.cmd !== undefined) {
+    requireString(value, "cmd", `Service "${name}" ${label}.cmd`);
+  }
+  if (value.cwd !== undefined && typeof value.cwd !== "string") {
+    throw new Error(`Service "${name}" ${label}.cwd must be a string`);
+  }
+
+  if (value.backends !== undefined) {
+    if (!isObject(value.backends)) {
+      throw new Error(`Service "${name}" ${label}.backends must be an object`);
+    }
+    for (const [backendName, override] of Object.entries(value.backends)) {
+      if (!isObject(override)) {
+        throw new Error(
+          `Service "${name}" ${label}.backends.${backendName} must be an object`
+        );
+      }
+      if (override.cmd !== undefined) {
+        requireString(override, "cmd", `Service "${name}" ${label}.backends.${backendName}.cmd`);
+      }
+      if (override.cwd !== undefined && typeof override.cwd !== "string") {
+        throw new Error(
+          `Service "${name}" ${label}.backends.${backendName}.cwd must be a string`
+        );
+      }
+    }
+  }
+
+  if (value.cmd === undefined && value.backends === undefined) {
+    throw new Error(`Service "${name}" ${label} must define cmd or backends`);
+  }
+}
+
+export function resolveLifecycleConfig(value, selectedBackend) {
+  if (!value) return value;
+
+  const override =
+    selectedBackend && isObject(value.backends) ? value.backends[selectedBackend] || {} : {};
+  const resolved = {
+    ...value,
+    ...override,
+  };
+  delete resolved.backends;
+  return resolved;
+}
+
+export function requireString(obj, key, label) {
+  if (typeof obj[key] !== "string" || obj[key].length === 0) {
+    throw new Error(`${label} must be a non-empty string`);
+  }
+}
+
+export function isDalSuiteType(suite, runnerService, suitesForType) {
+  if (suite.framework && suite.framework !== "k6") return false;
+  return suitesForType === runnerService.suites.dal;
+}
+
+export function isObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+export function normalizeTelemetryConfig(telemetry) {
+  if (!isObject(telemetry)) return null;
+  return {
+    enabled: telemetry.enabled === true,
+    endpoint: telemetry.endpoint || null,
+    tokenEnv: telemetry.tokenEnv || null,
+    timeoutMs: telemetry.timeoutMs || 3_000,
+  };
+}
+
+export function validateHttpUrl(value, label) {
+  let parsed;
+  try {
+    parsed = new URL(value);
+  } catch {
+    throw new Error(`${label} must be a valid URL`);
+  }
+  if (!["http:", "https:"].includes(parsed.protocol)) {
+    throw new Error(`${label} must use http or https`);
+  }
+}

package/lib/config/model.test.mjs

@@ -0,0 +1,193 @@
+import { describe, expect, it } from "vitest";
+import {
+  getServiceEnvFiles,
+  isDalSuiteType,
+  normalizeTelemetryConfig,
+  parseDotenvString,
+  resolveLifecycleConfig,
+  resolveSelectedDatabase,
+  validateConfigCoverage,
+  validateLifecycleConfig,
+  validateRunnerManifest,
+  validateServiceConfig,
+  validateTelemetryConfig,
+} from "./model.mjs";
+
+describe("config-model", () => {
+  it("parses dotenv text", () => {
+    expect(
+      parseDotenvString(`
+# comment
+FOO=bar
+BAR="baz"
+QUX='zap'
+      `)
+    ).toEqual({
+      FOO: "bar",
+      BAR: "baz",
+      QUX: "zap",
+    });
+  });
+
+  it("validates runner manifests", () => {
+    expect(() =>
+      validateRunnerManifest({
+        services: {
+          api: {
+            suites: {
+              integration: [
+                { name: "health", files: ["tests/health.js"] },
+              ],
+            },
+          },
+        },
+      })
+    ).not.toThrow();
+
+    expect(() =>
+      validateRunnerManifest({
+        services: {
+          api: {
+            suites: {
+              integration: [
+                { name: "health", files: ["a.js"], framework: "jest" },
+              ],
+            },
+          },
+        },
+      })
+    ).toThrow("unsupported framework");
+  });
+
+  it("validates config coverage", () => {
+    const runner = {
+      services: {
+        api: { suites: {} },
+      },
+    };
+    const config = {
+      services: {
+        frontend: {
+          dependsOn: ["api"],
+        },
+      },
+    };
+
+    expect(() => validateConfigCoverage(runner, config)).toThrow(
+      'Service "frontend" exists in testkit.config.json but not in runner.manifest.json'
+    );
+  });
+
+  it("validates service and telemetry config", () => {
+    expect(() =>
+      validateServiceConfig("api", {
+        local: {
+          start: "npm run dev",
+          baseUrl: "http://127.0.0.1:3000",
+          readyUrl: "http://127.0.0.1:3000/health",
+          port: 3000,
+        },
+      }, "testkit.config.json")
+    ).not.toThrow();
+
+    expect(() =>
+      validateServiceConfig("api", {
+        database: {
+          backends: {},
+        },
+      }, "testkit.config.json")
+    ).toThrow("database.backends is no longer supported");
+
+    expect(() =>
+      validateTelemetryConfig(
+        {
+          enabled: true,
+          endpoint: "ftp://bad.example.com/upload",
+          tokenEnv: "TOKEN",
+        },
+        "testkit.config.json"
+      )
+    ).toThrow("must use http or https");
+  });
+
+  it("collects env files and resolves lifecycle overrides", () => {
+    expect(
+      getServiceEnvFiles({
+        envFile: ".env.shared",
+        envFiles: [".env.local"],
+      })
+    ).toEqual([".env.shared", ".env.local"]);
+
+    expect(() =>
+      validateLifecycleConfig("api", { backends: {} }, "migrate")
+    ).not.toThrow();
+
+    expect(
+      resolveLifecycleConfig(
+        {
+          cmd: "npm run migrate",
+          backends: {
+            local: {
+              cmd: "npm run migrate:local",
+            },
+          },
+        },
+        "local"
+      )
+    ).toEqual({
+      cmd: "npm run migrate:local",
+    });
+  });
+
+  it("normalizes telemetry and database config", () => {
+    expect(
+      resolveSelectedDatabase("api", {
+        database: {
+          provider: "local",
+          image: "pgvector/pgvector:pg16",
+          template: {
+            inputs: ["schema"],
+          },
+        },
+      })
+    ).toEqual({
+      provider: "local",
+      image: "pgvector/pgvector:pg16",
+      selectedBackend: "local",
+      reset: true,
+      template: {
+        inputs: ["schema"],
+      },
+    });
+
+    expect(
+      normalizeTelemetryConfig({
+        enabled: true,
+        endpoint: "https://example.com",
+        tokenEnv: "TOKEN",
+      })
+    ).toEqual({
+      enabled: true,
+      endpoint: "https://example.com",
+      tokenEnv: "TOKEN",
+      timeoutMs: 3000,
+    });
+  });
+
+  it("detects dal suite types", () => {
+    const suite = { name: "db", files: ["a.js"] };
+    const runnerService = {
+      suites: {
+        dal: [suite],
+      },
+    };
+    expect(isDalSuiteType(suite, runnerService, runnerService.suites.dal)).toBe(true);
+    expect(
+      isDalSuiteType(
+        { ...suite, framework: "playwright" },
+        runnerService,
+        runnerService.suites.dal
+      )
+    ).toBe(false);
+  });
+});

package/lib/database/fingerprint.mjs

@@ -0,0 +1,61 @@
+import crypto from "crypto";
+import fs from "fs";
+import path from "path";
+import { resolveServiceCwd } from "../config/index.mjs";
+
+const LOCAL_IMAGE = "pgvector/pgvector:pg16";
+const LOCAL_USER = "testkit";
+
+export async function computeTemplateFingerprint(config) {
+  const hash = crypto.createHash("sha256");
+  const db = config.testkit.database;
+  hash.update(JSON.stringify({
+    provider: db.provider,
+    selectedBackend: db.selectedBackend,
+    image: db.image || LOCAL_IMAGE,
+    user: db.user || LOCAL_USER,
+    migrate: config.testkit.migrate || null,
+    seed: config.testkit.seed || null,
+  }));
+
+  for (const envFile of config.testkit.envFiles || []) {
+    appendFileToHash(hash, config.productDir, resolveServiceCwd(config.productDir, envFile));
+  }
+  for (const input of db.template.inputs || []) {
+    appendInputToHash(hash, config.productDir, input);
+  }
+
+  return hash.digest("hex");
+}
+
+export function appendInputToHash(hash, productDir, input) {
+  const absPath = resolveServiceCwd(productDir, input);
+  if (!fs.existsSync(absPath)) {
+    hash.update(`missing:${path.relative(productDir, absPath)}`);
+    return;
+  }
+
+  const stat = fs.statSync(absPath);
+  if (stat.isDirectory()) {
+    hash.update(`dir:${path.relative(productDir, absPath)}`);
+    for (const entry of fs.readdirSync(absPath).sort()) {
+      if (entry === ".git" || entry === "node_modules" || entry === ".testkit") continue;
+      appendInputToHash(hash, productDir, path.join(input, entry));
+    }
+    return;
+  }
+
+  appendFileToHash(hash, productDir, absPath);
+}
+
+export function appendFileToHash(hash, productDir, absPath) {
+  if (!fs.existsSync(absPath)) {
+    hash.update(`missing:${path.relative(productDir, absPath)}`);
+    return;
+  }
+  const stat = fs.statSync(absPath);
+  if (!stat.isFile()) return;
+
+  hash.update(`file:${path.relative(productDir, absPath)}:${stat.size}:${stat.mtimeMs}`);
+  hash.update(fs.readFileSync(absPath));
+}