@elench/testkit 0.1.16 → 0.1.18

package/lib/runner/results.mjs

@@ -0,0 +1,285 @@
+import path from "path";
+
+export function buildServiceTrackers(servicePlans, startedAt) {
+  const trackers = new Map();
+
+  for (const plan of servicePlans) {
+    if (plan.skipped) {
+      trackers.set(plan.config.name, {
+        name: plan.config.name,
+        dbBackend: plan.config.testkit.database?.selectedBackend || null,
+        skipped: true,
+        suiteCount: 0,
+        suites: [],
+        suitesByKey: new Map(),
+        errors: [],
+        errorSet: new Set(),
+        startedAt,
+        firstTaskAt: null,
+        lastTaskAt: null,
+      });
+      continue;
+    }
+
+    const suites = plan.suites.map((suite) => ({
+      key: `${suite.type}:${suite.name}`,
+      name: suite.name,
+      type: suite.type,
+      framework: suite.framework,
+      orderIndex: suite.orderIndex,
+      fileCount: suite.files.length,
+      completedFileCount: 0,
+      failedFiles: [],
+      failedFileSet: new Set(),
+      fileResults: [],
+      fileResultsByPath: new Map(),
+      durationMs: 0,
+      error: null,
+    }));
+
+    trackers.set(plan.config.name, {
+      name: plan.config.name,
+      dbBackend: plan.config.testkit.database?.selectedBackend || null,
+      skipped: false,
+      suiteCount: suites.length,
+      suites,
+      suitesByKey: new Map(suites.map((suite) => [suite.key, suite])),
+      errors: [],
+      errorSet: new Set(),
+      startedAt,
+      firstTaskAt: null,
+      lastTaskAt: null,
+    });
+  }
+
+  return trackers;
+}
+
+export function recordTaskOutcome(trackers, task, outcome, finishedAt = Date.now()) {
+  const tracker = trackers.get(task.serviceName);
+  if (!tracker || tracker.skipped) return;
+
+  if (!tracker.firstTaskAt) tracker.firstTaskAt = finishedAt;
+  tracker.lastTaskAt = finishedAt;
+
+  const suite = tracker.suitesByKey.get(task.suiteKey);
+  if (!suite) return;
+
+  suite.completedFileCount += 1;
+  suite.durationMs += outcome.durationMs;
+  const normalizedPath = normalizePathSeparators(task.file);
+  const existingFileResult = suite.fileResultsByPath.get(normalizedPath);
+  if (existingFileResult) {
+    existingFileResult.failed = outcome.failed;
+    existingFileResult.durationMs = outcome.durationMs;
+    existingFileResult.error = outcome.error;
+  } else {
+    const fileResult = {
+      path: normalizedPath,
+      failed: outcome.failed,
+      durationMs: outcome.durationMs,
+      error: outcome.error,
+    };
+    suite.fileResultsByPath.set(normalizedPath, fileResult);
+    suite.fileResults.push(fileResult);
+  }
+  if (outcome.failed && !suite.failedFileSet.has(task.file)) {
+    suite.failedFileSet.add(task.file);
+    suite.failedFiles.push(task.file);
+  }
+  if (outcome.error && !suite.error) {
+    suite.error = outcome.error;
+  }
+}
+
+export function recordGraphError(trackers, graph, message, now = Date.now()) {
+  const targetNames = graph?.assignedTargets || [];
+  for (const targetName of targetNames) {
+    const tracker = trackers.get(targetName);
+    if (tracker && !tracker.skipped) {
+      addTrackerError(tracker, message);
+      tracker.lastTaskAt = now;
+    }
+  }
+}
+
+export function addTrackerError(tracker, message) {
+  if (tracker.errorSet.has(message)) return;
+  tracker.errorSet.add(message);
+  tracker.errors.push(message);
+}
+
+export function finalizeServiceResult(tracker, startedAt, finishedAt) {
+  if (!tracker || tracker.skipped) {
+    return {
+      name: tracker?.name || "unknown",
+      failed: false,
+      skipped: true,
+      suiteCount: 0,
+      completedSuiteCount: 0,
+      failedSuiteCount: 0,
+      durationMs: 0,
+      suites: [],
+      errors: [],
+    };
+  }
+
+  const suites = [...tracker.suites].sort(
+    (a, b) => a.orderIndex - b.orderIndex || a.name.localeCompare(b.name)
+  );
+  const completedSuiteCount = suites.filter(
+    (suite) => suite.completedFileCount === suite.fileCount
+  ).length;
+  const failedSuiteCount = suites.filter((suite) => suite.failedFiles.length > 0).length;
+  const accumulatedDurationMs = suites.reduce((sum, suite) => sum + suite.durationMs, 0);
+  const durationMs =
+    tracker.firstTaskAt && tracker.lastTaskAt
+      ? Math.max(tracker.lastTaskAt - tracker.firstTaskAt, accumulatedDurationMs)
+      : Math.max(finishedAt - startedAt, accumulatedDurationMs);
+
+  return {
+    name: tracker.name,
+    dbBackend: tracker.dbBackend,
+    failed:
+      failedSuiteCount > 0 ||
+      tracker.errors.length > 0 ||
+      completedSuiteCount < tracker.suiteCount,
+    skipped: false,
+    suiteCount: tracker.suiteCount,
+    completedSuiteCount,
+    failedSuiteCount,
+    durationMs,
+    suites: suites.map((suite) => ({
+      name: suite.name,
+      type: suite.type,
+      framework: suite.framework,
+      failed: suite.failedFiles.length > 0,
+      fileCount: suite.fileCount,
+      failedFiles: suite.failedFiles,
+      durationMs: suite.durationMs,
+      error: suite.error,
+      files: suite.fileResults
+        .slice()
+        .sort((a, b) => a.path.localeCompare(b.path))
+        .map((file) => ({
+          path: file.path,
+          failed: file.failed,
+          durationMs: file.durationMs,
+          error: file.error,
+        })),
+    })),
+    errors: tracker.errors,
+  };
+}
+
+export function buildRunArtifact({
+  productDir,
+  results,
+  startedAt,
+  finishedAt,
+  requestedJobs,
+  workerCount,
+  suiteType,
+  suiteNames,
+  framework,
+  shard,
+  serviceFilter,
+  metadata,
+}) {
+  const executed = results.filter((result) => !result.skipped);
+  const failedServices = executed.filter((result) => result.failed);
+  const totalSuites = executed.reduce((sum, result) => sum + result.suiteCount, 0);
+  const completedSuites = executed.reduce((sum, result) => sum + result.completedSuiteCount, 0);
+  const failedSuites = executed.reduce((sum, result) => sum + result.failedSuiteCount, 0);
+  const dbBackend = summarizeDbBackend(results);
+
+  return {
+    schemaVersion: 1,
+    source: "testkit",
+    generatedAt: new Date(finishedAt).toISOString(),
+    product: {
+      name: path.basename(productDir),
+      directory: productDir,
+    },
+    git: metadata.git,
+    host: metadata.host,
+    run: {
+      status: failedServices.length > 0 ? "failed" : "passed",
+      startedAt: new Date(startedAt).toISOString(),
+      finishedAt: new Date(finishedAt).toISOString(),
+      durationMs: finishedAt - startedAt,
+      requestedJobs,
+      workerCount,
+      dbBackend,
+      suiteType,
+      suiteNames,
+      framework,
+      shard,
+      serviceFilter,
+      testkitVersion: metadata.testkitVersion,
+    },
+    summary: {
+      services: {
+        total: executed.length,
+        passed: executed.length - failedServices.length,
+        failed: failedServices.length,
+      },
+      suites: {
+        total: totalSuites,
+        completed: completedSuites,
+        passed: completedSuites - failedSuites,
+        failed: failedSuites,
+      },
+    },
+    services: results.map((result) => ({
+      name: result.name,
+      failed: result.failed,
+      skipped: result.skipped,
+      suiteCount: result.suiteCount,
+      completedSuiteCount: result.completedSuiteCount,
+      failedSuiteCount: result.failedSuiteCount,
+      durationMs: result.durationMs,
+      dbBackend: result.dbBackend,
+      suites: result.suites,
+      errors: result.errors,
+    })),
+  };
+}
+
+export function summarizeDbBackend(results) {
+  const values = [...new Set(results.map((result) => result.dbBackend).filter(Boolean))];
+  if (values.length === 0) return null;
+  if (values.length === 1) return values[0];
+  return "mixed";
+}
+
+export function formatDuration(durationMs) {
+  const totalSeconds = Math.max(0, Math.round(durationMs / 1000));
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  if (minutes === 0) return `${seconds}s`;
+  return `${minutes}m ${String(seconds).padStart(2, "0")}s`;
+}
+
+export function formatServiceSummary(result) {
+  const passedSuites = result.completedSuiteCount - result.failedSuiteCount;
+  const notRun = result.suiteCount - result.completedSuiteCount;
+  let detail = `${passedSuites}/${result.suiteCount} suites passed`;
+  if (notRun > 0) {
+    detail += `, ${notRun} not run`;
+  }
+  return detail;
+}
+
+export function formatError(error) {
+  if (error instanceof Error) return error.message;
+  return String(error);
+}
+
+export function longestServiceName(results) {
+  return results.reduce((max, result) => Math.max(max, result.name.length), 4);
+}
+
+function normalizePathSeparators(filePath) {
+  return filePath.split(path.sep).join("/");
+}

package/lib/runner/results.test.mjs

@@ -0,0 +1,144 @@
+import { describe, expect, it } from "vitest";
+import {
+  addTrackerError,
+  buildRunArtifact,
+  buildServiceTrackers,
+  finalizeServiceResult,
+  formatDuration,
+  formatError,
+  formatServiceSummary,
+  recordGraphError,
+  recordTaskOutcome,
+  summarizeDbBackend,
+} from "./results.mjs";
+
+describe("runner-results", () => {
+  it("tracks task outcomes and graph errors", () => {
+    const trackers = buildServiceTrackers(
+      [
+        {
+          skipped: false,
+          config: {
+            name: "api",
+            testkit: {
+              database: {
+                selectedBackend: "local",
+              },
+            },
+          },
+          suites: [
+            {
+              name: "health",
+              type: "integration",
+              framework: "k6",
+              files: ["tests/health.js"],
+              orderIndex: 0,
+            },
+          ],
+        },
+      ],
+      1000
+    );
+
+    recordTaskOutcome(
+      trackers,
+      {
+        serviceName: "api",
+        suiteKey: "integration:health",
+        file: "tests/health.js",
+      },
+      {
+        failed: true,
+        durationMs: 250,
+        error: "boom",
+      },
+      1250
+    );
+
+    const tracker = trackers.get("api");
+    addTrackerError(tracker, "worker failed");
+    addTrackerError(tracker, "worker failed");
+    recordGraphError(trackers, { assignedTargets: ["api"] }, "graph failed", 1300);
+
+    const result = finalizeServiceResult(tracker, 1000, 1500);
+    expect(result.failed).toBe(true);
+    expect(result.failedSuiteCount).toBe(1);
+    expect(result.errors).toEqual(["worker failed", "graph failed"]);
+    expect(result.suites[0].files).toEqual([
+      {
+        path: "tests/health.js",
+        failed: true,
+        durationMs: 250,
+        error: "boom",
+      },
+    ]);
+  });
+
+  it("builds run artifacts and formatting helpers", () => {
+    const results = [
+      {
+        name: "api",
+        failed: false,
+        skipped: false,
+        suiteCount: 1,
+        completedSuiteCount: 1,
+        failedSuiteCount: 0,
+        durationMs: 1200,
+        dbBackend: "local",
+        suites: [],
+        errors: [],
+      },
+      {
+        name: "frontend",
+        failed: false,
+        skipped: true,
+        suiteCount: 0,
+        completedSuiteCount: 0,
+        failedSuiteCount: 0,
+        durationMs: 0,
+        dbBackend: null,
+        suites: [],
+        errors: [],
+      },
+    ];
+
+    const artifact = buildRunArtifact({
+      productDir: "/tmp/my-product",
+      results,
+      startedAt: 1000,
+      finishedAt: 4000,
+      requestedJobs: 2,
+      workerCount: 1,
+      suiteType: "all",
+      suiteNames: [],
+      framework: "all",
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc",
+          repoRoot: "/tmp",
+        },
+        host: {
+          hostname: "local",
+          username: "dev",
+        },
+        testkitVersion: "0.1.17",
+      },
+    });
+
+    expect(artifact.product.name).toBe("my-product");
+    expect(artifact.summary.services.total).toBe(1);
+    expect(summarizeDbBackend(results)).toBe("local");
+    expect(formatDuration(65_000)).toBe("1m 05s");
+    expect(
+      formatServiceSummary({
+        completedSuiteCount: 2,
+        failedSuiteCount: 1,
+        suiteCount: 3,
+      })
+    ).toBe("1/3 suites passed, 1 not run");
+    expect(formatError(new Error("boom"))).toBe("boom");
+  });
+});

package/lib/runner/state.mjs

@@ -0,0 +1,71 @@
+import fs from "fs";
+import path from "path";
+
+const GRAPH_METADATA = "graph.json";
+
+export function findRuntimeStateDirs(rootDir, isDatabaseStateDir) {
+  const found = [];
+
+  const visit = (dir) => {
+    if (!fs.existsSync(dir)) return;
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    if (isDatabaseStateDir(dir)) {
+      found.push(dir);
+    }
+
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        visit(path.join(dir, entry.name));
+      }
+    }
+  };
+
+  visit(rootDir);
+  return found.sort((a, b) => b.length - a.length);
+}
+
+export function findGraphDirsForService(productDir, serviceName) {
+  const graphsRoot = path.join(productDir, ".testkit", "_graphs");
+  if (!fs.existsSync(graphsRoot)) return [];
+
+  const matches = [];
+  for (const entry of fs.readdirSync(graphsRoot, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const graphDir = path.join(graphsRoot, entry.name);
+    const metadata = readGraphMetadata(graphDir);
+    if (!metadata) continue;
+    if ((metadata.runtimeServices || []).includes(serviceName)) {
+      matches.push(graphDir);
+    }
+  }
+
+  return matches.sort();
+}
+
+export function writeGraphMetadata(graphDir, graph) {
+  fs.mkdirSync(graphDir, { recursive: true });
+  const metadata = {
+    runtimeServices: graph.runtimeNames,
+    assignedTargets: [...graph.assignedTargets].sort(),
+    rootService: graph.rootConfig.name,
+  };
+  fs.writeFileSync(
+    path.join(graphDir, GRAPH_METADATA),
+    JSON.stringify(metadata, null, 2)
+  );
+}
+
+export function readGraphMetadata(graphDir) {
+  const metadataPath = path.join(graphDir, GRAPH_METADATA);
+  if (!fs.existsSync(metadataPath)) return null;
+
+  try {
+    return JSON.parse(fs.readFileSync(metadataPath, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+export function normalizePathSeparators(filePath) {
+  return filePath.split(path.sep).join("/");
+}

package/lib/runner/state.test.mjs

@@ -0,0 +1,64 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  findGraphDirsForService,
+  findRuntimeStateDirs,
+  normalizePathSeparators,
+  readGraphMetadata,
+  writeGraphMetadata,
+} from "./state.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  for (const dir of tempDirs.splice(0)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+function mkTempDir() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-runner-state-"));
+  tempDirs.push(dir);
+  return dir;
+}
+
+describe("runner-state", () => {
+  it("finds runtime state directories", () => {
+    const rootDir = mkTempDir();
+    const deep = path.join(rootDir, "workers", "w1", "deps", "api");
+    fs.mkdirSync(deep, { recursive: true });
+    fs.writeFileSync(path.join(rootDir, "workers", "database_backend"), "local");
+    fs.writeFileSync(path.join(deep, "database_backend"), "local");
+
+    const found = findRuntimeStateDirs(
+      rootDir,
+      (dir) => fs.existsSync(path.join(dir, "database_backend"))
+    );
+
+    expect(found).toEqual([
+      deep,
+      path.join(rootDir, "workers"),
+    ]);
+  });
+
+  it("writes, reads, and finds graph metadata", () => {
+    const productDir = mkTempDir();
+    const graphDir = path.join(productDir, ".testkit", "_graphs", "api__frontend");
+    writeGraphMetadata(graphDir, {
+      runtimeNames: ["api", "frontend"],
+      assignedTargets: ["frontend", "api"],
+      rootConfig: { name: "api" },
+    });
+
+    expect(readGraphMetadata(graphDir)).toEqual({
+      runtimeServices: ["api", "frontend"],
+      assignedTargets: ["api", "frontend"],
+      rootService: "api",
+    });
+
+    expect(findGraphDirsForService(productDir, "frontend")).toEqual([graphDir]);
+    expect(normalizePathSeparators(`a${path.sep}b${path.sep}c`)).toBe("a/b/c");
+  });
+});