@elench/testkit 0.1.32 → 0.1.34

package/lib/runner/readiness.mjs

@@ -0,0 +1,117 @@
+import net from "net";
+import {
+  cleanupStaleRuns,
+  findPortOwner,
+  formatRunSummary,
+  isPidRunning,
+  listRunManifests,
+} from "./lifecycle.mjs";
+import { socketFromUrl } from "./template.mjs";
+
+export const DEFAULT_READY_TIMEOUT_MS = 120_000;
+
+export async function waitForReady({ name, url, timeoutMs, process, signal, sleep }) {
+  const start = Date.now();
+
+  while (Date.now() - start < timeoutMs) {
+    if (signal?.aborted) {
+      throw signal.reason || new Error(`Service "${name}" startup aborted`);
+    }
+    if (process.exitCode !== null) {
+      throw new Error(`Service "${name}" exited before becoming ready`);
+    }
+
+    try {
+      const response = await fetch(url);
+      if (response.ok) return;
+    } catch {
+      // Service still warming up.
+    }
+
+    await sleep(1_000);
+  }
+
+  throw new Error(`Timed out waiting for "${name}" to become ready at ${url}`);
+}
+
+export async function assertLocalServicePortsAvailable(config, isPortInUse) {
+  const endpoints = [config.testkit.local.baseUrl, config.testkit.local.readyUrl];
+  const seen = new Set();
+
+  for (const endpoint of endpoints) {
+    const socket = socketFromUrl(endpoint);
+    if (!socket) continue;
+
+    const key = `${socket.host}:${socket.port}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+
+    if (await isPortInUse(socket)) {
+      await cleanupStaleRuns(config.productDir);
+    }
+
+    if (await isPortInUse(socket)) {
+      const owner = findPortOwner(config.productDir, socket);
+      const ownerDetail = owner
+        ? owner.active
+          ? ` Active testkit run ${formatRunSummary(owner.manifest)} owns ${key} via ${owner.service.workerLabel}:${owner.service.serviceName}.`
+          : ` Stale testkit run ${formatRunSummary(owner.manifest)} owns ${key}.`
+        : "";
+      throw new Error(
+        `Cannot start "${config.workerLabel}:${config.name}" because ${key} is already in use. ` +
+          `Stop the existing process and rerun testkit.${ownerDetail}`
+      );
+    }
+  }
+}
+
+export async function isPortInUse({ host, port }) {
+  return new Promise((resolve, reject) => {
+    const socket = new net.Socket();
+    let settled = false;
+
+    const finish = (value, error = null) => {
+      if (settled) return;
+      settled = true;
+      socket.destroy();
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(value);
+    };
+
+    socket.setTimeout(1_000);
+    socket.once("connect", () => finish(true));
+    socket.once("timeout", () => finish(false));
+    socket.once("error", (error) => {
+      if (["ECONNREFUSED", "EHOSTUNREACH", "ENOTFOUND"].includes(error.code)) {
+        finish(false);
+        return;
+      }
+      finish(false, error);
+    });
+
+    socket.connect(port, host);
+  });
+}
+
+export function printRunStatus(productDir) {
+  const manifests = listRunManifests(productDir);
+  if (manifests.length === 0) return;
+
+  console.log("  runs/");
+  for (const manifest of manifests) {
+    const state = isPidRunning(manifest.pid) ? "active" : "stale";
+    const ports = [
+      ...new Set(
+        (manifest.services || []).flatMap((service) =>
+          (service.ports || []).map((socket) => `${socket.host}:${socket.port}`)
+        )
+      ),
+    ];
+    console.log(
+      `    ${manifest.runId}: ${state} pid=${manifest.pid}${ports.length > 0 ? ` ports=${ports.join(",")}` : ""}`
+    );
+  }
+}

package/lib/runner/reporting.mjs

@@ -0,0 +1,180 @@
+import path from "path";
+
+export function buildStatusArtifact({
+  productDir,
+  results,
+  suiteType,
+  suiteNames,
+  fileNames,
+  framework,
+  shard,
+  serviceFilter,
+  metadata,
+}) {
+  const executedResults = results.filter((result) => !result.skipped);
+  const tests = [];
+
+  for (const result of executedResults) {
+    for (const suite of result.suites) {
+      for (const file of suite.files) {
+        tests.push({
+          service: result.name,
+          type: suite.type,
+          path: file.path,
+          status: file.status,
+        });
+      }
+    }
+  }
+
+  tests.sort(
+    (left, right) =>
+      left.service.localeCompare(right.service) ||
+      left.type.localeCompare(right.type) ||
+      left.path.localeCompare(right.path)
+  );
+
+  const summary = {
+    services: {
+      total: executedResults.length,
+      passed: executedResults.filter((result) => !result.failed).length,
+      failed: executedResults.filter((result) => result.failed).length,
+    },
+    tests: {
+      total: tests.length,
+      passed: tests.filter((test) => test.status === "passed").length,
+      failed: tests.filter((test) => test.status === "failed").length,
+      notRun: tests.filter((test) => test.status === "not_run").length,
+    },
+  };
+
+  const scope = {
+    suiteType,
+    suiteNames: [...(suiteNames || [])].sort(),
+    fileNames: [...(fileNames || [])].sort(),
+    framework: formatFrameworkForArtifact(framework || "all"),
+    shard: shard || null,
+    serviceFilter: serviceFilter || null,
+  };
+  scope.isFullRun =
+    scope.suiteNames.length === 0 &&
+    scope.fileNames.length === 0 &&
+    scope.framework === "all" &&
+    scope.shard === null &&
+    scope.serviceFilter === null;
+
+  return {
+    schemaVersion: 1,
+    source: "testkit",
+    notice: "Generated file. Do not edit manually.",
+    product: {
+      name: path.basename(productDir),
+    },
+    git: {
+      branch: metadata.git?.branch || null,
+      commitSha: metadata.git?.commitSha || null,
+    },
+    testkitVersion: metadata.testkitVersion,
+    scope,
+    summary,
+    tests,
+  };
+}
+
+export function buildRunArtifact({
+  productDir,
+  results,
+  startedAt,
+  finishedAt,
+  requestedJobs,
+  workerCount,
+  suiteType,
+  suiteNames,
+  fileNames,
+  framework,
+  shard,
+  serviceFilter,
+  metadata,
+  summarizeDbBackend,
+}) {
+  const executed = results.filter((result) => !result.skipped);
+  const failedServices = executed.filter((result) => result.failed);
+  const totalSuites = executed.reduce((sum, result) => sum + result.suiteCount, 0);
+  const completedSuites = executed.reduce((sum, result) => sum + result.completedSuiteCount, 0);
+  const failedSuites = executed.reduce((sum, result) => sum + result.failedSuiteCount, 0);
+  const totalFiles = executed.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
+  const passedFiles = executed.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
+  const failedFiles = executed.reduce((sum, result) => sum + (result.failedFileCount || 0), 0);
+  const notRunFiles = executed.reduce((sum, result) => sum + (result.notRunFileCount || 0), 0);
+  const dbBackend = summarizeDbBackend(results);
+
+  return {
+    schemaVersion: 1,
+    source: "testkit",
+    generatedAt: new Date(finishedAt).toISOString(),
+    product: {
+      name: path.basename(productDir),
+      directory: productDir,
+    },
+    git: metadata.git,
+    host: metadata.host,
+    run: {
+      status: failedServices.length > 0 ? "failed" : "passed",
+      startedAt: new Date(startedAt).toISOString(),
+      finishedAt: new Date(finishedAt).toISOString(),
+      durationMs: finishedAt - startedAt,
+      requestedJobs,
+      workerCount,
+      dbBackend,
+      suiteType,
+      suiteNames,
+      fileNames,
+      framework: formatFrameworkForArtifact(framework),
+      shard,
+      serviceFilter,
+      testkitVersion: metadata.testkitVersion,
+    },
+    summary: {
+      services: {
+        total: executed.length,
+        passed: executed.length - failedServices.length,
+        failed: failedServices.length,
+      },
+      suites: {
+        total: totalSuites,
+        completed: completedSuites,
+        passed: completedSuites - failedSuites,
+        failed: failedSuites,
+      },
+      files: {
+        total: totalFiles,
+        passed: passedFiles,
+        failed: failedFiles,
+        notRun: notRunFiles,
+      },
+    },
+    services: results.map((result) => ({
+      name: result.name,
+      failed: result.failed,
+      skipped: result.skipped,
+      suiteCount: result.suiteCount,
+      completedSuiteCount: result.completedSuiteCount,
+      failedSuiteCount: result.failedSuiteCount,
+      totalFileCount: result.totalFileCount,
+      completedFileCount: result.completedFileCount,
+      passedFileCount: result.passedFileCount,
+      failedFileCount: result.failedFileCount,
+      notRunFileCount: result.notRunFileCount,
+      durationMs: result.durationMs,
+      totalTaskDurationMs: result.totalTaskDurationMs,
+      dbBackend: result.dbBackend,
+      suites: result.suites,
+      errors: result.errors,
+    })),
+  };
+}
+
+function formatFrameworkForArtifact(framework) {
+  if (framework === "k6") return "default";
+  return framework;
+}

package/lib/runner/reporting.test.mjs

@@ -0,0 +1,193 @@
+import { describe, expect, it } from "vitest";
+import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+
+describe("runner reporting", () => {
+  it("builds run artifacts", () => {
+    const results = [
+      {
+        name: "api",
+        failed: false,
+        skipped: false,
+        suiteCount: 1,
+        completedSuiteCount: 1,
+        failedSuiteCount: 0,
+        totalFileCount: 3,
+        completedFileCount: 3,
+        passedFileCount: 3,
+        failedFileCount: 0,
+        notRunFileCount: 0,
+        durationMs: 1200,
+        totalTaskDurationMs: 2400,
+        dbBackend: "local",
+        suites: [],
+        errors: [],
+      },
+      {
+        name: "frontend",
+        failed: false,
+        skipped: true,
+        suiteCount: 0,
+        completedSuiteCount: 0,
+        failedSuiteCount: 0,
+        totalFileCount: 0,
+        completedFileCount: 0,
+        passedFileCount: 0,
+        failedFileCount: 0,
+        notRunFileCount: 0,
+        durationMs: 0,
+        totalTaskDurationMs: 0,
+        dbBackend: null,
+        suites: [],
+        errors: [],
+      },
+    ];
+
+    const artifact = buildRunArtifact({
+      productDir: "/tmp/my-product",
+      results,
+      startedAt: 1000,
+      finishedAt: 4000,
+      requestedJobs: 2,
+      workerCount: 1,
+      suiteType: "all",
+      suiteNames: [],
+      fileNames: [],
+      framework: "all",
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc",
+          repoRoot: "/tmp",
+        },
+        host: {
+          hostname: "local",
+          username: "dev",
+        },
+        testkitVersion: "0.1.17",
+      },
+      summarizeDbBackend: () => "local",
+    });
+
+    expect(artifact.product.name).toBe("my-product");
+    expect(artifact.summary.services.total).toBe(1);
+    expect(artifact.summary.files).toEqual({
+      total: 3,
+      passed: 3,
+      failed: 0,
+      notRun: 0,
+    });
+    expect(artifact.services[0].durationMs).toBe(1200);
+    expect(artifact.services[0].totalTaskDurationMs).toBe(2400);
+  });
+
+  it("builds deterministic status artifacts", () => {
+    const status = buildStatusArtifact({
+      productDir: "/tmp/my-product",
+      results: [
+        {
+          name: "api",
+          failed: true,
+          skipped: false,
+          suites: [
+            {
+              name: "health",
+              type: "integration",
+              framework: "k6",
+              files: [
+                { path: "tests/api/integration/a.int.testkit.ts", status: "passed" },
+                { path: "tests/api/integration/b.int.testkit.ts", status: "failed" },
+              ],
+            },
+          ],
+        },
+      ],
+      suiteType: "int",
+      suiteNames: ["health"],
+      fileNames: ["tests/api/integration/b.int.testkit.ts"],
+      framework: "default",
+      shard: null,
+      serviceFilter: "api",
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc123",
+        },
+        testkitVersion: "0.1.20",
+      },
+    });
+
+    expect(status).toEqual({
+      schemaVersion: 1,
+      source: "testkit",
+      notice: "Generated file. Do not edit manually.",
+      product: {
+        name: "my-product",
+      },
+      git: {
+        branch: "main",
+        commitSha: "abc123",
+      },
+      testkitVersion: "0.1.20",
+      scope: {
+        suiteType: "int",
+        suiteNames: ["health"],
+        fileNames: ["tests/api/integration/b.int.testkit.ts"],
+        framework: "default",
+        shard: null,
+        serviceFilter: "api",
+        isFullRun: false,
+      },
+      summary: {
+        services: {
+          total: 1,
+          passed: 0,
+          failed: 1,
+        },
+        tests: {
+          total: 2,
+          passed: 1,
+          failed: 1,
+          notRun: 0,
+        },
+      },
+      tests: [
+        {
+          service: "api",
+          type: "integration",
+          path: "tests/api/integration/a.int.testkit.ts",
+          status: "passed",
+        },
+        {
+          service: "api",
+          type: "integration",
+          path: "tests/api/integration/b.int.testkit.ts",
+          status: "failed",
+        },
+      ],
+    });
+  });
+
+  it("marks unfiltered status artifacts as full runs", () => {
+    const status = buildStatusArtifact({
+      productDir: "/tmp/my-product",
+      results: [],
+      suiteType: "all",
+      suiteNames: [],
+      fileNames: [],
+      framework: "all",
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc123",
+        },
+        testkitVersion: "0.1.20",
+      },
+    });
+
+    expect(status.scope.isFullRun).toBe(true);
+  });
+});