@elench/testkit 0.1.32 → 0.1.33

package/lib/runner/reporting.mjs

@@ -0,0 +1,180 @@
+import path from "path";
+
+export function buildStatusArtifact({
+  productDir,
+  results,
+  suiteType,
+  suiteNames,
+  fileNames,
+  framework,
+  shard,
+  serviceFilter,
+  metadata,
+}) {
+  const executedResults = results.filter((result) => !result.skipped);
+  const tests = [];
+
+  for (const result of executedResults) {
+    for (const suite of result.suites) {
+      for (const file of suite.files) {
+        tests.push({
+          service: result.name,
+          type: suite.type,
+          path: file.path,
+          status: file.status,
+        });
+      }
+    }
+  }
+
+  tests.sort(
+    (left, right) =>
+      left.service.localeCompare(right.service) ||
+      left.type.localeCompare(right.type) ||
+      left.path.localeCompare(right.path)
+  );
+
+  const summary = {
+    services: {
+      total: executedResults.length,
+      passed: executedResults.filter((result) => !result.failed).length,
+      failed: executedResults.filter((result) => result.failed).length,
+    },
+    tests: {
+      total: tests.length,
+      passed: tests.filter((test) => test.status === "passed").length,
+      failed: tests.filter((test) => test.status === "failed").length,
+      notRun: tests.filter((test) => test.status === "not_run").length,
+    },
+  };
+
+  const scope = {
+    suiteType,
+    suiteNames: [...(suiteNames || [])].sort(),
+    fileNames: [...(fileNames || [])].sort(),
+    framework: formatFrameworkForArtifact(framework || "all"),
+    shard: shard || null,
+    serviceFilter: serviceFilter || null,
+  };
+  scope.isFullRun =
+    scope.suiteNames.length === 0 &&
+    scope.fileNames.length === 0 &&
+    scope.framework === "all" &&
+    scope.shard === null &&
+    scope.serviceFilter === null;
+
+  return {
+    schemaVersion: 1,
+    source: "testkit",
+    notice: "Generated file. Do not edit manually.",
+    product: {
+      name: path.basename(productDir),
+    },
+    git: {
+      branch: metadata.git?.branch || null,
+      commitSha: metadata.git?.commitSha || null,
+    },
+    testkitVersion: metadata.testkitVersion,
+    scope,
+    summary,
+    tests,
+  };
+}
+
+export function buildRunArtifact({
+  productDir,
+  results,
+  startedAt,
+  finishedAt,
+  requestedJobs,
+  workerCount,
+  suiteType,
+  suiteNames,
+  fileNames,
+  framework,
+  shard,
+  serviceFilter,
+  metadata,
+  summarizeDbBackend,
+}) {
+  const executed = results.filter((result) => !result.skipped);
+  const failedServices = executed.filter((result) => result.failed);
+  const totalSuites = executed.reduce((sum, result) => sum + result.suiteCount, 0);
+  const completedSuites = executed.reduce((sum, result) => sum + result.completedSuiteCount, 0);
+  const failedSuites = executed.reduce((sum, result) => sum + result.failedSuiteCount, 0);
+  const totalFiles = executed.reduce((sum, result) => sum + (result.totalFileCount || 0), 0);
+  const passedFiles = executed.reduce((sum, result) => sum + (result.passedFileCount || 0), 0);
+  const failedFiles = executed.reduce((sum, result) => sum + (result.failedFileCount || 0), 0);
+  const notRunFiles = executed.reduce((sum, result) => sum + (result.notRunFileCount || 0), 0);
+  const dbBackend = summarizeDbBackend(results);
+
+  return {
+    schemaVersion: 1,
+    source: "testkit",
+    generatedAt: new Date(finishedAt).toISOString(),
+    product: {
+      name: path.basename(productDir),
+      directory: productDir,
+    },
+    git: metadata.git,
+    host: metadata.host,
+    run: {
+      status: failedServices.length > 0 ? "failed" : "passed",
+      startedAt: new Date(startedAt).toISOString(),
+      finishedAt: new Date(finishedAt).toISOString(),
+      durationMs: finishedAt - startedAt,
+      requestedJobs,
+      workerCount,
+      dbBackend,
+      suiteType,
+      suiteNames,
+      fileNames,
+      framework: formatFrameworkForArtifact(framework),
+      shard,
+      serviceFilter,
+      testkitVersion: metadata.testkitVersion,
+    },
+    summary: {
+      services: {
+        total: executed.length,
+        passed: executed.length - failedServices.length,
+        failed: failedServices.length,
+      },
+      suites: {
+        total: totalSuites,
+        completed: completedSuites,
+        passed: completedSuites - failedSuites,
+        failed: failedSuites,
+      },
+      files: {
+        total: totalFiles,
+        passed: passedFiles,
+        failed: failedFiles,
+        notRun: notRunFiles,
+      },
+    },
+    services: results.map((result) => ({
+      name: result.name,
+      failed: result.failed,
+      skipped: result.skipped,
+      suiteCount: result.suiteCount,
+      completedSuiteCount: result.completedSuiteCount,
+      failedSuiteCount: result.failedSuiteCount,
+      totalFileCount: result.totalFileCount,
+      completedFileCount: result.completedFileCount,
+      passedFileCount: result.passedFileCount,
+      failedFileCount: result.failedFileCount,
+      notRunFileCount: result.notRunFileCount,
+      durationMs: result.durationMs,
+      totalTaskDurationMs: result.totalTaskDurationMs,
+      dbBackend: result.dbBackend,
+      suites: result.suites,
+      errors: result.errors,
+    })),
+  };
+}
+
+function formatFrameworkForArtifact(framework) {
+  if (framework === "k6") return "default";
+  return framework;
+}

package/lib/runner/reporting.test.mjs

@@ -0,0 +1,193 @@
+import { describe, expect, it } from "vitest";
+import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+
+describe("runner reporting", () => {
+  it("builds run artifacts", () => {
+    const results = [
+      {
+        name: "api",
+        failed: false,
+        skipped: false,
+        suiteCount: 1,
+        completedSuiteCount: 1,
+        failedSuiteCount: 0,
+        totalFileCount: 3,
+        completedFileCount: 3,
+        passedFileCount: 3,
+        failedFileCount: 0,
+        notRunFileCount: 0,
+        durationMs: 1200,
+        totalTaskDurationMs: 2400,
+        dbBackend: "local",
+        suites: [],
+        errors: [],
+      },
+      {
+        name: "frontend",
+        failed: false,
+        skipped: true,
+        suiteCount: 0,
+        completedSuiteCount: 0,
+        failedSuiteCount: 0,
+        totalFileCount: 0,
+        completedFileCount: 0,
+        passedFileCount: 0,
+        failedFileCount: 0,
+        notRunFileCount: 0,
+        durationMs: 0,
+        totalTaskDurationMs: 0,
+        dbBackend: null,
+        suites: [],
+        errors: [],
+      },
+    ];
+
+    const artifact = buildRunArtifact({
+      productDir: "/tmp/my-product",
+      results,
+      startedAt: 1000,
+      finishedAt: 4000,
+      requestedJobs: 2,
+      workerCount: 1,
+      suiteType: "all",
+      suiteNames: [],
+      fileNames: [],
+      framework: "all",
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc",
+          repoRoot: "/tmp",
+        },
+        host: {
+          hostname: "local",
+          username: "dev",
+        },
+        testkitVersion: "0.1.17",
+      },
+      summarizeDbBackend: () => "local",
+    });
+
+    expect(artifact.product.name).toBe("my-product");
+    expect(artifact.summary.services.total).toBe(1);
+    expect(artifact.summary.files).toEqual({
+      total: 3,
+      passed: 3,
+      failed: 0,
+      notRun: 0,
+    });
+    expect(artifact.services[0].durationMs).toBe(1200);
+    expect(artifact.services[0].totalTaskDurationMs).toBe(2400);
+  });
+
+  it("builds deterministic status artifacts", () => {
+    const status = buildStatusArtifact({
+      productDir: "/tmp/my-product",
+      results: [
+        {
+          name: "api",
+          failed: true,
+          skipped: false,
+          suites: [
+            {
+              name: "health",
+              type: "integration",
+              framework: "k6",
+              files: [
+                { path: "tests/api/integration/a.int.testkit.ts", status: "passed" },
+                { path: "tests/api/integration/b.int.testkit.ts", status: "failed" },
+              ],
+            },
+          ],
+        },
+      ],
+      suiteType: "int",
+      suiteNames: ["health"],
+      fileNames: ["tests/api/integration/b.int.testkit.ts"],
+      framework: "default",
+      shard: null,
+      serviceFilter: "api",
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc123",
+        },
+        testkitVersion: "0.1.20",
+      },
+    });
+
+    expect(status).toEqual({
+      schemaVersion: 1,
+      source: "testkit",
+      notice: "Generated file. Do not edit manually.",
+      product: {
+        name: "my-product",
+      },
+      git: {
+        branch: "main",
+        commitSha: "abc123",
+      },
+      testkitVersion: "0.1.20",
+      scope: {
+        suiteType: "int",
+        suiteNames: ["health"],
+        fileNames: ["tests/api/integration/b.int.testkit.ts"],
+        framework: "default",
+        shard: null,
+        serviceFilter: "api",
+        isFullRun: false,
+      },
+      summary: {
+        services: {
+          total: 1,
+          passed: 0,
+          failed: 1,
+        },
+        tests: {
+          total: 2,
+          passed: 1,
+          failed: 1,
+          notRun: 0,
+        },
+      },
+      tests: [
+        {
+          service: "api",
+          type: "integration",
+          path: "tests/api/integration/a.int.testkit.ts",
+          status: "passed",
+        },
+        {
+          service: "api",
+          type: "integration",
+          path: "tests/api/integration/b.int.testkit.ts",
+          status: "failed",
+        },
+      ],
+    });
+  });
+
+  it("marks unfiltered status artifacts as full runs", () => {
+    const status = buildStatusArtifact({
+      productDir: "/tmp/my-product",
+      results: [],
+      suiteType: "all",
+      suiteNames: [],
+      fileNames: [],
+      framework: "all",
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc123",
+        },
+        testkitVersion: "0.1.20",
+      },
+    });
+
+    expect(status.scope.isFullRun).toBe(true);
+  });
+});