@elench/testkit 0.1.83 → 0.1.84

package/lib/runner/reporting.test.mjs

@@ -1,310 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { buildLiveRunArtifact, buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
-
-describe("runner reporting", () => {
-  it("builds run artifacts", () => {
-    const results = [
-      {
-        name: "api",
-        failed: false,
-        skipped: false,
-        suiteCount: 1,
-        completedSuiteCount: 1,
-        skippedSuiteCount: 0,
-        failedSuiteCount: 0,
-        totalFileCount: 3,
-        completedFileCount: 3,
-        passedFileCount: 3,
-        failedFileCount: 0,
-        skippedFileCount: 0,
-        notRunFileCount: 0,
-        durationMs: 1200,
-        totalTaskDurationMs: 2400,
-        dbBackend: "local",
-        suites: [],
-        errors: [],
-      },
-      {
-        name: "frontend",
-        failed: false,
-        skipped: true,
-        suiteCount: 0,
-        completedSuiteCount: 0,
-        skippedSuiteCount: 0,
-        failedSuiteCount: 0,
-        totalFileCount: 0,
-        completedFileCount: 0,
-        passedFileCount: 0,
-        failedFileCount: 0,
-        skippedFileCount: 0,
-        notRunFileCount: 0,
-        durationMs: 0,
-        totalTaskDurationMs: 0,
-        dbBackend: null,
-        suites: [],
-        errors: [],
-      },
-    ];
-
-    const artifact = buildRunArtifact({
-      productDir: "/tmp/my-product",
-      results,
-      startedAt: 1000,
-      finishedAt: 4000,
-      execution: {
-        workers: 2,
-        fileTimeoutSeconds: 60,
-      },
-      workerCount: 1,
-      runtimeInstanceCount: 2,
-      typeValues: ["all"],
-      suiteSelectors: [],
-      fileNames: [],
-      shard: null,
-      serviceFilter: null,
-      scenarioSeed: "demo-seed",
-      metadata: {
-        git: {
-          branch: "main",
-          commitSha: "abc",
-          repoRoot: "/tmp",
-        },
-        host: {
-          hostname: "local",
-          username: "dev",
-        },
-        testkitVersion: "0.1.17",
-      },
-      summarizeDbBackend: () => "local",
-    });
-
-    expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(9);
-    expect(artifact.run).toMatchObject({
-      workers: 2,
-      fileTimeoutSeconds: 60,
-      workerCount: 1,
-      runtimeInstanceCount: 2,
-      scenarioSeed: "demo-seed",
-    });
-    expect(artifact.summary.services).toEqual({
-      total: 1,
-      passed: 1,
-      failed: 0,
-      skipped: 0,
-    });
-    expect(artifact.summary.suites).toEqual({
-      total: 1,
-      completed: 1,
-      passed: 1,
-      failed: 0,
-      skipped: 0,
-    });
-    expect(artifact.summary.files).toEqual({
-      total: 3,
-      passed: 3,
-      failed: 0,
-      skipped: 0,
-      notRun: 0,
-    });
-    expect(artifact.services[0].durationMs).toBe(1200);
-    expect(artifact.services[0].totalTaskDurationMs).toBe(2400);
-    expect(artifact.logs).toEqual({
-      services: [],
-      setup: [],
-    });
-    expect(artifact.setup).toEqual({
-      operations: [],
-    });
-  });
-
-  it("builds live run artifacts with running status and setup details", () => {
-    const artifact = buildLiveRunArtifact({
-      productDir: "/tmp/my-product",
-      results: [],
-      startedAt: 1_000,
-      updatedAt: 2_000,
-      execution: {
-        workers: 2,
-        fileTimeoutSeconds: 60,
-      },
-      workerCount: 1,
-      runtimeInstanceCount: 1,
-      runtimeStats: [],
-      typeValues: ["int"],
-      suiteSelectors: [],
-      fileNames: [],
-      shard: null,
-      serviceFilter: null,
-      metadata: {
-        git: {
-          branch: "main",
-          commitSha: "abc",
-          repoRoot: "/tmp",
-        },
-        host: {
-          hostname: "local",
-          username: "dev",
-        },
-        testkitVersion: "0.1.54",
-      },
-      summarizeDbBackend: () => "local",
-      serviceLogs: [],
-      setupLogs: [
-        {
-          serviceName: "api",
-          runtimeLabel: "api",
-          stage: "runtime:prepare",
-          path: ".testkit/results/setup/api__api__runtime-prepare.log",
-        },
-      ],
-      setupOperations: [
-        {
-          id: "setup-1",
-          serviceName: "api",
-          runtimeLabel: "api",
-          stage: "runtime:prepare",
-          kind: "runtime-prepare",
-          summary: "runtime prepare",
-          parentId: null,
-          status: "running",
-          startedAt: "1970-01-01T00:00:01.000Z",
-          finishedAt: null,
-          durationMs: null,
-          error: null,
-          logRef: {
-            path: ".testkit/results/setup/api__api__runtime-prepare.log",
-            stage: "runtime:prepare",
-          },
-        },
-      ],
-    });
-
-    expect(artifact.run.status).toBe("running");
-    expect(artifact.logs.setup).toEqual([
-      {
-        serviceName: "api",
-        runtimeLabel: "api",
-        stage: "runtime:prepare",
-        path: ".testkit/results/setup/api__api__runtime-prepare.log",
-      },
-    ]);
-    expect(artifact.setup.operations).toHaveLength(1);
-  });
-
-  it("builds deterministic status artifacts", () => {
-    const status = buildStatusArtifact({
-      productDir: "/tmp/my-product",
-      results: [
-        {
-          name: "api",
-          failed: true,
-          skipped: false,
-          suites: [
-            {
-              name: "health",
-              type: "int",
-              framework: "k6",
-              files: [
-                { path: "tests/api/integration/a.int.testkit.ts", status: "passed" },
-                {
-                  path: "tests/api/integration/b.int.testkit.ts",
-                  status: "skipped",
-                  reason: "Billing is stubbed",
-                },
-              ],
-            },
-          ],
-        },
-      ],
-      typeValues: ["int"],
-      suiteSelectors: [{ kind: "plain", name: "health", raw: "health" }],
-      fileNames: ["tests/api/integration/b.int.testkit.ts"],
-      shard: null,
-      serviceFilter: "api",
-      scenarioSeed: "demo-seed",
-      metadata: {
-        git: {
-          branch: "main",
-          commitSha: "abc123",
-        },
-        testkitVersion: "0.1.20",
-      },
-    });
-
-    expect(status).toEqual({
-      schemaVersion: 7,
-      source: "testkit",
-      notice: "Generated file. Do not edit manually.",
-      product: {
-        name: "my-product",
-      },
-      git: {
-        branch: "main",
-        commitSha: "abc123",
-      },
-      testkitVersion: "0.1.20",
-      scope: {
-        types: ["int"],
-        suiteSelectors: ["health"],
-        fileNames: ["tests/api/integration/b.int.testkit.ts"],
-        shard: null,
-        serviceFilter: "api",
-        scenarioSeed: "demo-seed",
-        isFullRun: false,
-      },
-      summary: {
-        services: {
-          total: 1,
-          passed: 0,
-          failed: 1,
-          skipped: 0,
-        },
-        tests: {
-          total: 2,
-          passed: 1,
-          failed: 0,
-          skipped: 1,
-          notRun: 0,
-        },
-      },
-      tests: [
-        {
-          service: "api",
-          type: "int",
-          path: "tests/api/integration/a.int.testkit.ts",
-          status: "passed",
-        },
-        {
-          service: "api",
-          type: "int",
-          path: "tests/api/integration/b.int.testkit.ts",
-          status: "skipped",
-          reason: "Billing is stubbed",
-        },
-      ],
-    });
-  });
-
-  it("marks unfiltered status artifacts as full runs", () => {
-    const status = buildStatusArtifact({
-      productDir: "/tmp/my-product",
-      results: [],
-      typeValues: ["all"],
-      suiteSelectors: [],
-      fileNames: [],
-      shard: null,
-      serviceFilter: null,
-      scenarioSeed: null,
-      metadata: {
-        git: {
-          branch: "main",
-          commitSha: "abc123",
-        },
-        testkitVersion: "0.1.20",
-      },
-    });
-
-    expect(status.scope.isFullRun).toBe(true);
-  });
-});

package/lib/runner/results.test.mjs

@@ -1,376 +0,0 @@
-import { describe, expect, it } from "vitest";
-import {
-  addTrackerError,
-  buildServiceTrackers,
-  finalizeServiceResult,
-  recordGraphError,
-  recordTaskOutcome,
-  summarizeDbBackend,
-} from "./results.mjs";
-
-describe("runner results", () => {
-  it("tracks task outcomes and graph errors", () => {
-    const trackers = buildServiceTrackers(
-      [
-        {
-          skipped: false,
-          config: {
-            name: "api",
-            testkit: {
-              database: {
-                selectedBackend: "local",
-              },
-            },
-          },
-          suites: [
-            {
-              name: "health",
-              type: "integration",
-              framework: "k6",
-              files: ["tests/health.js"],
-              orderIndex: 0,
-            },
-          ],
-        },
-      ],
-      1000
-    );
-
-    recordTaskOutcome(
-      trackers,
-      {
-        serviceName: "api",
-        suiteKey: "integration:health",
-        file: "tests/health.js",
-      },
-      {
-        failed: true,
-        durationMs: 250,
-        error: "boom",
-      },
-      1250
-    );
-
-    const tracker = trackers.get("api");
-    addTrackerError(tracker, "worker failed");
-    addTrackerError(tracker, "worker failed");
-    recordGraphError(trackers, { targetNames: ["api"] }, "graph failed", 1300);
-
-    const result = finalizeServiceResult(tracker, 1000, 1500);
-    expect(result.failed).toBe(true);
-    expect(result.failedSuiteCount).toBe(1);
-    expect(result.totalFileCount).toBe(1);
-    expect(result.failedFileCount).toBe(1);
-    expect(result.passedFileCount).toBe(0);
-    expect(result.errors).toEqual(["worker failed", "graph failed"]);
-    expect(result.suites[0].framework).toBe("default");
-    expect(result.suites[0].failedFileCount).toBe(1);
-    expect(result.suites[0].files).toEqual([
-      {
-        path: "tests/health.js",
-        failed: true,
-        status: "failed",
-        durationMs: 250,
-        error: "boom",
-        reason: null,
-      },
-    ]);
-  });
-
-  it("reports service duration as wall-clock time, not accumulated file time", () => {
-    const trackers = buildServiceTrackers(
-      [
-        {
-          skipped: false,
-          config: {
-            name: "frontend",
-            testkit: {
-              database: {
-                selectedBackend: null,
-              },
-            },
-          },
-          suites: [
-            {
-              name: "browser",
-              type: "e2e",
-              framework: "playwright",
-              files: ["tests/a.pw.testkit.ts", "tests/b.pw.testkit.ts"],
-              orderIndex: 0,
-            },
-          ],
-        },
-      ],
-      1000
-    );
-
-    recordTaskOutcome(
-      trackers,
-      {
-        serviceName: "frontend",
-        suiteKey: "e2e:browser",
-        file: "tests/a.pw.testkit.ts",
-      },
-      {
-        failed: false,
-        durationMs: 30_000,
-        startedAt: 1000,
-        finishedAt: 8000,
-        error: null,
-      },
-      8000
-    );
-    recordTaskOutcome(
-      trackers,
-      {
-        serviceName: "frontend",
-        suiteKey: "e2e:browser",
-        file: "tests/b.pw.testkit.ts",
-      },
-      {
-        failed: false,
-        durationMs: 45_000,
-        startedAt: 3000,
-        finishedAt: 9000,
-        error: null,
-      },
-      9000
-    );
-
-    const result = finalizeServiceResult(trackers.get("frontend"), 1000, 10_000);
-
-    expect(result.durationMs).toBe(8000);
-    expect(result.totalTaskDurationMs).toBe(75_000);
-    expect(result.suites[0].durationMs).toBe(75_000);
-  });
-
-  it("handles epoch-zero task timestamps when calculating wall-clock duration", () => {
-    const trackers = buildServiceTrackers(
-      [
-        {
-          skipped: false,
-          config: {
-            name: "frontend",
-            testkit: {
-              database: {
-                selectedBackend: null,
-              },
-            },
-          },
-          suites: [
-            {
-              name: "browser",
-              type: "e2e",
-              framework: "playwright",
-              files: ["tests/a.pw.testkit.ts"],
-              orderIndex: 0,
-            },
-          ],
-        },
-      ],
-      0
-    );
-
-    recordTaskOutcome(
-      trackers,
-      {
-        serviceName: "frontend",
-        suiteKey: "e2e:browser",
-        file: "tests/a.pw.testkit.ts",
-      },
-      {
-        failed: false,
-        durationMs: 3_000,
-        startedAt: 0,
-        finishedAt: 8_000,
-        error: null,
-      },
-      8_000
-    );
-
-    const result = finalizeServiceResult(trackers.get("frontend"), 0, 10_000);
-
-    expect(result.durationMs).toBe(8_000);
-    expect(result.totalTaskDurationMs).toBe(3_000);
-  });
-
-  it("counts config-skipped files and runtime-skipped files separately from passed work", () => {
-    const trackers = buildServiceTrackers(
-      [
-        {
-          skipped: false,
-          config: {
-            name: "api",
-            testkit: {
-              database: {
-                selectedBackend: null,
-              },
-            },
-          },
-          suites: [
-            {
-              name: "billing",
-              type: "integration",
-              framework: "k6",
-              files: ["tests/billing-live.int.testkit.ts"],
-              skippedFiles: [
-                {
-                  path: "tests/billing-stubbed.int.testkit.ts",
-                  reason: "Billing is stubbed",
-                },
-              ],
-              totalFileCount: 2,
-              orderIndex: 0,
-            },
-            {
-              name: "frontend-smoke",
-              type: "e2e",
-              framework: "playwright",
-              files: ["tests/frontend-smoke.pw.testkit.ts"],
-              orderIndex: 1,
-            },
-          ],
-        },
-      ],
-      1000
-    );
-
-    recordTaskOutcome(
-      trackers,
-      {
-        serviceName: "api",
-        suiteKey: "integration:billing",
-        file: "tests/billing-live.int.testkit.ts",
-      },
-      {
-        failed: false,
-        durationMs: 100,
-        error: null,
-      },
-      1100
-    );
-    recordTaskOutcome(
-      trackers,
-      {
-        serviceName: "api",
-        suiteKey: "e2e:frontend-smoke",
-        file: "tests/frontend-smoke.pw.testkit.ts",
-      },
-      {
-        failed: false,
-        status: "skipped",
-        reason: "Playwright test.skip()",
-        durationMs: 50,
-        error: null,
-      },
-      1150
-    );
-
-    const result = finalizeServiceResult(trackers.get("api"), 1000, 1200);
-
-    expect(result.failed).toBe(false);
-    expect(result.completedSuiteCount).toBe(2);
-    expect(result.skippedSuiteCount).toBe(1);
-    expect(result.totalFileCount).toBe(3);
-    expect(result.completedFileCount).toBe(1);
-    expect(result.passedFileCount).toBe(1);
-    expect(result.failedFileCount).toBe(0);
-    expect(result.skippedFileCount).toBe(2);
-    expect(result.notRunFileCount).toBe(0);
-    expect(result.suites[0].files).toEqual([
-      {
-        path: "tests/billing-live.int.testkit.ts",
-        failed: false,
-        status: "passed",
-        durationMs: 100,
-        error: null,
-        reason: null,
-      },
-      {
-        path: "tests/billing-stubbed.int.testkit.ts",
-        failed: false,
-        status: "skipped",
-        durationMs: 0,
-        error: null,
-        reason: "Billing is stubbed",
-      },
-    ]);
-    expect(result.suites[1].files).toEqual([
-      {
-        path: "tests/frontend-smoke.pw.testkit.ts",
-        failed: false,
-        status: "skipped",
-        durationMs: 50,
-        error: null,
-        reason: "Playwright test.skip()",
-      },
-    ]);
-  });
-
-  it("preserves not-run task outcomes in suite and service summaries", () => {
-    const trackers = buildServiceTrackers(
-      [
-        {
-          skipped: false,
-          config: {
-            name: "api",
-            testkit: {
-              database: {
-                selectedBackend: null,
-              },
-            },
-          },
-          suites: [
-            {
-              name: "health",
-              type: "integration",
-              framework: "k6",
-              files: ["tests/a.int.testkit.ts", "tests/b.int.testkit.ts"],
-              orderIndex: 0,
-            },
-          ],
-        },
-      ],
-      1000
-    );
-
-    recordTaskOutcome(
-      trackers,
-      {
-        serviceName: "api",
-        suiteKey: "integration:health",
-        file: "tests/a.int.testkit.ts",
-      },
-      {
-        failed: false,
-        status: "not_run",
-        reason: "Graph initialization failed",
-        durationMs: 0,
-        error: null,
-      },
-      1000
-    );
-
-    const result = finalizeServiceResult(trackers.get("api"), 1000, 1200);
-
-    expect(result.completedFileCount).toBe(0);
-    expect(result.passedFileCount).toBe(0);
-    expect(result.failedFileCount).toBe(0);
-    expect(result.skippedFileCount).toBe(0);
-    expect(result.notRunFileCount).toBe(2);
-    expect(result.suites[0].files).toContainEqual({
-      path: "tests/a.int.testkit.ts",
-      failed: false,
-      status: "not_run",
-      durationMs: 0,
-      error: null,
-      reason: "Graph initialization failed",
-    });
-  });
-
-  it("summarizes mixed db backends", () => {
-    expect(
-      summarizeDbBackend([{ dbBackend: "local" }, { dbBackend: "neon" }])
-    ).toBe("mixed");
-  });
-});