@elench/testkit 0.1.54 → 0.1.56

package/lib/runner/orchestrator.mjs

@@ -14,7 +14,7 @@ import {
   recordTaskOutcome,
   summarizeDbBackend,
 } from "./results.mjs";
-import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { buildLiveRunArtifact, buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
 import {
   applyKnownFailureIssueValidationToArtifacts,
   applyKnownFailuresToArtifacts,
@@ -29,10 +29,12 @@ import {
   loadTimings,
   resetResultArtifacts,
   saveTimings,
+  writeLiveRunArtifact,
   writeRunArtifact,
   writeStatusArtifact,
 } from "./artifacts.mjs";
 import { createRunLogRegistry } from "./logs.mjs";
+import { createSetupOperationRegistry } from "./setup-operations.mjs";
 import {
   cleanupRunById,
   cleanupRuns,
@@ -72,6 +74,9 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   );
   const reporter = opts.reporter || null;
   const logRegistry = createRunLogRegistry(productDir);
+  let workerCount = 0;
+  let runtimeInstanceCount = 0;
+  let runtimeStats = [];
   const requestedFiles = opts.fileNames || [];
   if (requestedFiles.length > 0) {
     const unmatchedFiles = findUnmatchedRequestedFiles(
@@ -120,10 +125,41 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     reporter
   );
   const trackers = buildServiceTrackers(servicePlans, startedAt);
+  const writeLiveSnapshot = () => {
+    const now = Date.now();
+    const partialResults = configs.map((config) =>
+      finalizeServiceResult(trackers.get(config.name), startedAt, now)
+    );
+    writeLiveRunArtifact(
+      productDir,
+      buildLiveRunArtifact({
+        productDir,
+        results: partialResults,
+        startedAt,
+        updatedAt: now,
+        execution,
+        workerCount,
+        runtimeInstanceCount,
+        runtimeStats,
+        typeValues,
+        suiteSelectors,
+        fileNames: requestedFiles,
+        shard: opts.shard || null,
+        serviceFilter: opts.serviceFilter || null,
+        scenarioSeed: opts.scenarioSeed || null,
+        metadata,
+        summarizeDbBackend,
+        serviceLogs: logRegistry.listServiceLogs(),
+        setupLogs: logRegistry.listSetupLogs(),
+        setupOperations: setupRegistry.listOperations(),
+      })
+    );
+  };
+  const setupRegistry = createSetupOperationRegistry({
+    logRegistry,
+    onChange: writeLiveSnapshot,
+  });
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
-  let workerCount = 0;
-  let runtimeInstanceCount = 0;
-  let runtimeStats = [];
   let exitCode = 0;
   const lifecycle = createRunLifecycle(productDir);
   lifecycle.markRunning();
@@ -131,12 +167,16 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   let results = [];
   let finishedAt = Date.now();
   let knownFailureIssueValidation = null;
+  writeLiveSnapshot();
 
   try {
     if (executedPlans.length > 0) {
       const timings = loadTimings(productDir);
       const graphs = buildRuntimeGraphs(executedPlans);
       const queue = buildTaskQueue(executedPlans, graphs, timings);
+      for (const task of queue) {
+        task.scenarioSeed = opts.scenarioSeed || null;
+      }
       workerCount = Math.max(1, Math.min(execution.workers, queue.length));
       runtimeInstanceCount = graphs.reduce((sum, graph) => sum + graph.instanceCount, 0);
       const workers = Array.from({ length: workerCount }, (_unused, index) =>
@@ -149,6 +189,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
         runtimeOptions: {
           reporter,
           logRegistry,
+          setupRegistry,
         },
       });
       const timingUpdates = [];
@@ -164,8 +205,14 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
               timingUpdates,
               lifecycle,
               claimNextTask,
-              recordTaskOutcome,
-              recordGraphError,
+              (allTrackers, task, outcome, now) => {
+                recordTaskOutcome(allTrackers, task, outcome, now);
+                writeLiveSnapshot();
+              },
+              (allTrackers, graph, message, now) => {
+                recordGraphError(allTrackers, graph, message, now);
+                writeLiveSnapshot();
+              },
               reporter
             )
           )
@@ -177,9 +224,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
             for (const tracker of trackers.values()) {
               if (!tracker.skipped) addTrackerError(tracker, message);
             }
+            writeLiveSnapshot();
           }
         }
         runtimeStats = runtimeManager.getStats();
+        writeLiveSnapshot();
       } finally {
         await runtimeManager.cleanupAll();
       }
@@ -202,12 +251,15 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       runtimeStats,
       typeValues,
       suiteSelectors,
-      fileNames: requestedFiles,
-      shard: opts.shard || null,
-      serviceFilter: opts.serviceFilter || null,
-      metadata,
+            fileNames: requestedFiles,
+            shard: opts.shard || null,
+            serviceFilter: opts.serviceFilter || null,
+            scenarioSeed: opts.scenarioSeed || null,
+            metadata,
       summarizeDbBackend,
       serviceLogs: logRegistry.listServiceLogs(),
+      setupLogs: logRegistry.listSetupLogs(),
+      setupOperations: setupRegistry.listOperations(),
     });
     const statusArtifact = opts.writeStatus
       ? buildStatusArtifact({
@@ -215,10 +267,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
           results,
           typeValues,
           suiteSelectors,
-          fileNames: requestedFiles,
-          shard: opts.shard || null,
-          serviceFilter: opts.serviceFilter || null,
-          metadata,
+      fileNames: requestedFiles,
+      shard: opts.shard || null,
+      serviceFilter: opts.serviceFilter || null,
+      scenarioSeed: opts.scenarioSeed || null,
+      metadata,
         })
       : null;
     const enrichedArtifacts = applyKnownFailuresToArtifacts(

package/lib/runner/planning.mjs

@@ -5,7 +5,7 @@ import {
   suiteSelectionType,
 } from "./suite-selection.mjs";
 
-const TYPE_ORDER = ["dal", "integration", "e2e", "load"];
+const TYPE_ORDER = ["dal", "integration", "e2e", "scenario", "load"];
 
 export function taskNeedsLocalRuntime(task) {
   return task.type !== "dal";

package/lib/runner/reporting.mjs

@@ -8,6 +8,7 @@ export function buildStatusArtifact({
   fileNames,
   shard,
   serviceFilter,
+  scenarioSeed,
   metadata,
 }) {
   const executedResults = results.filter((result) => !result.skipped);
@@ -68,6 +69,7 @@ export function buildStatusArtifact({
     fileNames: [...(fileNames || [])].sort(),
     shard: shard || null,
     serviceFilter: serviceFilter || null,
+    scenarioSeed: scenarioSeed || null,
   };
   scope.isFullRun =
     scope.types.length === 1 &&
@@ -109,9 +111,13 @@ export function buildRunArtifact({
   fileNames,
   shard,
   serviceFilter,
+  scenarioSeed,
   metadata,
   summarizeDbBackend,
   serviceLogs = [],
+  setupLogs = [],
+  setupOperations = [],
+  runStatus = null,
 }) {
   const executed = results.filter((result) => !result.skipped);
   const effectivelySkippedServices = executed.filter(isEffectivelySkippedService);
@@ -128,7 +134,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 7,
+    schemaVersion: 8,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -138,7 +144,7 @@ export function buildRunArtifact({
     git: metadata.git,
     host: metadata.host,
     run: {
-      status: failedServices.length > 0 ? "failed" : "passed",
+      status: runStatus || (failedServices.length > 0 ? "failed" : "passed"),
       startedAt: new Date(startedAt).toISOString(),
       finishedAt: new Date(finishedAt).toISOString(),
       durationMs: finishedAt - startedAt,
@@ -153,6 +159,7 @@ export function buildRunArtifact({
       fileNames,
       shard,
       serviceFilter,
+      scenarioSeed: scenarioSeed || null,
       testkitVersion: metadata.testkitVersion,
     },
     summary: {
@@ -179,6 +186,10 @@ export function buildRunArtifact({
     },
     logs: {
       services: serviceLogs,
+      setup: setupLogs,
+    },
+    setup: {
+      operations: setupOperations,
     },
     services: results.map((result) => ({
       name: result.name,
@@ -203,6 +214,51 @@ export function buildRunArtifact({
   };
 }
 
+export function buildLiveRunArtifact({
+  productDir,
+  results,
+  startedAt,
+  updatedAt,
+  execution,
+  workerCount,
+  runtimeInstanceCount,
+  runtimeStats,
+  typeValues,
+  suiteSelectors,
+  fileNames,
+  shard,
+  serviceFilter,
+  scenarioSeed,
+  metadata,
+  summarizeDbBackend,
+  serviceLogs = [],
+  setupLogs = [],
+  setupOperations = [],
+}) {
+  return buildRunArtifact({
+    productDir,
+    results,
+    startedAt,
+    finishedAt: updatedAt,
+    execution,
+    workerCount,
+    runtimeInstanceCount,
+    runtimeStats,
+    typeValues,
+    suiteSelectors,
+    fileNames,
+    shard,
+    serviceFilter,
+    scenarioSeed,
+    metadata,
+    summarizeDbBackend,
+    serviceLogs,
+    setupLogs,
+    setupOperations,
+    runStatus: "running",
+  });
+}
+
 function isEffectivelySkippedService(result) {
   return (
     !result.skipped &&

package/lib/runner/reporting.test.mjs

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { buildLiveRunArtifact, buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
 
 describe("runner reporting", () => {
   it("builds run artifacts", () => {
@@ -62,6 +62,7 @@ describe("runner reporting", () => {
       fileNames: [],
       shard: null,
       serviceFilter: null,
+      scenarioSeed: "demo-seed",
       metadata: {
         git: {
           branch: "main",
@@ -78,12 +79,13 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(7);
+    expect(artifact.schemaVersion).toBe(8);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
       workerCount: 1,
       runtimeInstanceCount: 2,
+      scenarioSeed: "demo-seed",
     });
     expect(artifact.summary.services).toEqual({
       total: 1,
@@ -109,7 +111,85 @@ describe("runner reporting", () => {
     expect(artifact.services[0].totalTaskDurationMs).toBe(2400);
     expect(artifact.logs).toEqual({
       services: [],
+      setup: [],
     });
+    expect(artifact.setup).toEqual({
+      operations: [],
+    });
+  });
+
+  it("builds live run artifacts with running status and setup details", () => {
+    const artifact = buildLiveRunArtifact({
+      productDir: "/tmp/my-product",
+      results: [],
+      startedAt: 1_000,
+      updatedAt: 2_000,
+      execution: {
+        workers: 2,
+        fileTimeoutSeconds: 60,
+      },
+      workerCount: 1,
+      runtimeInstanceCount: 1,
+      runtimeStats: [],
+      typeValues: ["int"],
+      suiteSelectors: [],
+      fileNames: [],
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc",
+          repoRoot: "/tmp",
+        },
+        host: {
+          hostname: "local",
+          username: "dev",
+        },
+        testkitVersion: "0.1.54",
+      },
+      summarizeDbBackend: () => "local",
+      serviceLogs: [],
+      setupLogs: [
+        {
+          serviceName: "api",
+          runtimeLabel: "api",
+          stage: "runtime:prepare",
+          path: ".testkit/results/setup/api__api__runtime-prepare.log",
+        },
+      ],
+      setupOperations: [
+        {
+          id: "setup-1",
+          serviceName: "api",
+          runtimeLabel: "api",
+          stage: "runtime:prepare",
+          kind: "runtime-prepare",
+          summary: "runtime prepare",
+          parentId: null,
+          status: "running",
+          startedAt: "1970-01-01T00:00:01.000Z",
+          finishedAt: null,
+          durationMs: null,
+          error: null,
+          logRef: {
+            path: ".testkit/results/setup/api__api__runtime-prepare.log",
+            stage: "runtime:prepare",
+          },
+        },
+      ],
+    });
+
+    expect(artifact.run.status).toBe("running");
+    expect(artifact.logs.setup).toEqual([
+      {
+        serviceName: "api",
+        runtimeLabel: "api",
+        stage: "runtime:prepare",
+        path: ".testkit/results/setup/api__api__runtime-prepare.log",
+      },
+    ]);
+    expect(artifact.setup.operations).toHaveLength(1);
   });
 
   it("builds deterministic status artifacts", () => {
@@ -142,6 +222,7 @@ describe("runner reporting", () => {
       fileNames: ["tests/api/integration/b.int.testkit.ts"],
       shard: null,
       serviceFilter: "api",
+      scenarioSeed: "demo-seed",
       metadata: {
         git: {
           branch: "main",
@@ -169,6 +250,7 @@ describe("runner reporting", () => {
         fileNames: ["tests/api/integration/b.int.testkit.ts"],
         shard: null,
         serviceFilter: "api",
+        scenarioSeed: "demo-seed",
         isFullRun: false,
       },
       summary: {
@@ -213,6 +295,7 @@ describe("runner reporting", () => {
       fileNames: [],
       shard: null,
       serviceFilter: null,
+      scenarioSeed: null,
       metadata: {
         git: {
           branch: "main",

package/lib/runner/runtime-contexts.mjs

@@ -39,7 +39,7 @@ export async function ensureRuntimeInstanceReady(context, task, lifecycle, optio
   if (!context.prepared) {
     if (!context.preparationPromise) {
       context.preparationPromise = (async () => {
-        await prepareDatabases(context.runtimeConfigs);
+        await prepareDatabases(context.runtimeConfigs, options);
         await prepareRuntimeServices(context.runtimeConfigs, options);
         context.prepared = true;
       })().finally(() => {
@@ -80,8 +80,8 @@ export async function cleanupRuntimeInstanceContext(context, lifecycle) {
   await deactivateRuntimeInstanceContext(context, lifecycle);
 }
 
-export async function prepareDatabases(runtimeConfigs) {
+export async function prepareDatabases(runtimeConfigs, options = {}) {
   for (const config of runtimeConfigs) {
-    await prepareDatabaseRuntime(config);
+    await prepareDatabaseRuntime(config, options);
   }
 }

package/lib/runner/runtime-preparation.mjs

@@ -28,6 +28,12 @@ export async function prepareRuntimeService(config, options = {}) {
   const manifestPath = path.join(prepareDir, MANIFEST_FILE);
   const existingManifest = readPrepareManifest(manifestPath);
   if (existingManifest?.fingerprint === fingerprint) {
+    options.setupRegistry?.recordCached({
+      config,
+      stage: "runtime:prepare",
+      kind: "runtime-prepare",
+      summary: "runtime prepare cache hit",
+    });
     return;
   }
 
@@ -42,6 +48,14 @@ export async function prepareRuntimeService(config, options = {}) {
     env.DATABASE_URL = databaseUrl;
   }
 
+  const prepareOperation = options.setupRegistry?.start({
+    config,
+    stage: "runtime:prepare",
+    kind: "runtime-prepare",
+    summary: "runtime prepare",
+    recordLog: false,
+  });
+
   try {
     await announceResolvedToolchain(
       config,
@@ -54,7 +68,16 @@ export async function prepareRuntimeService(config, options = {}) {
       env,
       labelPrefix: "runtime:prepare",
       reporter: options.reporter,
+      setupRegistry: options.setupRegistry || null,
+      parentOperation: prepareOperation,
     });
+    const finished = prepareOperation
+      ? options.setupRegistry.finish(prepareOperation, {
+          status: "passed",
+          summary: "runtime prepare",
+        })
+      : null;
+    if (finished) options.reporter?.setupOperationFinished?.(finished);
     writePrepareManifest(manifestPath, {
       fingerprint,
       preparedAt: new Date().toISOString(),
@@ -62,6 +85,14 @@ export async function prepareRuntimeService(config, options = {}) {
       serviceName: config.name,
     });
   } catch (error) {
+    const finished = prepareOperation
+      ? options.setupRegistry.finish(prepareOperation, {
+          status: "failed",
+          summary: "runtime prepare",
+          error: error?.message || error,
+        })
+      : null;
+    if (finished) options.reporter?.setupOperationFinished?.(finished);
     fs.rmSync(prepareDir, { recursive: true, force: true });
     throw error;
   }

package/lib/runner/setup-operations.mjs

@@ -0,0 +1,115 @@
+export function createSetupOperationRegistry({ logRegistry = null, onChange = null } = {}) {
+  const operations = [];
+  const byId = new Map();
+  let nextId = 1;
+
+  function emitChange() {
+    onChange?.(listOperations());
+  }
+
+  function start({
+    config,
+    stage,
+    kind = "setup",
+    summary = null,
+    parentId = null,
+    recordLog = true,
+  }) {
+    const startedAt = new Date().toISOString();
+    const logRecord = recordLog ? logRegistry?.ensureSetupLogRecord(config, stage) || null : null;
+    const operation = {
+      id: `setup-${nextId++}`,
+      serviceName: config.name,
+      runtimeLabel: config.runtimeLabel || config.name,
+      stage,
+      kind,
+      summary,
+      parentId,
+      status: "running",
+      startedAt,
+      finishedAt: null,
+      durationMs: null,
+      error: null,
+      logRef: logRecord
+        ? {
+            path: logRecord.path,
+            stage: logRecord.stage,
+          }
+        : null,
+      _logRecord: logRecord,
+    };
+    operations.push(operation);
+    byId.set(operation.id, operation);
+    emitChange();
+    return operation;
+  }
+
+  function finish(target, { status = "passed", summary, error = null } = {}) {
+    const operation = typeof target === "string" ? byId.get(target) : target;
+    if (!operation) return null;
+    const finishedAt = new Date().toISOString();
+    operation.finishedAt = finishedAt;
+    operation.durationMs = Math.max(
+      0,
+      Date.parse(finishedAt) - Date.parse(operation.startedAt || finishedAt)
+    );
+    operation.status = status;
+    if (summary !== undefined) operation.summary = summary;
+    if (error !== null) operation.error = String(error);
+    emitChange();
+    return operation;
+  }
+
+  function recordCached({ config, stage, kind = "setup", summary = null, parentId = null }) {
+    const now = new Date().toISOString();
+    const operation = {
+      id: `setup-${nextId++}`,
+      serviceName: config.name,
+      runtimeLabel: config.runtimeLabel || config.name,
+      stage,
+      kind,
+      summary,
+      parentId,
+      status: "cached",
+      startedAt: now,
+      finishedAt: now,
+      durationMs: 0,
+      error: null,
+      logRef: null,
+      _logRecord: null,
+    };
+    operations.push(operation);
+    byId.set(operation.id, operation);
+    emitChange();
+    return operation;
+  }
+
+  function listOperations() {
+    return operations.map(cloneOperation);
+  }
+
+  return {
+    start,
+    finish,
+    recordCached,
+    listOperations,
+  };
+}
+
+function cloneOperation(operation) {
+  return {
+    id: operation.id,
+    serviceName: operation.serviceName,
+    runtimeLabel: operation.runtimeLabel,
+    stage: operation.stage,
+    kind: operation.kind,
+    summary: operation.summary,
+    parentId: operation.parentId,
+    status: operation.status,
+    startedAt: operation.startedAt,
+    finishedAt: operation.finishedAt,
+    durationMs: operation.durationMs,
+    error: operation.error,
+    logRef: operation.logRef,
+  };
+}