@elench/testkit 0.1.54 → 0.1.55

package/lib/database/template-steps.mjs

@@ -6,8 +6,9 @@ import {
   collectConfiguredInputs,
   runConfiguredSteps,
 } from "../runner/template-steps.mjs";
+import { captureOutput } from "../runner/processes.mjs";
 
-export async function runTemplateStage(config, stageName, databaseUrl) {
+export async function runTemplateStage(config, stageName, databaseUrl, options = {}) {
   const steps = config.testkit.database?.template?.[stageName] || [];
   if (steps.length === 0) return;
 
@@ -21,6 +22,9 @@ export async function runTemplateStage(config, stageName, databaseUrl) {
     steps,
     env,
     labelPrefix: `template:${stageName}`,
+    reporter: options.reporter || null,
+    setupRegistry: options.setupRegistry || null,
+    parentOperation: options.parentOperation || null,
   });
 }
 
@@ -32,12 +36,12 @@ export function collectTemplateInputs(productDir, template = {}) {
   });
 }
 
-export async function captureTemplateSnapshot(config, outputPath, databaseUrl) {
+export async function captureTemplateSnapshot(config, outputPath, databaseUrl, options = {}) {
   const templateDbUrl = databaseUrl;
   const absoluteOutputPath = path.resolve(config.productDir, outputPath);
   fs.mkdirSync(path.dirname(absoluteOutputPath), { recursive: true });
 
-  await execa(
+  const child = execa(
     "pg_dump",
     [
       "--schema-only",
@@ -53,19 +57,54 @@ export async function captureTemplateSnapshot(config, outputPath, databaseUrl) {
         ...buildExecutionEnv(config, {}, process.env),
         DATABASE_URL: templateDbUrl,
       },
-      stdio: "inherit",
+      stdout: "pipe",
+      stderr: "pipe",
+      reject: false,
     }
   );
+  const liveWriter =
+    options.reporter?.outputMode === "debug"
+      ? (line) => options.reporter.writeDebugLine?.(line)
+      : null;
+  const logRecord = options.logRecord || null;
+  const drains = [
+    captureOutput(child.stdout, {
+      livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
+      liveWriter,
+      onLine(line) {
+        if (logRecord) logRecord.stream.write(`${new Date().toISOString()} [stdout] ${line}\n`);
+      },
+    }),
+    captureOutput(child.stderr, {
+      livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
+      liveWriter,
+      onLine(line) {
+        if (logRecord) logRecord.stream.write(`${new Date().toISOString()} [stderr] ${line}\n`);
+      },
+    }),
+  ];
+  const result = await child;
+  await Promise.all(drains);
+  if (result.exitCode !== 0) {
+    throw new Error(result.shortMessage || result.stderr || result.stdout || "pg_dump failed");
+  }
 
   sanitizeSnapshotFile(absoluteOutputPath);
   return absoluteOutputPath;
 }
 
-function sanitizeSnapshotFile(filePath) {
+export function sanitizeSnapshotFile(filePath) {
   const dump = fs.readFileSync(filePath, "utf8");
   const sanitized = dump
     .split("\n")
-    .filter((line) => line.trim() !== "SET transaction_timeout = 0;")
+    .filter((line) => {
+      const trimmed = line.trim();
+      return (
+        trimmed !== "SET transaction_timeout = 0;" &&
+        !trimmed.startsWith("\\restrict ") &&
+        !trimmed.startsWith("\\unrestrict ")
+      );
+    })
     .join("\n");
 
   if (sanitized !== dump) {

package/lib/database/template-steps.test.mjs

@@ -0,0 +1,43 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { sanitizeSnapshotFile } from "./template-steps.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+describe("template snapshot sanitization", () => {
+  it("removes volatile pg_dump control lines", () => {
+    const dir = makeTempDir("testkit-template-snapshot-");
+    const filePath = path.join(dir, "schema.sql");
+    fs.writeFileSync(
+      filePath,
+      [
+        "SET statement_timeout = 0;",
+        "SET transaction_timeout = 0;",
+        "\\restrict abc123",
+        "CREATE TABLE public.widgets (id integer);",
+        "\\unrestrict abc123",
+        "",
+      ].join("\n")
+    );
+
+    sanitizeSnapshotFile(filePath);
+
+    expect(fs.readFileSync(filePath, "utf8")).toBe(
+      ["SET statement_timeout = 0;", "CREATE TABLE public.widgets (id integer);", ""].join("\n")
+    );
+  });
+});

package/lib/runner/artifacts.mjs

@@ -9,11 +9,20 @@ import {
 const TIMINGS_FILENAME = "timings.json";
 const RESULT_ARTIFACTS_DIRNAME = "artifacts";
 const RESULT_LOGS_DIRNAME = "logs";
+const RESULT_SETUP_DIRNAME = "setup";
+const LIVE_ARTIFACT_FILENAME = "live.json";
 
 export function writeRunArtifact(productDir, artifact) {
   const resultsDir = path.join(productDir, ".testkit", "results");
   fs.mkdirSync(resultsDir, { recursive: true });
   fs.writeFileSync(path.join(resultsDir, "latest.json"), JSON.stringify(artifact, null, 2));
+  fs.rmSync(path.join(resultsDir, LIVE_ARTIFACT_FILENAME), { force: true });
+}
+
+export function writeLiveRunArtifact(productDir, artifact) {
+  const resultsDir = path.join(productDir, ".testkit", "results");
+  fs.mkdirSync(resultsDir, { recursive: true });
+  fs.writeFileSync(path.join(resultsDir, LIVE_ARTIFACT_FILENAME), JSON.stringify(artifact, null, 2));
 }
 
 export function writeStatusArtifact(productDir, artifact) {
@@ -32,6 +41,13 @@ export function resetResultArtifacts(productDir) {
     recursive: true,
     force: true,
   });
+  fs.rmSync(path.join(productDir, ".testkit", "results", RESULT_SETUP_DIRNAME), {
+    recursive: true,
+    force: true,
+  });
+  fs.rmSync(path.join(productDir, ".testkit", "results", LIVE_ARTIFACT_FILENAME), {
+    force: true,
+  });
 }
 
 export function persistTaskArtifacts(productDir, task, emittedArtifacts) {

package/lib/runner/logs.mjs

@@ -2,21 +2,26 @@ import fs from "fs";
 import path from "path";
 
 const RESULT_LOGS_DIRNAME = "logs";
+const RESULT_SETUP_DIRNAME = "setup";
 
 export function createRunLogRegistry(productDir) {
-  const records = new Map();
+  const serviceRecords = new Map();
+  const setupRecords = new Map();
 
   return {
     ensureServiceLogRecord(config) {
       const key = `${config.runtimeLabel || config.name}:${config.name}`;
-      const existing = records.get(key);
+      const existing = serviceRecords.get(key);
       if (existing) return existing;
 
       const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}.log`;
       const relativePath = path.join(".testkit", "results", RESULT_LOGS_DIRNAME, fileName);
       const absolutePath = path.join(productDir, relativePath);
       fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
-      const stream = fs.createWriteStream(absolutePath, { flags: "a" });
+      const stream = fs.createWriteStream(absolutePath, {
+        fd: fs.openSync(absolutePath, "a"),
+        flags: "a",
+      });
       const record = {
         key,
         serviceName: config.name,
@@ -25,7 +30,32 @@ export function createRunLogRegistry(productDir) {
         absolutePath,
         stream,
       };
-      records.set(key, record);
+      serviceRecords.set(key, record);
+      return record;
+    },
+    ensureSetupLogRecord(config, stage) {
+      const key = `${config.runtimeLabel || config.name}:${config.name}:${stage}`;
+      const existing = setupRecords.get(key);
+      if (existing) return existing;
+
+      const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}__${sanitizePathSegment(stage)}.log`;
+      const relativePath = path.join(".testkit", "results", RESULT_SETUP_DIRNAME, fileName);
+      const absolutePath = path.join(productDir, relativePath);
+      fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
+      const stream = fs.createWriteStream(absolutePath, {
+        fd: fs.openSync(absolutePath, "a"),
+        flags: "a",
+      });
+      const record = {
+        key,
+        serviceName: config.name,
+        runtimeLabel: config.runtimeLabel || config.name,
+        stage,
+        path: normalizePath(relativePath),
+        absolutePath,
+        stream,
+      };
+      setupRecords.set(key, record);
       return record;
     },
     append(record, streamName, line) {
@@ -33,7 +63,7 @@ export function createRunLogRegistry(productDir) {
       record.stream.write(`${new Date().toISOString()} [${streamName}] ${line}\n`);
     },
     listServiceLogs() {
-      return [...records.values()]
+      return [...serviceRecords.values()]
         .map((record) => ({
           serviceName: record.serviceName,
           runtimeLabel: record.runtimeLabel,
@@ -45,8 +75,26 @@ export function createRunLogRegistry(productDir) {
             left.runtimeLabel.localeCompare(right.runtimeLabel)
         );
     },
+    listSetupLogs() {
+      return [...setupRecords.values()]
+        .map((record) => ({
+          serviceName: record.serviceName,
+          runtimeLabel: record.runtimeLabel,
+          stage: record.stage,
+          path: record.path,
+        }))
+        .sort(
+          (left, right) =>
+            left.serviceName.localeCompare(right.serviceName) ||
+            left.runtimeLabel.localeCompare(right.runtimeLabel) ||
+            left.stage.localeCompare(right.stage)
+        );
+    },
     closeAll() {
-      for (const record of records.values()) {
+      for (const record of serviceRecords.values()) {
+        record.stream.end();
+      }
+      for (const record of setupRecords.values()) {
         record.stream.end();
       }
     },

package/lib/runner/orchestrator.mjs

@@ -14,7 +14,7 @@ import {
   recordTaskOutcome,
   summarizeDbBackend,
 } from "./results.mjs";
-import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { buildLiveRunArtifact, buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
 import {
   applyKnownFailureIssueValidationToArtifacts,
   applyKnownFailuresToArtifacts,
@@ -29,10 +29,12 @@ import {
   loadTimings,
   resetResultArtifacts,
   saveTimings,
+  writeLiveRunArtifact,
   writeRunArtifact,
   writeStatusArtifact,
 } from "./artifacts.mjs";
 import { createRunLogRegistry } from "./logs.mjs";
+import { createSetupOperationRegistry } from "./setup-operations.mjs";
 import {
   cleanupRunById,
   cleanupRuns,
@@ -72,6 +74,9 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   );
   const reporter = opts.reporter || null;
   const logRegistry = createRunLogRegistry(productDir);
+  let workerCount = 0;
+  let runtimeInstanceCount = 0;
+  let runtimeStats = [];
   const requestedFiles = opts.fileNames || [];
   if (requestedFiles.length > 0) {
     const unmatchedFiles = findUnmatchedRequestedFiles(
@@ -120,10 +125,40 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
     reporter
   );
   const trackers = buildServiceTrackers(servicePlans, startedAt);
+  const writeLiveSnapshot = () => {
+    const now = Date.now();
+    const partialResults = configs.map((config) =>
+      finalizeServiceResult(trackers.get(config.name), startedAt, now)
+    );
+    writeLiveRunArtifact(
+      productDir,
+      buildLiveRunArtifact({
+        productDir,
+        results: partialResults,
+        startedAt,
+        updatedAt: now,
+        execution,
+        workerCount,
+        runtimeInstanceCount,
+        runtimeStats,
+        typeValues,
+        suiteSelectors,
+        fileNames: requestedFiles,
+        shard: opts.shard || null,
+        serviceFilter: opts.serviceFilter || null,
+        metadata,
+        summarizeDbBackend,
+        serviceLogs: logRegistry.listServiceLogs(),
+        setupLogs: logRegistry.listSetupLogs(),
+        setupOperations: setupRegistry.listOperations(),
+      })
+    );
+  };
+  const setupRegistry = createSetupOperationRegistry({
+    logRegistry,
+    onChange: writeLiveSnapshot,
+  });
   const executedPlans = servicePlans.filter((plan) => !plan.skipped);
-  let workerCount = 0;
-  let runtimeInstanceCount = 0;
-  let runtimeStats = [];
   let exitCode = 0;
   const lifecycle = createRunLifecycle(productDir);
   lifecycle.markRunning();
@@ -131,6 +166,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
   let results = [];
   let finishedAt = Date.now();
   let knownFailureIssueValidation = null;
+  writeLiveSnapshot();
 
   try {
     if (executedPlans.length > 0) {
@@ -149,6 +185,7 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
         runtimeOptions: {
           reporter,
           logRegistry,
+          setupRegistry,
         },
       });
       const timingUpdates = [];
@@ -164,8 +201,14 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
               timingUpdates,
               lifecycle,
               claimNextTask,
-              recordTaskOutcome,
-              recordGraphError,
+              (allTrackers, task, outcome, now) => {
+                recordTaskOutcome(allTrackers, task, outcome, now);
+                writeLiveSnapshot();
+              },
+              (allTrackers, graph, message, now) => {
+                recordGraphError(allTrackers, graph, message, now);
+                writeLiveSnapshot();
+              },
               reporter
             )
           )
@@ -177,9 +220,11 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
             for (const tracker of trackers.values()) {
               if (!tracker.skipped) addTrackerError(tracker, message);
             }
+            writeLiveSnapshot();
           }
         }
         runtimeStats = runtimeManager.getStats();
+        writeLiveSnapshot();
       } finally {
         await runtimeManager.cleanupAll();
       }
@@ -208,6 +253,8 @@ export async function runAll(configs, typeValues, suiteSelectors, opts, allConfi
       metadata,
       summarizeDbBackend,
       serviceLogs: logRegistry.listServiceLogs(),
+      setupLogs: logRegistry.listSetupLogs(),
+      setupOperations: setupRegistry.listOperations(),
     });
     const statusArtifact = opts.writeStatus
       ? buildStatusArtifact({

package/lib/runner/reporting.mjs

@@ -112,6 +112,9 @@ export function buildRunArtifact({
   metadata,
   summarizeDbBackend,
   serviceLogs = [],
+  setupLogs = [],
+  setupOperations = [],
+  runStatus = null,
 }) {
   const executed = results.filter((result) => !result.skipped);
   const effectivelySkippedServices = executed.filter(isEffectivelySkippedService);
@@ -128,7 +131,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 7,
+    schemaVersion: 8,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -138,7 +141,7 @@ export function buildRunArtifact({
     git: metadata.git,
     host: metadata.host,
     run: {
-      status: failedServices.length > 0 ? "failed" : "passed",
+      status: runStatus || (failedServices.length > 0 ? "failed" : "passed"),
       startedAt: new Date(startedAt).toISOString(),
       finishedAt: new Date(finishedAt).toISOString(),
       durationMs: finishedAt - startedAt,
@@ -179,6 +182,10 @@ export function buildRunArtifact({
     },
     logs: {
       services: serviceLogs,
+      setup: setupLogs,
+    },
+    setup: {
+      operations: setupOperations,
     },
     services: results.map((result) => ({
       name: result.name,
@@ -203,6 +210,49 @@ export function buildRunArtifact({
   };
 }
 
+export function buildLiveRunArtifact({
+  productDir,
+  results,
+  startedAt,
+  updatedAt,
+  execution,
+  workerCount,
+  runtimeInstanceCount,
+  runtimeStats,
+  typeValues,
+  suiteSelectors,
+  fileNames,
+  shard,
+  serviceFilter,
+  metadata,
+  summarizeDbBackend,
+  serviceLogs = [],
+  setupLogs = [],
+  setupOperations = [],
+}) {
+  return buildRunArtifact({
+    productDir,
+    results,
+    startedAt,
+    finishedAt: updatedAt,
+    execution,
+    workerCount,
+    runtimeInstanceCount,
+    runtimeStats,
+    typeValues,
+    suiteSelectors,
+    fileNames,
+    shard,
+    serviceFilter,
+    metadata,
+    summarizeDbBackend,
+    serviceLogs,
+    setupLogs,
+    setupOperations,
+    runStatus: "running",
+  });
+}
+
 function isEffectivelySkippedService(result) {
   return (
     !result.skipped &&

package/lib/runner/reporting.test.mjs

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { buildLiveRunArtifact, buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
 
 describe("runner reporting", () => {
   it("builds run artifacts", () => {
@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(7);
+    expect(artifact.schemaVersion).toBe(8);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -109,7 +109,85 @@ describe("runner reporting", () => {
     expect(artifact.services[0].totalTaskDurationMs).toBe(2400);
     expect(artifact.logs).toEqual({
       services: [],
+      setup: [],
     });
+    expect(artifact.setup).toEqual({
+      operations: [],
+    });
+  });
+
+  it("builds live run artifacts with running status and setup details", () => {
+    const artifact = buildLiveRunArtifact({
+      productDir: "/tmp/my-product",
+      results: [],
+      startedAt: 1_000,
+      updatedAt: 2_000,
+      execution: {
+        workers: 2,
+        fileTimeoutSeconds: 60,
+      },
+      workerCount: 1,
+      runtimeInstanceCount: 1,
+      runtimeStats: [],
+      typeValues: ["int"],
+      suiteSelectors: [],
+      fileNames: [],
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc",
+          repoRoot: "/tmp",
+        },
+        host: {
+          hostname: "local",
+          username: "dev",
+        },
+        testkitVersion: "0.1.54",
+      },
+      summarizeDbBackend: () => "local",
+      serviceLogs: [],
+      setupLogs: [
+        {
+          serviceName: "api",
+          runtimeLabel: "api",
+          stage: "runtime:prepare",
+          path: ".testkit/results/setup/api__api__runtime-prepare.log",
+        },
+      ],
+      setupOperations: [
+        {
+          id: "setup-1",
+          serviceName: "api",
+          runtimeLabel: "api",
+          stage: "runtime:prepare",
+          kind: "runtime-prepare",
+          summary: "runtime prepare",
+          parentId: null,
+          status: "running",
+          startedAt: "1970-01-01T00:00:01.000Z",
+          finishedAt: null,
+          durationMs: null,
+          error: null,
+          logRef: {
+            path: ".testkit/results/setup/api__api__runtime-prepare.log",
+            stage: "runtime:prepare",
+          },
+        },
+      ],
+    });
+
+    expect(artifact.run.status).toBe("running");
+    expect(artifact.logs.setup).toEqual([
+      {
+        serviceName: "api",
+        runtimeLabel: "api",
+        stage: "runtime:prepare",
+        path: ".testkit/results/setup/api__api__runtime-prepare.log",
+      },
+    ]);
+    expect(artifact.setup.operations).toHaveLength(1);
   });
 
   it("builds deterministic status artifacts", () => {

package/lib/runner/runtime-contexts.mjs

@@ -39,7 +39,7 @@ export async function ensureRuntimeInstanceReady(context, task, lifecycle, optio
   if (!context.prepared) {
     if (!context.preparationPromise) {
       context.preparationPromise = (async () => {
-        await prepareDatabases(context.runtimeConfigs);
+        await prepareDatabases(context.runtimeConfigs, options);
         await prepareRuntimeServices(context.runtimeConfigs, options);
         context.prepared = true;
       })().finally(() => {
@@ -80,8 +80,8 @@ export async function cleanupRuntimeInstanceContext(context, lifecycle) {
   await deactivateRuntimeInstanceContext(context, lifecycle);
 }
 
-export async function prepareDatabases(runtimeConfigs) {
+export async function prepareDatabases(runtimeConfigs, options = {}) {
   for (const config of runtimeConfigs) {
-    await prepareDatabaseRuntime(config);
+    await prepareDatabaseRuntime(config, options);
   }
 }

package/lib/runner/runtime-preparation.mjs

@@ -28,6 +28,12 @@ export async function prepareRuntimeService(config, options = {}) {
   const manifestPath = path.join(prepareDir, MANIFEST_FILE);
   const existingManifest = readPrepareManifest(manifestPath);
   if (existingManifest?.fingerprint === fingerprint) {
+    options.setupRegistry?.recordCached({
+      config,
+      stage: "runtime:prepare",
+      kind: "runtime-prepare",
+      summary: "runtime prepare cache hit",
+    });
     return;
   }
 
@@ -42,6 +48,14 @@ export async function prepareRuntimeService(config, options = {}) {
     env.DATABASE_URL = databaseUrl;
   }
 
+  const prepareOperation = options.setupRegistry?.start({
+    config,
+    stage: "runtime:prepare",
+    kind: "runtime-prepare",
+    summary: "runtime prepare",
+    recordLog: false,
+  });
+
   try {
     await announceResolvedToolchain(
       config,
@@ -54,7 +68,16 @@ export async function prepareRuntimeService(config, options = {}) {
       env,
       labelPrefix: "runtime:prepare",
       reporter: options.reporter,
+      setupRegistry: options.setupRegistry || null,
+      parentOperation: prepareOperation,
     });
+    const finished = prepareOperation
+      ? options.setupRegistry.finish(prepareOperation, {
+          status: "passed",
+          summary: "runtime prepare",
+        })
+      : null;
+    if (finished) options.reporter?.setupOperationFinished?.(finished);
     writePrepareManifest(manifestPath, {
       fingerprint,
       preparedAt: new Date().toISOString(),
@@ -62,6 +85,14 @@ export async function prepareRuntimeService(config, options = {}) {
       serviceName: config.name,
     });
   } catch (error) {
+    const finished = prepareOperation
+      ? options.setupRegistry.finish(prepareOperation, {
+          status: "failed",
+          summary: "runtime prepare",
+          error: error?.message || error,
+        })
+      : null;
+    if (finished) options.reporter?.setupOperationFinished?.(finished);
     fs.rmSync(prepareDir, { recursive: true, force: true });
     throw error;
   }