@elench/testkit 0.1.53 → 0.1.55

package/lib/runner/reporting.mjs

@@ -112,6 +112,9 @@ export function buildRunArtifact({
   metadata,
   summarizeDbBackend,
   serviceLogs = [],
+  setupLogs = [],
+  setupOperations = [],
+  runStatus = null,
 }) {
   const executed = results.filter((result) => !result.skipped);
   const effectivelySkippedServices = executed.filter(isEffectivelySkippedService);
@@ -128,7 +131,7 @@ export function buildRunArtifact({
   const dbBackend = summarizeDbBackend(results);
 
   return {
-    schemaVersion: 7,
+    schemaVersion: 8,
     source: "testkit",
     generatedAt: new Date(finishedAt).toISOString(),
     product: {
@@ -138,7 +141,7 @@ export function buildRunArtifact({
     git: metadata.git,
     host: metadata.host,
     run: {
-      status: failedServices.length > 0 ? "failed" : "passed",
+      status: runStatus || (failedServices.length > 0 ? "failed" : "passed"),
       startedAt: new Date(startedAt).toISOString(),
       finishedAt: new Date(finishedAt).toISOString(),
       durationMs: finishedAt - startedAt,
@@ -179,6 +182,10 @@ export function buildRunArtifact({
     },
     logs: {
       services: serviceLogs,
+      setup: setupLogs,
+    },
+    setup: {
+      operations: setupOperations,
     },
     services: results.map((result) => ({
       name: result.name,
@@ -203,6 +210,49 @@ export function buildRunArtifact({
   };
 }
 
+export function buildLiveRunArtifact({
+  productDir,
+  results,
+  startedAt,
+  updatedAt,
+  execution,
+  workerCount,
+  runtimeInstanceCount,
+  runtimeStats,
+  typeValues,
+  suiteSelectors,
+  fileNames,
+  shard,
+  serviceFilter,
+  metadata,
+  summarizeDbBackend,
+  serviceLogs = [],
+  setupLogs = [],
+  setupOperations = [],
+}) {
+  return buildRunArtifact({
+    productDir,
+    results,
+    startedAt,
+    finishedAt: updatedAt,
+    execution,
+    workerCount,
+    runtimeInstanceCount,
+    runtimeStats,
+    typeValues,
+    suiteSelectors,
+    fileNames,
+    shard,
+    serviceFilter,
+    metadata,
+    summarizeDbBackend,
+    serviceLogs,
+    setupLogs,
+    setupOperations,
+    runStatus: "running",
+  });
+}
+
 function isEffectivelySkippedService(result) {
   return (
     !result.skipped &&

package/lib/runner/reporting.test.mjs

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
+import { buildLiveRunArtifact, buildRunArtifact, buildStatusArtifact } from "./reporting.mjs";
 
 describe("runner reporting", () => {
   it("builds run artifacts", () => {
@@ -78,7 +78,7 @@ describe("runner reporting", () => {
     });
 
     expect(artifact.product.name).toBe("my-product");
-    expect(artifact.schemaVersion).toBe(7);
+    expect(artifact.schemaVersion).toBe(8);
     expect(artifact.run).toMatchObject({
       workers: 2,
       fileTimeoutSeconds: 60,
@@ -109,7 +109,85 @@ describe("runner reporting", () => {
     expect(artifact.services[0].totalTaskDurationMs).toBe(2400);
     expect(artifact.logs).toEqual({
       services: [],
+      setup: [],
     });
+    expect(artifact.setup).toEqual({
+      operations: [],
+    });
+  });
+
+  it("builds live run artifacts with running status and setup details", () => {
+    const artifact = buildLiveRunArtifact({
+      productDir: "/tmp/my-product",
+      results: [],
+      startedAt: 1_000,
+      updatedAt: 2_000,
+      execution: {
+        workers: 2,
+        fileTimeoutSeconds: 60,
+      },
+      workerCount: 1,
+      runtimeInstanceCount: 1,
+      runtimeStats: [],
+      typeValues: ["int"],
+      suiteSelectors: [],
+      fileNames: [],
+      shard: null,
+      serviceFilter: null,
+      metadata: {
+        git: {
+          branch: "main",
+          commitSha: "abc",
+          repoRoot: "/tmp",
+        },
+        host: {
+          hostname: "local",
+          username: "dev",
+        },
+        testkitVersion: "0.1.54",
+      },
+      summarizeDbBackend: () => "local",
+      serviceLogs: [],
+      setupLogs: [
+        {
+          serviceName: "api",
+          runtimeLabel: "api",
+          stage: "runtime:prepare",
+          path: ".testkit/results/setup/api__api__runtime-prepare.log",
+        },
+      ],
+      setupOperations: [
+        {
+          id: "setup-1",
+          serviceName: "api",
+          runtimeLabel: "api",
+          stage: "runtime:prepare",
+          kind: "runtime-prepare",
+          summary: "runtime prepare",
+          parentId: null,
+          status: "running",
+          startedAt: "1970-01-01T00:00:01.000Z",
+          finishedAt: null,
+          durationMs: null,
+          error: null,
+          logRef: {
+            path: ".testkit/results/setup/api__api__runtime-prepare.log",
+            stage: "runtime:prepare",
+          },
+        },
+      ],
+    });
+
+    expect(artifact.run.status).toBe("running");
+    expect(artifact.logs.setup).toEqual([
+      {
+        serviceName: "api",
+        runtimeLabel: "api",
+        stage: "runtime:prepare",
+        path: ".testkit/results/setup/api__api__runtime-prepare.log",
+      },
+    ]);
+    expect(artifact.setup.operations).toHaveLength(1);
   });
 
   it("builds deterministic status artifacts", () => {

package/lib/runner/runtime-contexts.mjs

@@ -39,7 +39,7 @@ export async function ensureRuntimeInstanceReady(context, task, lifecycle, optio
   if (!context.prepared) {
     if (!context.preparationPromise) {
       context.preparationPromise = (async () => {
-        await prepareDatabases(context.runtimeConfigs);
+        await prepareDatabases(context.runtimeConfigs, options);
         await prepareRuntimeServices(context.runtimeConfigs, options);
         context.prepared = true;
       })().finally(() => {
@@ -80,8 +80,8 @@ export async function cleanupRuntimeInstanceContext(context, lifecycle) {
   await deactivateRuntimeInstanceContext(context, lifecycle);
 }
 
-export async function prepareDatabases(runtimeConfigs) {
+export async function prepareDatabases(runtimeConfigs, options = {}) {
   for (const config of runtimeConfigs) {
-    await prepareDatabaseRuntime(config);
+    await prepareDatabaseRuntime(config, options);
   }
 }

package/lib/runner/runtime-preparation.mjs

@@ -28,6 +28,12 @@ export async function prepareRuntimeService(config, options = {}) {
   const manifestPath = path.join(prepareDir, MANIFEST_FILE);
   const existingManifest = readPrepareManifest(manifestPath);
   if (existingManifest?.fingerprint === fingerprint) {
+    options.setupRegistry?.recordCached({
+      config,
+      stage: "runtime:prepare",
+      kind: "runtime-prepare",
+      summary: "runtime prepare cache hit",
+    });
     return;
   }
 
@@ -42,6 +48,14 @@ export async function prepareRuntimeService(config, options = {}) {
     env.DATABASE_URL = databaseUrl;
   }
 
+  const prepareOperation = options.setupRegistry?.start({
+    config,
+    stage: "runtime:prepare",
+    kind: "runtime-prepare",
+    summary: "runtime prepare",
+    recordLog: false,
+  });
+
   try {
     await announceResolvedToolchain(
       config,
@@ -54,7 +68,16 @@ export async function prepareRuntimeService(config, options = {}) {
       env,
       labelPrefix: "runtime:prepare",
       reporter: options.reporter,
+      setupRegistry: options.setupRegistry || null,
+      parentOperation: prepareOperation,
     });
+    const finished = prepareOperation
+      ? options.setupRegistry.finish(prepareOperation, {
+          status: "passed",
+          summary: "runtime prepare",
+        })
+      : null;
+    if (finished) options.reporter?.setupOperationFinished?.(finished);
     writePrepareManifest(manifestPath, {
       fingerprint,
       preparedAt: new Date().toISOString(),
@@ -62,6 +85,14 @@ export async function prepareRuntimeService(config, options = {}) {
       serviceName: config.name,
     });
   } catch (error) {
+    const finished = prepareOperation
+      ? options.setupRegistry.finish(prepareOperation, {
+          status: "failed",
+          summary: "runtime prepare",
+          error: error?.message || error,
+        })
+      : null;
+    if (finished) options.reporter?.setupOperationFinished?.(finished);
     fs.rmSync(prepareDir, { recursive: true, force: true });
     throw error;
   }

package/lib/runner/setup-operations.mjs

@@ -0,0 +1,115 @@
+export function createSetupOperationRegistry({ logRegistry = null, onChange = null } = {}) {
+  const operations = [];
+  const byId = new Map();
+  let nextId = 1;
+
+  function emitChange() {
+    onChange?.(listOperations());
+  }
+
+  function start({
+    config,
+    stage,
+    kind = "setup",
+    summary = null,
+    parentId = null,
+    recordLog = true,
+  }) {
+    const startedAt = new Date().toISOString();
+    const logRecord = recordLog ? logRegistry?.ensureSetupLogRecord(config, stage) || null : null;
+    const operation = {
+      id: `setup-${nextId++}`,
+      serviceName: config.name,
+      runtimeLabel: config.runtimeLabel || config.name,
+      stage,
+      kind,
+      summary,
+      parentId,
+      status: "running",
+      startedAt,
+      finishedAt: null,
+      durationMs: null,
+      error: null,
+      logRef: logRecord
+        ? {
+            path: logRecord.path,
+            stage: logRecord.stage,
+          }
+        : null,
+      _logRecord: logRecord,
+    };
+    operations.push(operation);
+    byId.set(operation.id, operation);
+    emitChange();
+    return operation;
+  }
+
+  function finish(target, { status = "passed", summary, error = null } = {}) {
+    const operation = typeof target === "string" ? byId.get(target) : target;
+    if (!operation) return null;
+    const finishedAt = new Date().toISOString();
+    operation.finishedAt = finishedAt;
+    operation.durationMs = Math.max(
+      0,
+      Date.parse(finishedAt) - Date.parse(operation.startedAt || finishedAt)
+    );
+    operation.status = status;
+    if (summary !== undefined) operation.summary = summary;
+    if (error !== null) operation.error = String(error);
+    emitChange();
+    return operation;
+  }
+
+  function recordCached({ config, stage, kind = "setup", summary = null, parentId = null }) {
+    const now = new Date().toISOString();
+    const operation = {
+      id: `setup-${nextId++}`,
+      serviceName: config.name,
+      runtimeLabel: config.runtimeLabel || config.name,
+      stage,
+      kind,
+      summary,
+      parentId,
+      status: "cached",
+      startedAt: now,
+      finishedAt: now,
+      durationMs: 0,
+      error: null,
+      logRef: null,
+      _logRecord: null,
+    };
+    operations.push(operation);
+    byId.set(operation.id, operation);
+    emitChange();
+    return operation;
+  }
+
+  function listOperations() {
+    return operations.map(cloneOperation);
+  }
+
+  return {
+    start,
+    finish,
+    recordCached,
+    listOperations,
+  };
+}
+
+function cloneOperation(operation) {
+  return {
+    id: operation.id,
+    serviceName: operation.serviceName,
+    runtimeLabel: operation.runtimeLabel,
+    stage: operation.stage,
+    kind: operation.kind,
+    summary: operation.summary,
+    parentId: operation.parentId,
+    status: operation.status,
+    startedAt: operation.startedAt,
+    finishedAt: operation.finishedAt,
+    durationMs: operation.durationMs,
+    error: operation.error,
+    logRef: operation.logRef,
+  };
+}

package/lib/runner/setup-operations.test.mjs

@@ -0,0 +1,94 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { createRunLogRegistry } from "./logs.mjs";
+import { createSetupOperationRegistry } from "./setup-operations.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function makeConfig(productDir) {
+  return {
+    name: "api",
+    runtimeLabel: "api",
+    productDir,
+  };
+}
+
+describe("setup operation registry", () => {
+  it("tracks running and finished setup operations with log refs", () => {
+    const productDir = makeTempDir("testkit-setup-ops-");
+    const logRegistry = createRunLogRegistry(productDir);
+    const changes = [];
+    const registry = createSetupOperationRegistry({
+      logRegistry,
+      onChange(operations) {
+        changes.push(operations);
+      },
+    });
+
+    const operation = registry.start({
+      config: makeConfig(productDir),
+      stage: "template:migrate:api:1",
+      kind: "setup-step",
+      summary: "command: node scripts/migrate.mjs",
+    });
+
+    expect(operation.logRef).toMatchObject({
+      path: ".testkit/results/setup/api__api__template-migrate-api-1.log",
+      stage: "template:migrate:api:1",
+    });
+
+    const finished = registry.finish(operation, {
+      status: "passed",
+      summary: "command: node scripts/migrate.mjs",
+    });
+
+    expect(finished.status).toBe("passed");
+    expect(finished.durationMs).toBeGreaterThanOrEqual(0);
+    expect(registry.listOperations()).toEqual([
+      expect.objectContaining({
+        stage: "template:migrate:api:1",
+        kind: "setup-step",
+        status: "passed",
+        summary: "command: node scripts/migrate.mjs",
+        logRef: operation.logRef,
+      }),
+    ]);
+    expect(changes).toHaveLength(2);
+
+    logRegistry.closeAll();
+  });
+
+  it("records cached setup operations without logs", () => {
+    const productDir = makeTempDir("testkit-setup-ops-cached-");
+    const registry = createSetupOperationRegistry();
+
+    const operation = registry.recordCached({
+      config: makeConfig(productDir),
+      stage: "template",
+      kind: "database-template",
+      summary: "template cache hit",
+    });
+
+    expect(operation).toMatchObject({
+      stage: "template",
+      kind: "database-template",
+      status: "cached",
+      summary: "template cache hit",
+      logRef: null,
+    });
+  });
+});

package/lib/runner/template-steps.mjs

@@ -23,16 +23,52 @@ const MODULE_RUNNER_ENTRY = path.join(
   "template-step-module-runner.mjs"
 );
 
-export async function runConfiguredSteps({ config, steps = [], env, labelPrefix, reporter = null }) {
+export async function runConfiguredSteps({
+  config,
+  steps = [],
+  env,
+  labelPrefix,
+  reporter = null,
+  setupRegistry = null,
+  parentOperation = null,
+}) {
   if (steps.length === 0) return;
   const resolvedToolchain = await resolveConfiguredToolchain(config);
   await announceResolvedToolchain(config, resolvedToolchain, reporter);
 
   for (const [index, step] of steps.entries()) {
     const label = `${labelPrefix}:${config.name}:${index + 1}`;
-    if (reporter?.phaseStarted) reporter.phaseStarted(label);
-    else console.log(`\n── ${label} ──`);
-    await runConfiguredStep(config, step, env, resolvedToolchain);
+    const stepOperation = setupRegistry?.start({
+      config,
+      stage: label,
+      kind: "setup-step",
+      summary: summarizeConfiguredStep(step),
+      parentId: parentOperation?.id || parentOperation || null,
+    });
+    reporter?.phaseStarted?.(label);
+    try {
+      await runConfiguredStep(config, step, env, resolvedToolchain, {
+        reporter,
+        logRecord: stepOperation?._logRecord || null,
+      });
+      const finished = stepOperation
+        ? setupRegistry.finish(stepOperation, {
+            status: "passed",
+            summary: summarizeConfiguredStep(step),
+          })
+        : null;
+      if (finished) reporter?.setupOperationFinished?.(finished);
+    } catch (error) {
+      const finished = stepOperation
+        ? setupRegistry.finish(stepOperation, {
+            status: "failed",
+            summary: summarizeConfiguredStep(step),
+            error: error?.message || error,
+          })
+        : null;
+      if (finished) reporter?.setupOperationFinished?.(finished);
+      throw error;
+    }
   }
 }
 
@@ -65,22 +101,33 @@ export function resolveConfiguredPath(productDir, stepCwd, targetPath) {
   return path.resolve(resolveConfiguredCwd(productDir, stepCwd), targetPath);
 }
 
-async function runConfiguredStep(config, step, env, resolvedToolchain) {
+async function runConfiguredStep(config, step, env, resolvedToolchain, options = {}) {
   const runtimeEnv = applyToolchainEnv(env, resolvedToolchain);
   const cwd = resolveConfiguredCwd(config.productDir, step.cwd);
+  const liveWriter =
+    options.reporter?.outputMode === "debug"
+      ? (line) => options.reporter.writeDebugLine?.(line)
+      : null;
 
   if (step.kind === "command") {
-    await execaCommand(step.cmd, {
+    const child = execaCommand(step.cmd, {
       cwd,
       env: runtimeEnv,
-      stdio: "inherit",
       shell: true,
+      stdout: "pipe",
+      stderr: "pipe",
+      reject: false,
+    });
+    await awaitCapturedProcess(child, {
+      livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
+      liveWriter,
+      logRecord: options.logRecord || null,
     });
     return;
   }
 
   if (step.kind === "sql-file") {
-    await execa(
+    const child = execa(
       "psql",
       [
         runtimeEnv.DATABASE_URL,
@@ -93,9 +140,16 @@ async function runConfiguredStep(config, step, env, resolvedToolchain) {
       {
         cwd,
         env: runtimeEnv,
-        stdio: "inherit",
+        stdout: "pipe",
+        stderr: "pipe",
+        reject: false,
       }
     );
+    await awaitCapturedProcess(child, {
+      livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
+      liveWriter,
+      logRecord: options.logRecord || null,
+    });
     return;
   }
 
@@ -116,15 +170,22 @@ async function runConfiguredStep(config, step, env, resolvedToolchain) {
     fs.writeFileSync(contextPath, JSON.stringify(context));
 
     try {
-      await execa(
+      const child = execa(
         resolvedToolchain?.nodeExecutable || process.execPath,
         [MODULE_RUNNER_ENTRY, bundledModule.outputFile, exportName, contextPath],
         {
           cwd,
           env: runtimeEnv,
-          stdio: "inherit",
+          stdout: "pipe",
+          stderr: "pipe",
+          reject: false,
         }
       );
+      await awaitCapturedProcess(child, {
+        livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
+        liveWriter,
+        logRecord: options.logRecord || null,
+      });
     } finally {
       fs.rmSync(contextPath, { force: true });
     }
@@ -199,3 +260,60 @@ function parseModuleSpecifier(specifier) {
     exportName: exportName || "default",
   };
 }
+
+function summarizeConfiguredStep(step) {
+  if (step.kind === "command") return `command: ${String(step.cmd).trim()}`;
+  if (step.kind === "sql-file") return `sql: ${step.path}`;
+  if (step.kind === "module") return `module: ${step.specifier}`;
+  return step.kind;
+}
+
+async function awaitCapturedProcess(child, { livePrefix = "", liveWriter = null, logRecord = null } = {}) {
+  const drains = [
+    captureProcessOutput(child.stdout, "stdout", livePrefix, liveWriter, logRecord),
+    captureProcessOutput(child.stderr, "stderr", livePrefix, liveWriter, logRecord),
+  ];
+  const result = await child;
+  await Promise.all(drains);
+  if (result.exitCode !== 0) {
+    throw new Error(result.shortMessage || result.stderr || result.stdout || `Step failed with exit code ${result.exitCode}`);
+  }
+}
+
+function captureProcessOutput(stream, streamName, livePrefix, liveWriter, logRecord) {
+  if (!stream) return Promise.resolve();
+
+  let pending = "";
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = () => {
+      if (settled) return;
+      settled = true;
+      resolve();
+    };
+
+    stream.on("data", (chunk) => {
+      pending += chunk.toString();
+      const lines = pending.split(/\r?\n/);
+      pending = lines.pop() || "";
+      for (const line of lines) {
+        if (line.length === 0) continue;
+        if (logRecord) {
+          logRecord.stream.write(`${new Date().toISOString()} [${streamName}] ${line}\n`);
+        }
+        if (liveWriter) liveWriter(livePrefix ? `${livePrefix} ${line}` : line);
+      }
+    });
+    stream.on("end", () => {
+      if (pending.length > 0) {
+        if (logRecord) {
+          logRecord.stream.write(`${new Date().toISOString()} [${streamName}] ${pending}\n`);
+        }
+        if (liveWriter) liveWriter(livePrefix ? `${livePrefix} ${pending}` : pending);
+      }
+      settle();
+    });
+    stream.on("close", settle);
+    stream.on("error", settle);
+  });
+}