@elench/testkit 0.1.54 → 0.1.56

package/lib/runner/setup-operations.test.mjs

@@ -0,0 +1,94 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { createRunLogRegistry } from "./logs.mjs";
+import { createSetupOperationRegistry } from "./setup-operations.mjs";
+
+const tempDirs = [];
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function makeConfig(productDir) {
+  return {
+    name: "api",
+    runtimeLabel: "api",
+    productDir,
+  };
+}
+
+describe("setup operation registry", () => {
+  it("tracks running and finished setup operations with log refs", () => {
+    const productDir = makeTempDir("testkit-setup-ops-");
+    const logRegistry = createRunLogRegistry(productDir);
+    const changes = [];
+    const registry = createSetupOperationRegistry({
+      logRegistry,
+      onChange(operations) {
+        changes.push(operations);
+      },
+    });
+
+    const operation = registry.start({
+      config: makeConfig(productDir),
+      stage: "template:migrate:api:1",
+      kind: "setup-step",
+      summary: "command: node scripts/migrate.mjs",
+    });
+
+    expect(operation.logRef).toMatchObject({
+      path: ".testkit/results/setup/api__api__template-migrate-api-1.log",
+      stage: "template:migrate:api:1",
+    });
+
+    const finished = registry.finish(operation, {
+      status: "passed",
+      summary: "command: node scripts/migrate.mjs",
+    });
+
+    expect(finished.status).toBe("passed");
+    expect(finished.durationMs).toBeGreaterThanOrEqual(0);
+    expect(registry.listOperations()).toEqual([
+      expect.objectContaining({
+        stage: "template:migrate:api:1",
+        kind: "setup-step",
+        status: "passed",
+        summary: "command: node scripts/migrate.mjs",
+        logRef: operation.logRef,
+      }),
+    ]);
+    expect(changes).toHaveLength(2);
+
+    logRegistry.closeAll();
+  });
+
+  it("records cached setup operations without logs", () => {
+    const productDir = makeTempDir("testkit-setup-ops-cached-");
+    const registry = createSetupOperationRegistry();
+
+    const operation = registry.recordCached({
+      config: makeConfig(productDir),
+      stage: "template",
+      kind: "database-template",
+      summary: "template cache hit",
+    });
+
+    expect(operation).toMatchObject({
+      stage: "template",
+      kind: "database-template",
+      status: "cached",
+      summary: "template cache hit",
+      logRef: null,
+    });
+  });
+});

package/lib/runner/suite-selection.mjs

@@ -1,4 +1,4 @@
-const USER_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
+const USER_TYPES = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 
 export function normalizeTypeValues(values = []) {
   const expanded = [];
@@ -9,7 +9,7 @@ export function normalizeTypeValues(values = []) {
       if (!value) continue;
       if (!USER_TYPES.has(value)) {
         throw new Error(
-          `Unknown type "${value}". Expected one of: int, e2e, dal, load, pw, all.`
+          `Unknown type "${value}". Expected one of: int, e2e, scenario, dal, load, pw, all.`
         );
       }
       expanded.push(value);
@@ -25,7 +25,7 @@ export function normalizeTypeValues(values = []) {
     throw new Error(`"--type all" cannot be combined with other types.`);
   }
 
-  const order = ["int", "e2e", "dal", "load", "pw", "all"];
+  const order = ["int", "e2e", "scenario", "dal", "load", "pw", "all"];
   return deduped.sort((left, right) => order.indexOf(left) - order.indexOf(right));
 }
 
@@ -52,7 +52,7 @@ export function parseSuiteSelectors(values = []) {
       const name = typeMatch[2].trim();
       if (!USER_TYPES.has(type) || type === "all") {
         throw new Error(
-          `Unknown suite selector type "${type}". Expected one of: int, e2e, dal, load, pw.`
+          `Unknown suite selector type "${type}". Expected one of: int, e2e, scenario, dal, load, pw.`
         );
       }
       if (!name) {

package/lib/runner/suite-selection.test.mjs

@@ -11,14 +11,20 @@ import {
 describe("runner suite selection", () => {
   it("normalizes selected type values", () => {
     expect(normalizeTypeValues([])).toEqual(["all"]);
-    expect(normalizeTypeValues(["int,e2e", "dal"])).toEqual(["int", "e2e", "dal"]);
+    expect(normalizeTypeValues(["int,e2e,scenario", "dal"])).toEqual([
+      "int",
+      "e2e",
+      "scenario",
+      "dal",
+    ]);
     expect(() => normalizeTypeValues(["all", "int"])).toThrow("cannot be combined");
     expect(() => normalizeTypeValues(["jest"])).toThrow("Unknown type");
   });
 
   it("parses suite selectors", () => {
-    expect(parseSuiteSelectors(["auth,dal:queries"])).toEqual([
+    expect(parseSuiteSelectors(["auth,scenario:journeys,dal:queries"])).toEqual([
       { kind: "plain", name: "auth", raw: "auth" },
+      { kind: "typed", type: "scenario", name: "journeys", raw: "scenario:journeys" },
       { kind: "typed", type: "dal", name: "queries", raw: "dal:queries" },
     ]);
     expect(() => parseSuiteSelectors(["all:auth"])).toThrow("Unknown suite selector type");
@@ -36,6 +42,7 @@ describe("runner suite selection", () => {
 
   it("maps discovered suites to user-facing selection types", () => {
     expect(suiteSelectionType("integration", "k6")).toBe("int");
+    expect(suiteSelectionType("scenario", "k6")).toBe("scenario");
     expect(suiteSelectionType("e2e", "playwright")).toBe("pw");
     expect(suiteSelectionType("dal", "k6")).toBe("dal");
   });

package/lib/runner/template-steps.mjs

@@ -23,16 +23,52 @@ const MODULE_RUNNER_ENTRY = path.join(
   "template-step-module-runner.mjs"
 );
 
-export async function runConfiguredSteps({ config, steps = [], env, labelPrefix, reporter = null }) {
+export async function runConfiguredSteps({
+  config,
+  steps = [],
+  env,
+  labelPrefix,
+  reporter = null,
+  setupRegistry = null,
+  parentOperation = null,
+}) {
   if (steps.length === 0) return;
   const resolvedToolchain = await resolveConfiguredToolchain(config);
   await announceResolvedToolchain(config, resolvedToolchain, reporter);
 
   for (const [index, step] of steps.entries()) {
     const label = `${labelPrefix}:${config.name}:${index + 1}`;
-    if (reporter?.phaseStarted) reporter.phaseStarted(label);
-    else console.log(`\n── ${label} ──`);
-    await runConfiguredStep(config, step, env, resolvedToolchain);
+    const stepOperation = setupRegistry?.start({
+      config,
+      stage: label,
+      kind: "setup-step",
+      summary: summarizeConfiguredStep(step),
+      parentId: parentOperation?.id || parentOperation || null,
+    });
+    reporter?.phaseStarted?.(label);
+    try {
+      await runConfiguredStep(config, step, env, resolvedToolchain, {
+        reporter,
+        logRecord: stepOperation?._logRecord || null,
+      });
+      const finished = stepOperation
+        ? setupRegistry.finish(stepOperation, {
+            status: "passed",
+            summary: summarizeConfiguredStep(step),
+          })
+        : null;
+      if (finished) reporter?.setupOperationFinished?.(finished);
+    } catch (error) {
+      const finished = stepOperation
+        ? setupRegistry.finish(stepOperation, {
+            status: "failed",
+            summary: summarizeConfiguredStep(step),
+            error: error?.message || error,
+          })
+        : null;
+      if (finished) reporter?.setupOperationFinished?.(finished);
+      throw error;
+    }
   }
 }
 
@@ -65,22 +101,33 @@ export function resolveConfiguredPath(productDir, stepCwd, targetPath) {
   return path.resolve(resolveConfiguredCwd(productDir, stepCwd), targetPath);
 }
 
-async function runConfiguredStep(config, step, env, resolvedToolchain) {
+async function runConfiguredStep(config, step, env, resolvedToolchain, options = {}) {
   const runtimeEnv = applyToolchainEnv(env, resolvedToolchain);
   const cwd = resolveConfiguredCwd(config.productDir, step.cwd);
+  const liveWriter =
+    options.reporter?.outputMode === "debug"
+      ? (line) => options.reporter.writeDebugLine?.(line)
+      : null;
 
   if (step.kind === "command") {
-    await execaCommand(step.cmd, {
+    const child = execaCommand(step.cmd, {
       cwd,
       env: runtimeEnv,
-      stdio: "inherit",
       shell: true,
+      stdout: "pipe",
+      stderr: "pipe",
+      reject: false,
+    });
+    await awaitCapturedProcess(child, {
+      livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
+      liveWriter,
+      logRecord: options.logRecord || null,
     });
     return;
   }
 
   if (step.kind === "sql-file") {
-    await execa(
+    const child = execa(
       "psql",
       [
         runtimeEnv.DATABASE_URL,
@@ -93,9 +140,16 @@ async function runConfiguredStep(config, step, env, resolvedToolchain) {
       {
         cwd,
         env: runtimeEnv,
-        stdio: "inherit",
+        stdout: "pipe",
+        stderr: "pipe",
+        reject: false,
       }
     );
+    await awaitCapturedProcess(child, {
+      livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
+      liveWriter,
+      logRecord: options.logRecord || null,
+    });
     return;
   }
 
@@ -116,15 +170,22 @@ async function runConfiguredStep(config, step, env, resolvedToolchain) {
     fs.writeFileSync(contextPath, JSON.stringify(context));
 
     try {
-      await execa(
+      const child = execa(
         resolvedToolchain?.nodeExecutable || process.execPath,
         [MODULE_RUNNER_ENTRY, bundledModule.outputFile, exportName, contextPath],
         {
           cwd,
           env: runtimeEnv,
-          stdio: "inherit",
+          stdout: "pipe",
+          stderr: "pipe",
+          reject: false,
         }
       );
+      await awaitCapturedProcess(child, {
+        livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
+        liveWriter,
+        logRecord: options.logRecord || null,
+      });
     } finally {
       fs.rmSync(contextPath, { force: true });
     }
@@ -199,3 +260,60 @@ function parseModuleSpecifier(specifier) {
     exportName: exportName || "default",
   };
 }
+
+function summarizeConfiguredStep(step) {
+  if (step.kind === "command") return `command: ${String(step.cmd).trim()}`;
+  if (step.kind === "sql-file") return `sql: ${step.path}`;
+  if (step.kind === "module") return `module: ${step.specifier}`;
+  return step.kind;
+}
+
+async function awaitCapturedProcess(child, { livePrefix = "", liveWriter = null, logRecord = null } = {}) {
+  const drains = [
+    captureProcessOutput(child.stdout, "stdout", livePrefix, liveWriter, logRecord),
+    captureProcessOutput(child.stderr, "stderr", livePrefix, liveWriter, logRecord),
+  ];
+  const result = await child;
+  await Promise.all(drains);
+  if (result.exitCode !== 0) {
+    throw new Error(result.shortMessage || result.stderr || result.stdout || `Step failed with exit code ${result.exitCode}`);
+  }
+}
+
+function captureProcessOutput(stream, streamName, livePrefix, liveWriter, logRecord) {
+  if (!stream) return Promise.resolve();
+
+  let pending = "";
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = () => {
+      if (settled) return;
+      settled = true;
+      resolve();
+    };
+
+    stream.on("data", (chunk) => {
+      pending += chunk.toString();
+      const lines = pending.split(/\r?\n/);
+      pending = lines.pop() || "";
+      for (const line of lines) {
+        if (line.length === 0) continue;
+        if (logRecord) {
+          logRecord.stream.write(`${new Date().toISOString()} [${streamName}] ${line}\n`);
+        }
+        if (liveWriter) liveWriter(livePrefix ? `${livePrefix} ${line}` : line);
+      }
+    });
+    stream.on("end", () => {
+      if (pending.length > 0) {
+        if (logRecord) {
+          logRecord.stream.write(`${new Date().toISOString()} [${streamName}] ${pending}\n`);
+        }
+        if (liveWriter) liveWriter(livePrefix ? `${livePrefix} ${pending}` : pending);
+      }
+      settle();
+    });
+    stream.on("close", settle);
+    stream.on("error", settle);
+  });
+}

package/lib/runner/worker-loop.mjs

@@ -2,7 +2,7 @@ import { formatError } from "./formatting.mjs";
 import { runDalTask, runHttpK6Task } from "./default-runtime-runner.mjs";
 import { runPlaywrightTask } from "./playwright-runner.mjs";
 
-const HTTP_K6_TYPES = new Set(["integration", "e2e", "load"]);
+const HTTP_K6_TYPES = new Set(["integration", "e2e", "scenario", "load"]);
 
 export function createWorker(workerId, productDir) {
   return {

package/lib/runtime-src/k6/checks.js

@@ -136,6 +136,15 @@ export function recordFailureDetail(detail) {
   existing.count += normalized.count;
 }
 
+export function getFailureCollectionSnapshot() {
+  return {
+    phase: failureState.phase,
+    groupStack: [...failureState.groupStack],
+    failureCount: failureState.detailsByKey.size,
+    keys: [...failureState.detailsByKey.keys()].sort(),
+  };
+}
+
 function createFailureState() {
   return {
     phase: "exec",

package/lib/runtime-src/k6/scenario-runtime.js

@@ -0,0 +1,234 @@
+import { emitArtifact } from "./artifacts.js";
+import { getFailureCollectionSnapshot, group } from "./checks.js";
+
+const DEFAULT_SCENARIO_SEED = "default";
+
+export function createScenarioRuntime(options = {}) {
+  const suiteSeed = normalizeSeed(options.seed);
+  const state = {
+    seed: suiteSeed,
+    scenarioName: null,
+    choices: {},
+    notes: {},
+    resources: [],
+    resourceState: new Map(),
+    steps: [],
+  };
+
+  return {
+    get seed() {
+      return state.seed;
+    },
+    get scenarioName() {
+      return state.scenarioName;
+    },
+    resource(name, factory, options = {}) {
+      return createScenarioResource(state, name, factory, options);
+    },
+    pick(name, choices) {
+      return pickChoice(state, name, choices);
+    },
+    maybe(name, probability = 0.5) {
+      return maybeChoice(state, name, probability);
+    },
+    choose(name, shapeOrChoices) {
+      return chooseScenario(state, name, shapeOrChoices);
+    },
+    note(name, value) {
+      const normalizedName = normalizeLabel(name, "note");
+      state.notes[normalizedName] = cloneForArtifact(value);
+      return value;
+    },
+    step(name, fn) {
+      return runScenarioStep(state, name, fn);
+    },
+    emitArtifact() {
+      emitScenarioArtifact(state);
+    },
+    snapshot() {
+      return buildScenarioArtifactPayload(state);
+    },
+  };
+}
+
+function createScenarioResource(state, name, factory, options = {}) {
+  const normalizedName = normalizeLabel(name, "resource");
+  if (typeof factory !== "function") {
+    throw new Error(`scenario.resource("${normalizedName}") requires a factory function`);
+  }
+
+  const scope = normalizeResourceScope(options.scope);
+  if (!state.resources.some((entry) => entry.name === normalizedName)) {
+    state.resources.push({ name: normalizedName, scope });
+  }
+
+  return {
+    get() {
+      if (scope === "step") {
+        return factory();
+      }
+
+      if (state.resourceState.has(normalizedName)) {
+        return state.resourceState.get(normalizedName);
+      }
+
+      const value = factory();
+      state.resourceState.set(normalizedName, value);
+      return value;
+    },
+  };
+}
+
+function pickChoice(state, name, choices) {
+  const normalizedName = normalizeLabel(name, "choice");
+  const values = Array.isArray(choices) ? choices : [];
+  if (values.length === 0) {
+    throw new Error(`scenario.pick("${normalizedName}") requires at least one choice`);
+  }
+
+  const index = chooseIndex(state.seed, state.scenarioName || "scenario", normalizedName, values.length);
+  const value = values[index];
+  state.choices[normalizedName] = cloneForArtifact(value);
+  return value;
+}
+
+function maybeChoice(state, name, probability = 0.5) {
+  const normalizedName = normalizeLabel(name, "choice");
+  const numericProbability = Number(probability);
+  if (!Number.isFinite(numericProbability) || numericProbability < 0 || numericProbability > 1) {
+    throw new Error(
+      `scenario.maybe("${normalizedName}") probability must be between 0 and 1`
+    );
+  }
+
+  const ratio = chooseRatio(state.seed, state.scenarioName || "scenario", normalizedName);
+  const value = ratio < numericProbability;
+  state.choices[normalizedName] = value;
+  return value;
+}
+
+function chooseScenario(state, name, shapeOrChoices) {
+  const normalizedName = normalizeLabel(name, "scenario");
+  state.scenarioName = normalizedName;
+
+  if (Array.isArray(shapeOrChoices)) {
+    const selected = pickChoice(state, `${normalizedName}:variant`, shapeOrChoices);
+    return selected;
+  }
+
+  if (!shapeOrChoices || typeof shapeOrChoices !== "object") {
+    throw new Error(`scenario.choose("${normalizedName}") requires an object or array`);
+  }
+
+  return shapeOrChoices;
+}
+
+function runScenarioStep(state, name, fn) {
+  const stepName = normalizeLabel(name, "unnamed step");
+  if (typeof fn !== "function") {
+    throw new Error(`scenario.step("${stepName}") requires a function`);
+  }
+
+  const entry = {
+    name: stepName,
+    startedAt: new Date().toISOString(),
+    durationMs: 0,
+    status: "passed",
+  };
+  state.steps.push(entry);
+
+  const startedAt = Date.now();
+  const before = getFailureCollectionSnapshot();
+
+  try {
+    return group(stepName, () => fn());
+  } catch (error) {
+    entry.status = "failed";
+    entry.error = error instanceof Error ? error.message : String(error);
+    throw error;
+  } finally {
+    const after = getFailureCollectionSnapshot();
+    entry.durationMs = Date.now() - startedAt;
+    entry.finishedAt = new Date().toISOString();
+    if (entry.status !== "failed" && after.failureCount > before.failureCount) {
+      entry.status = "failed";
+      entry.failureCount = after.failureCount - before.failureCount;
+    } else if (after.failureCount > before.failureCount) {
+      entry.failureCount = after.failureCount - before.failureCount;
+    }
+  }
+}
+
+function emitScenarioArtifact(state) {
+  const payload = buildScenarioArtifactPayload(state);
+  emitArtifact("scenario", payload, {
+    kind: "testkit.scenario",
+    summary: buildScenarioSummary(payload),
+  });
+}
+
+function buildScenarioArtifactPayload(state) {
+  return {
+    schemaVersion: 1,
+    seed: state.seed,
+    scenarioName: state.scenarioName,
+    choices: cloneForArtifact(state.choices),
+    notes: cloneForArtifact(state.notes),
+    resources: state.resources.map((entry) => ({ ...entry })),
+    steps: state.steps.map((entry) => ({ ...entry })),
+  };
+}
+
+function buildScenarioSummary(payload) {
+  const failedStep = payload.steps.find((entry) => entry.status === "failed");
+  if (failedStep) {
+    return `${payload.scenarioName || "scenario"} seed=${payload.seed} failed at ${failedStep.name}`;
+  }
+  return `${payload.scenarioName || "scenario"} seed=${payload.seed} (${payload.steps.length} steps)`;
+}
+
+function chooseIndex(seed, scenarioName, choiceName, length) {
+  if (length <= 1) return 0;
+  const value = hashString(`${seed}:${scenarioName}:${choiceName}`);
+  return value % length;
+}
+
+function chooseRatio(seed, scenarioName, choiceName) {
+  const value = hashString(`${seed}:${scenarioName}:${choiceName}`);
+  return value / 0xffffffff;
+}
+
+function hashString(input) {
+  let hash = 2166136261;
+  const source = String(input);
+  for (let index = 0; index < source.length; index += 1) {
+    hash ^= source.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return hash >>> 0;
+}
+
+function normalizeSeed(value) {
+  if (value === undefined || value === null) return DEFAULT_SCENARIO_SEED;
+  const normalized = String(value).trim();
+  return normalized.length > 0 ? normalized : DEFAULT_SCENARIO_SEED;
+}
+
+function normalizeLabel(value, fallback) {
+  if (typeof value !== "string") return fallback;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : fallback;
+}
+
+function normalizeResourceScope(value) {
+  const normalized = normalizeLabel(value, "scenario");
+  if (normalized === "file") return "file";
+  if (normalized === "scenario") return "scenario";
+  if (normalized === "step") return "step";
+  throw new Error(`Unsupported scenario resource scope "${value}"`);
+}
+
+function cloneForArtifact(value) {
+  if (value === undefined) return null;
+  return JSON.parse(JSON.stringify(value));
+}