@elench/testkit 0.1.55 → 0.1.57

package/lib/runner/planning.mjs

@@ -5,7 +5,7 @@ import {
   suiteSelectionType,
 } from "./suite-selection.mjs";
 
-const TYPE_ORDER = ["dal", "integration", "e2e", "load"];
+const TYPE_ORDER = ["dal", "integration", "e2e", "scenario", "load"];
 
 export function taskNeedsLocalRuntime(task) {
   return task.type !== "dal";

package/lib/runner/reporting.mjs

@@ -8,6 +8,7 @@ export function buildStatusArtifact({
   fileNames,
   shard,
   serviceFilter,
+  scenarioSeed,
   metadata,
 }) {
   const executedResults = results.filter((result) => !result.skipped);
@@ -68,6 +69,7 @@ export function buildStatusArtifact({
     fileNames: [...(fileNames || [])].sort(),
     shard: shard || null,
     serviceFilter: serviceFilter || null,
+    scenarioSeed: scenarioSeed || null,
   };
   scope.isFullRun =
     scope.types.length === 1 &&
@@ -109,6 +111,7 @@ export function buildRunArtifact({
   fileNames,
   shard,
   serviceFilter,
+  scenarioSeed,
   metadata,
   summarizeDbBackend,
   serviceLogs = [],
@@ -156,6 +159,7 @@ export function buildRunArtifact({
       fileNames,
       shard,
       serviceFilter,
+      scenarioSeed: scenarioSeed || null,
       testkitVersion: metadata.testkitVersion,
     },
     summary: {
@@ -224,6 +228,7 @@ export function buildLiveRunArtifact({
   fileNames,
   shard,
   serviceFilter,
+  scenarioSeed,
   metadata,
   summarizeDbBackend,
   serviceLogs = [],
@@ -244,6 +249,7 @@ export function buildLiveRunArtifact({
     fileNames,
     shard,
     serviceFilter,
+    scenarioSeed,
     metadata,
     summarizeDbBackend,
     serviceLogs,

package/lib/runner/reporting.test.mjs

@@ -62,6 +62,7 @@ describe("runner reporting", () => {
       fileNames: [],
       shard: null,
       serviceFilter: null,
+      scenarioSeed: "demo-seed",
       metadata: {
         git: {
           branch: "main",
@@ -84,6 +85,7 @@ describe("runner reporting", () => {
       fileTimeoutSeconds: 60,
       workerCount: 1,
       runtimeInstanceCount: 2,
+      scenarioSeed: "demo-seed",
     });
     expect(artifact.summary.services).toEqual({
       total: 1,
@@ -220,6 +222,7 @@ describe("runner reporting", () => {
       fileNames: ["tests/api/integration/b.int.testkit.ts"],
       shard: null,
       serviceFilter: "api",
+      scenarioSeed: "demo-seed",
       metadata: {
         git: {
           branch: "main",
@@ -247,6 +250,7 @@ describe("runner reporting", () => {
         fileNames: ["tests/api/integration/b.int.testkit.ts"],
         shard: null,
         serviceFilter: "api",
+        scenarioSeed: "demo-seed",
         isFullRun: false,
       },
       summary: {
@@ -291,6 +295,7 @@ describe("runner reporting", () => {
       fileNames: [],
       shard: null,
       serviceFilter: null,
+      scenarioSeed: null,
       metadata: {
         git: {
           branch: "main",

package/lib/runner/suite-selection.mjs

@@ -1,4 +1,4 @@
-const USER_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
+const USER_TYPES = new Set(["int", "e2e", "scenario", "dal", "load", "pw", "all"]);
 
 export function normalizeTypeValues(values = []) {
   const expanded = [];
@@ -9,7 +9,7 @@ export function normalizeTypeValues(values = []) {
       if (!value) continue;
       if (!USER_TYPES.has(value)) {
         throw new Error(
-          `Unknown type "${value}". Expected one of: int, e2e, dal, load, pw, all.`
+          `Unknown type "${value}". Expected one of: int, e2e, scenario, dal, load, pw, all.`
         );
       }
       expanded.push(value);
@@ -25,7 +25,7 @@ export function normalizeTypeValues(values = []) {
     throw new Error(`"--type all" cannot be combined with other types.`);
   }
 
-  const order = ["int", "e2e", "dal", "load", "pw", "all"];
+  const order = ["int", "e2e", "scenario", "dal", "load", "pw", "all"];
   return deduped.sort((left, right) => order.indexOf(left) - order.indexOf(right));
 }
 
@@ -52,7 +52,7 @@ export function parseSuiteSelectors(values = []) {
       const name = typeMatch[2].trim();
       if (!USER_TYPES.has(type) || type === "all") {
         throw new Error(
-          `Unknown suite selector type "${type}". Expected one of: int, e2e, dal, load, pw.`
+          `Unknown suite selector type "${type}". Expected one of: int, e2e, scenario, dal, load, pw.`
         );
       }
       if (!name) {

package/lib/runner/suite-selection.test.mjs

@@ -11,14 +11,20 @@ import {
 describe("runner suite selection", () => {
   it("normalizes selected type values", () => {
     expect(normalizeTypeValues([])).toEqual(["all"]);
-    expect(normalizeTypeValues(["int,e2e", "dal"])).toEqual(["int", "e2e", "dal"]);
+    expect(normalizeTypeValues(["int,e2e,scenario", "dal"])).toEqual([
+      "int",
+      "e2e",
+      "scenario",
+      "dal",
+    ]);
     expect(() => normalizeTypeValues(["all", "int"])).toThrow("cannot be combined");
     expect(() => normalizeTypeValues(["jest"])).toThrow("Unknown type");
   });
 
   it("parses suite selectors", () => {
-    expect(parseSuiteSelectors(["auth,dal:queries"])).toEqual([
+    expect(parseSuiteSelectors(["auth,scenario:journeys,dal:queries"])).toEqual([
       { kind: "plain", name: "auth", raw: "auth" },
+      { kind: "typed", type: "scenario", name: "journeys", raw: "scenario:journeys" },
       { kind: "typed", type: "dal", name: "queries", raw: "dal:queries" },
     ]);
     expect(() => parseSuiteSelectors(["all:auth"])).toThrow("Unknown suite selector type");
@@ -36,6 +42,7 @@ describe("runner suite selection", () => {
 
   it("maps discovered suites to user-facing selection types", () => {
     expect(suiteSelectionType("integration", "k6")).toBe("int");
+    expect(suiteSelectionType("scenario", "k6")).toBe("scenario");
     expect(suiteSelectionType("e2e", "playwright")).toBe("pw");
     expect(suiteSelectionType("dal", "k6")).toBe("dal");
   });

package/lib/runner/worker-loop.mjs

@@ -2,7 +2,7 @@ import { formatError } from "./formatting.mjs";
 import { runDalTask, runHttpK6Task } from "./default-runtime-runner.mjs";
 import { runPlaywrightTask } from "./playwright-runner.mjs";
 
-const HTTP_K6_TYPES = new Set(["integration", "e2e", "load"]);
+const HTTP_K6_TYPES = new Set(["integration", "e2e", "scenario", "load"]);
 
 export function createWorker(workerId, productDir) {
   return {

package/lib/runtime-src/k6/checks.js

@@ -136,6 +136,15 @@ export function recordFailureDetail(detail) {
   existing.count += normalized.count;
 }
 
+export function getFailureCollectionSnapshot() {
+  return {
+    phase: failureState.phase,
+    groupStack: [...failureState.groupStack],
+    failureCount: failureState.detailsByKey.size,
+    keys: [...failureState.detailsByKey.keys()].sort(),
+  };
+}
+
 function createFailureState() {
   return {
     phase: "exec",

package/lib/runtime-src/k6/scenario-runtime.js

@@ -0,0 +1,234 @@
+import { emitArtifact } from "./artifacts.js";
+import { getFailureCollectionSnapshot, group } from "./checks.js";
+
+const DEFAULT_SCENARIO_SEED = "default";
+
+export function createScenarioRuntime(options = {}) {
+  const suiteSeed = normalizeSeed(options.seed);
+  const state = {
+    seed: suiteSeed,
+    scenarioName: null,
+    choices: {},
+    notes: {},
+    resources: [],
+    resourceState: new Map(),
+    steps: [],
+  };
+
+  return {
+    get seed() {
+      return state.seed;
+    },
+    get scenarioName() {
+      return state.scenarioName;
+    },
+    resource(name, factory, options = {}) {
+      return createScenarioResource(state, name, factory, options);
+    },
+    pick(name, choices) {
+      return pickChoice(state, name, choices);
+    },
+    maybe(name, probability = 0.5) {
+      return maybeChoice(state, name, probability);
+    },
+    choose(name, shapeOrChoices) {
+      return chooseScenario(state, name, shapeOrChoices);
+    },
+    note(name, value) {
+      const normalizedName = normalizeLabel(name, "note");
+      state.notes[normalizedName] = cloneForArtifact(value);
+      return value;
+    },
+    step(name, fn) {
+      return runScenarioStep(state, name, fn);
+    },
+    emitArtifact() {
+      emitScenarioArtifact(state);
+    },
+    snapshot() {
+      return buildScenarioArtifactPayload(state);
+    },
+  };
+}
+
+function createScenarioResource(state, name, factory, options = {}) {
+  const normalizedName = normalizeLabel(name, "resource");
+  if (typeof factory !== "function") {
+    throw new Error(`scenario.resource("${normalizedName}") requires a factory function`);
+  }
+
+  const scope = normalizeResourceScope(options.scope);
+  if (!state.resources.some((entry) => entry.name === normalizedName)) {
+    state.resources.push({ name: normalizedName, scope });
+  }
+
+  return {
+    get() {
+      if (scope === "step") {
+        return factory();
+      }
+
+      if (state.resourceState.has(normalizedName)) {
+        return state.resourceState.get(normalizedName);
+      }
+
+      const value = factory();
+      state.resourceState.set(normalizedName, value);
+      return value;
+    },
+  };
+}
+
+function pickChoice(state, name, choices) {
+  const normalizedName = normalizeLabel(name, "choice");
+  const values = Array.isArray(choices) ? choices : [];
+  if (values.length === 0) {
+    throw new Error(`scenario.pick("${normalizedName}") requires at least one choice`);
+  }
+
+  const index = chooseIndex(state.seed, state.scenarioName || "scenario", normalizedName, values.length);
+  const value = values[index];
+  state.choices[normalizedName] = cloneForArtifact(value);
+  return value;
+}
+
+function maybeChoice(state, name, probability = 0.5) {
+  const normalizedName = normalizeLabel(name, "choice");
+  const numericProbability = Number(probability);
+  if (!Number.isFinite(numericProbability) || numericProbability < 0 || numericProbability > 1) {
+    throw new Error(
+      `scenario.maybe("${normalizedName}") probability must be between 0 and 1`
+    );
+  }
+
+  const ratio = chooseRatio(state.seed, state.scenarioName || "scenario", normalizedName);
+  const value = ratio < numericProbability;
+  state.choices[normalizedName] = value;
+  return value;
+}
+
+function chooseScenario(state, name, shapeOrChoices) {
+  const normalizedName = normalizeLabel(name, "scenario");
+  state.scenarioName = normalizedName;
+
+  if (Array.isArray(shapeOrChoices)) {
+    const selected = pickChoice(state, `${normalizedName}:variant`, shapeOrChoices);
+    return selected;
+  }
+
+  if (!shapeOrChoices || typeof shapeOrChoices !== "object") {
+    throw new Error(`scenario.choose("${normalizedName}") requires an object or array`);
+  }
+
+  return shapeOrChoices;
+}
+
+function runScenarioStep(state, name, fn) {
+  const stepName = normalizeLabel(name, "unnamed step");
+  if (typeof fn !== "function") {
+    throw new Error(`scenario.step("${stepName}") requires a function`);
+  }
+
+  const entry = {
+    name: stepName,
+    startedAt: new Date().toISOString(),
+    durationMs: 0,
+    status: "passed",
+  };
+  state.steps.push(entry);
+
+  const startedAt = Date.now();
+  const before = getFailureCollectionSnapshot();
+
+  try {
+    return group(stepName, () => fn());
+  } catch (error) {
+    entry.status = "failed";
+    entry.error = error instanceof Error ? error.message : String(error);
+    throw error;
+  } finally {
+    const after = getFailureCollectionSnapshot();
+    entry.durationMs = Date.now() - startedAt;
+    entry.finishedAt = new Date().toISOString();
+    if (entry.status !== "failed" && after.failureCount > before.failureCount) {
+      entry.status = "failed";
+      entry.failureCount = after.failureCount - before.failureCount;
+    } else if (after.failureCount > before.failureCount) {
+      entry.failureCount = after.failureCount - before.failureCount;
+    }
+  }
+}
+
+function emitScenarioArtifact(state) {
+  const payload = buildScenarioArtifactPayload(state);
+  emitArtifact("scenario", payload, {
+    kind: "testkit.scenario",
+    summary: buildScenarioSummary(payload),
+  });
+}
+
+function buildScenarioArtifactPayload(state) {
+  return {
+    schemaVersion: 1,
+    seed: state.seed,
+    scenarioName: state.scenarioName,
+    choices: cloneForArtifact(state.choices),
+    notes: cloneForArtifact(state.notes),
+    resources: state.resources.map((entry) => ({ ...entry })),
+    steps: state.steps.map((entry) => ({ ...entry })),
+  };
+}
+
+function buildScenarioSummary(payload) {
+  const failedStep = payload.steps.find((entry) => entry.status === "failed");
+  if (failedStep) {
+    return `${payload.scenarioName || "scenario"} seed=${payload.seed} failed at ${failedStep.name}`;
+  }
+  return `${payload.scenarioName || "scenario"} seed=${payload.seed} (${payload.steps.length} steps)`;
+}
+
+function chooseIndex(seed, scenarioName, choiceName, length) {
+  if (length <= 1) return 0;
+  const value = hashString(`${seed}:${scenarioName}:${choiceName}`);
+  return value % length;
+}
+
+function chooseRatio(seed, scenarioName, choiceName) {
+  const value = hashString(`${seed}:${scenarioName}:${choiceName}`);
+  return value / 0xffffffff;
+}
+
+function hashString(input) {
+  let hash = 2166136261;
+  const source = String(input);
+  for (let index = 0; index < source.length; index += 1) {
+    hash ^= source.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return hash >>> 0;
+}
+
+function normalizeSeed(value) {
+  if (value === undefined || value === null) return DEFAULT_SCENARIO_SEED;
+  const normalized = String(value).trim();
+  return normalized.length > 0 ? normalized : DEFAULT_SCENARIO_SEED;
+}
+
+function normalizeLabel(value, fallback) {
+  if (typeof value !== "string") return fallback;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : fallback;
+}
+
+function normalizeResourceScope(value) {
+  const normalized = normalizeLabel(value, "scenario");
+  if (normalized === "file") return "file";
+  if (normalized === "scenario") return "scenario";
+  if (normalized === "step") return "step";
+  throw new Error(`Unsupported scenario resource scope "${value}"`);
+}
+
+function cloneForArtifact(value) {
+  if (value === undefined) return null;
+  return JSON.parse(JSON.stringify(value));
+}

package/lib/runtime-src/k6/scenario-suite.js

@@ -0,0 +1,179 @@
+import { fail } from "k6";
+import {
+  defaultOptions,
+  emitFailureCollectionArtifact,
+  recordFailureDetail,
+  recordRuntimeFailure,
+  startFailureCollection,
+} from "./checks.js";
+import {
+  createHttpClient,
+  emitHttpTraceCollectionArtifact,
+  getEnv,
+  startHttpTraceCollection,
+} from "./http.js";
+import {
+  clearRuntimeContext,
+  registerRuntimeContext,
+  resolveHttpProfile,
+} from "../../setup/runtime.mjs";
+import { createScenarioRuntime } from "./scenario-runtime.js";
+
+export function defineScenarioSuite(configOrRun, maybeRun) {
+  const { config, run } = normalizeSuiteArgs(configOrRun, maybeRun);
+
+  return {
+    get options() {
+      return mergeProfileConfig(config).options || defaultOptions;
+    },
+    setup() {
+      const resolved = resolveRuntimeConfig(config);
+      startFailureCollection("setup");
+      startHttpTraceCollection("setup");
+      try {
+        registerRuntimeContext({ env: resolved.env, http: resolved.client.rawHttp || null });
+        if (typeof resolved.auth?.setup !== "function") return null;
+        return resolved.auth.setup({ env: resolved.env });
+      } catch (error) {
+        recordFailureDetail(buildRuntimeExceptionDetail("setup", error));
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("setup", error));
+      } finally {
+        emitFailureCollectionArtifact();
+        emitHttpTraceCollectionArtifact();
+        clearRuntimeContext();
+      }
+    },
+    exec(setupData) {
+      const resolved = resolveRuntimeConfig(config);
+      const scenario = createScenarioRuntime({
+        seed: resolved.env.rawEnv.TESTKIT_SCENARIO_SEED,
+      });
+      startFailureCollection("exec");
+      startHttpTraceCollection("exec");
+      try {
+        registerRuntimeContext({ env: resolved.env, http: resolved.client.rawHttp || null });
+        return run({
+          env: resolved.env,
+          req: resolved.client.request,
+          rawReq: resolved.client.raw,
+          getWithHeaders: resolved.client.getWithHeaders,
+          setupData,
+          session: setupData,
+          scenario,
+        });
+      } catch (error) {
+        recordFailureDetail(buildRuntimeExceptionDetail("exec", error));
+        recordRuntimeFailure();
+        fail(formatFatalSuiteError("exec", error));
+      } finally {
+        scenario.emitArtifact();
+        emitFailureCollectionArtifact();
+        emitHttpTraceCollectionArtifact();
+        clearRuntimeContext();
+      }
+    },
+  };
+}
+
+function normalizeSuiteArgs(configOrRun, maybeRun) {
+  if (typeof configOrRun === "function") {
+    return { config: {}, run: configOrRun };
+  }
+  if (typeof maybeRun !== "function") {
+    throw new Error("suite factory requires a run callback");
+  }
+  return { config: configOrRun || {}, run: maybeRun };
+}
+
+function callHeaders(builder, setupData, env) {
+  if (typeof builder !== "function") return {};
+  return builder(setupData, { env }) || {};
+}
+
+function mergeProfileConfig(config) {
+  if (!config?.profile) return config || {};
+
+  const profile = resolveHttpProfile(config.profile) || {};
+  return {
+    ...profile,
+    ...config,
+    auth: config.auth ?? profile.auth ?? null,
+    headers: config.headers ?? profile.headers,
+    rawHeaders: config.rawHeaders ?? profile.rawHeaders,
+    options: config.options ?? profile.options,
+    env: config.env ?? profile.env,
+  };
+}
+
+function resolveRuntimeConfig(config) {
+  const resolvedConfig = mergeProfileConfig(config);
+  const env = {
+    ...(resolvedConfig.env || getEnv()),
+    rawEnv: __ENV,
+  };
+  const auth = resolvedConfig.auth || null;
+  const client = createHttpClient({
+    baseUrl: env.BASE,
+    routeHeaders: env.routeParams,
+    getHeaders(setupData) {
+      return {
+        ...callHeaders(auth?.headers, setupData, env),
+        ...callHeaders(resolvedConfig.headers, setupData, env),
+      };
+    },
+    getRawHeaders(setupData) {
+      return callHeaders(resolvedConfig.rawHeaders, setupData, env);
+    },
+  });
+
+  return {
+    resolvedConfig,
+    env,
+    auth,
+    client,
+  };
+}
+
+function formatFatalSuiteError(phase, error) {
+  if (error instanceof Error) {
+    return `Uncaught testkit suite error during ${phase}: ${error.message}`;
+  }
+  return `Uncaught testkit suite error during ${phase}: ${String(error)}`;
+}
+
+function buildRuntimeExceptionDetail(phase, error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const stack = error instanceof Error && typeof error.stack === "string" ? error.stack : "";
+  const location = extractLocationFromStack(stack);
+  return {
+    kind: "runtime-exception",
+    key: location
+      ? `${location.path}:${location.line}:${location.column}`
+      : `runtime-exception:${phase}:${message}`,
+    title: "Uncaught runtime exception",
+    message: `Uncaught testkit suite error during ${phase}: ${message}`,
+    location,
+    stack,
+  };
+}
+
+function extractLocationFromStack(stack) {
+  if (!stack) return null;
+  const matches = [...String(stack).matchAll(/(file:\/\/[^\s)]+|\/[^\s):]+):(\d+):(\d+)/g)].map(
+    (match) => ({
+      path: normalizeStackPath(match[1]),
+      line: Number(match[2]),
+      column: Number(match[3]),
+    })
+  );
+  return matches[0] || null;
+}
+
+function normalizeStackPath(rawPath) {
+  if (typeof rawPath !== "string") return rawPath;
+  if (rawPath.startsWith("file://")) {
+    return rawPath.slice("file://".length);
+  }
+  return rawPath;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elench/testkit",
-  "version": "0.1.55",
+  "version": "0.1.57",
   "description": "CLI for discovering and running local HTTP, DAL, and Playwright test suites",
   "type": "module",
   "types": "./lib/index.d.ts",
@@ -17,6 +17,10 @@
       "types": "./lib/runtime/index.d.ts",
       "default": "./lib/runtime/index.mjs"
     },
+    "./discovery": {
+      "types": "./lib/discovery/index.d.ts",
+      "default": "./lib/discovery/index.mjs"
+    },
     "./known-failures": {
       "types": "./lib/known-failures/index.d.ts",
       "default": "./lib/known-failures/index.mjs"