@elench/testkit 0.1.54 → 0.1.56

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +22 -0
  2. package/lib/bundler/index.mjs +1 -1
  3. package/lib/bundler/index.test.mjs +29 -0
  4. package/lib/cli/args.mjs +2 -2
  5. package/lib/cli/args.test.mjs +8 -2
  6. package/lib/cli/command-helpers.mjs +5 -1
  7. package/lib/cli/commands/artifacts.mjs +2 -2
  8. package/lib/cli/commands/logs.mjs +2 -2
  9. package/lib/cli/commands/run.mjs +2 -2
  10. package/lib/cli/commands/show.mjs +2 -2
  11. package/lib/cli/db.mjs +17 -2
  12. package/lib/cli/entrypoint.mjs +2 -1
  13. package/lib/cli/presentation/run-reporter.mjs +25 -0
  14. package/lib/cli/presentation/run-reporter.test.mjs +80 -0
  15. package/lib/cli/tui/watch-app.mjs +134 -18
  16. package/lib/cli/viewer.mjs +67 -0
  17. package/lib/config/discovery.mjs +1 -0
  18. package/lib/config/discovery.test.mjs +8 -0
  19. package/lib/database/index.mjs +85 -11
  20. package/lib/database/template-steps.mjs +45 -6
  21. package/lib/database/template-steps.test.mjs +43 -0
  22. package/lib/index.d.ts +58 -0
  23. package/lib/index.mjs +3 -0
  24. package/lib/runner/artifacts.mjs +16 -0
  25. package/lib/runner/default-runtime-runner.mjs +4 -1
  26. package/lib/runner/logs.mjs +54 -6
  27. package/lib/runner/orchestrator.mjs +67 -14
  28. package/lib/runner/planning.mjs +1 -1
  29. package/lib/runner/reporting.mjs +58 -2
  30. package/lib/runner/reporting.test.mjs +85 -2
  31. package/lib/runner/runtime-contexts.mjs +3 -3
  32. package/lib/runner/runtime-preparation.mjs +31 -0
  33. package/lib/runner/setup-operations.mjs +115 -0
  34. package/lib/runner/setup-operations.test.mjs +94 -0
  35. package/lib/runner/suite-selection.mjs +4 -4
  36. package/lib/runner/suite-selection.test.mjs +9 -2
  37. package/lib/runner/template-steps.mjs +129 -11
  38. package/lib/runner/worker-loop.mjs +1 -1
  39. package/lib/runtime-src/k6/checks.js +9 -0
  40. package/lib/runtime-src/k6/scenario-runtime.js +234 -0
  41. package/lib/runtime-src/k6/scenario-suite.js +179 -0
  42. package/lib/toolchains/index.mjs +0 -4
  43. package/package.json +1 -1
@@ -16,6 +16,14 @@ export function loadLatestRunArtifact(productDir) {
16
16
  return JSON.parse(fs.readFileSync(artifactPath, "utf8"));
17
17
  }
18
18
 
19
+ export function loadCurrentRunArtifact(productDir) {
20
+ const livePath = path.join(productDir, ".testkit", "results", "live.json");
21
+ if (fs.existsSync(livePath)) {
22
+ return JSON.parse(fs.readFileSync(livePath, "utf8"));
23
+ }
24
+ return loadLatestRunArtifact(productDir);
25
+ }
26
+
19
27
  export function resolveFileSubject(runArtifact, selector = null, serviceFilter = null) {
20
28
  const files = collectFiles(runArtifact, serviceFilter);
21
29
  if (files.length === 0) {
@@ -108,6 +116,25 @@ export function formatFileDetail(productDir, runArtifact, subject, options = {})
108
116
  for (const line of triageLines) lines.push(` ${line}`);
109
117
  }
110
118
 
119
+ const setupOperations = getSetupOperationsForService(runArtifact, subject.service.name);
120
+ if (setupOperations.length > 0) {
121
+ lines.push("");
122
+ lines.push("Setup:");
123
+ for (const operation of setupOperations.slice(0, 8)) {
124
+ const duration = operation.durationMs == null ? "" : ` ${formatDuration(operation.durationMs)}`;
125
+ const suffix = operation.summary ? ` ${operation.summary}` : "";
126
+ lines.push(` ${operation.status} ${operation.stage}${duration}${suffix}`);
127
+ if (operation.logRef?.path) lines.push(` ${operation.logRef.path}`);
128
+ if (operation.error) lines.push(` ${operation.error}`);
129
+ if (operation.logRef?.path) {
130
+ const setupLogPath = path.join(productDir, operation.logRef.path);
131
+ for (const line of readLogTail(setupLogPath, 4).slice(-4)) {
132
+ lines.push(` ${line}`);
133
+ }
134
+ }
135
+ }
136
+ }
137
+
111
138
  const artifacts = collectArtifactEntries(productDir, runArtifact, subject.file.path, subject.service.name);
112
139
  if (artifacts.length > 0) {
113
140
  lines.push("");
@@ -148,11 +175,24 @@ export function getServiceLogRefs(runArtifact, serviceName) {
148
175
  return (runArtifact.logs?.services || []).filter((entry) => entry.serviceName === serviceName);
149
176
  }
150
177
 
178
+ export function getSetupOperationsForService(runArtifact, serviceName) {
179
+ return (runArtifact.setup?.operations || [])
180
+ .filter((entry) => entry.serviceName === serviceName)
181
+ .sort(
182
+ (left, right) =>
183
+ String(left.startedAt || "").localeCompare(String(right.startedAt || "")) ||
184
+ String(left.stage || "").localeCompare(String(right.stage || ""))
185
+ );
186
+ }
187
+
151
188
  export function formatArtifactPreview(payload, maxLines = 6) {
152
189
  if (!payload) return ["artifact payload missing"];
153
190
  if (payload.kind === "agentic-query") {
154
191
  return formatAgenticArtifact(payload, maxLines);
155
192
  }
193
+ if (payload.kind === "testkit.scenario") {
194
+ return formatScenarioArtifact(payload, maxLines);
195
+ }
156
196
  if (payload.kind === "testkit.http-traces") {
157
197
  return formatHttpTraceArtifact(payload, maxLines);
158
198
  }
@@ -193,12 +233,39 @@ function formatHttpTraceArtifact(payload, maxLines) {
193
233
  return lines;
194
234
  }
195
235
 
236
+ function formatScenarioArtifact(payload, maxLines) {
237
+ const artifact = payload.data || {};
238
+ const lines = [];
239
+ if (artifact.scenarioName) lines.push(`Scenario: ${artifact.scenarioName}`);
240
+ if (artifact.seed) lines.push(`Seed: ${artifact.seed}`);
241
+ const choiceEntries = Object.entries(artifact.choices || {});
242
+ if (choiceEntries.length > 0) {
243
+ lines.push(
244
+ `Choices: ${choiceEntries
245
+ .map(([key, value]) => `${key}=${formatScenarioChoiceValue(value)}`)
246
+ .join(", ")}`
247
+ );
248
+ }
249
+ const failedStep = (artifact.steps || []).find((step) => step.status === "failed");
250
+ if (failedStep) {
251
+ lines.push(`Failed Step: ${failedStep.name}`);
252
+ } else if (Array.isArray(artifact.steps) && artifact.steps.length > 0) {
253
+ lines.push(`Steps: ${artifact.steps.map((step) => step.name).join(" -> ")}`);
254
+ }
255
+ return lines.slice(0, maxLines);
256
+ }
257
+
196
258
  function rankFailureDetails(details) {
197
259
  return [...(Array.isArray(details) ? details : [])].sort((left, right) => {
198
260
  return failureDetailRank(left) - failureDetailRank(right) || String(left?.key || "").localeCompare(String(right?.key || ""));
199
261
  });
200
262
  }
201
263
 
264
+ function formatScenarioChoiceValue(value) {
265
+ if (typeof value === "string") return value;
266
+ return JSON.stringify(value);
267
+ }
268
+
202
269
  function failureDetailRank(detail) {
203
270
  if (detail?.kind === "http-assertion") return 1;
204
271
  if (detail?.request && detail?.response) return 2;
@@ -5,6 +5,7 @@ const TESTKIT_DIRNAME = "__testkit__";
5
5
  const DISCOVERY_RULES = [
6
6
  { suffix: ".int.testkit.ts", type: "integration", framework: "k6" },
7
7
  { suffix: ".e2e.testkit.ts", type: "e2e", framework: "k6" },
8
+ { suffix: ".scenario.testkit.ts", type: "scenario", framework: "k6" },
8
9
  { suffix: ".dal.testkit.ts", type: "dal", framework: "k6" },
9
10
  { suffix: ".load.testkit.ts", type: "load", framework: "k6" },
10
11
  { suffix: ".pw.testkit.ts", type: "e2e", framework: "playwright" },
@@ -19,6 +19,7 @@ describe("filesystem-discovery", () => {
19
19
 
20
20
  writeFile(productDir, "src/api/routes/__testkit__/auth/me.int.testkit.ts");
21
21
  writeFile(productDir, "src/api/routes/__testkit__/health/ready.int.testkit.ts");
22
+ writeFile(productDir, "src/api/routes/__testkit__/journeys/smoke.scenario.testkit.ts");
22
23
  writeFile(productDir, "frontend/app/__testkit__/homepage/homepage.pw.testkit.ts");
23
24
 
24
25
  const suites = discoverSuites(productDir, {
@@ -46,6 +47,13 @@ describe("filesystem-discovery", () => {
46
47
  framework: "k6",
47
48
  },
48
49
  ]);
50
+ expect(suites.api.scenario).toEqual([
51
+ {
52
+ name: "journeys",
53
+ files: ["src/api/routes/__testkit__/journeys/smoke.scenario.testkit.ts"],
54
+ framework: "k6",
55
+ },
56
+ ]);
49
57
  expect(suites.frontend.e2e).toEqual([
50
58
  {
51
59
  name: "homepage",
@@ -36,25 +36,25 @@ const LOCAL_POLL_INTERVAL_MS = 1_000;
36
36
  const LOCAL_DROP_DATABASE_TIMEOUT_MS = 15_000;
37
37
  const LOCAL_DROP_DATABASE_POLL_INTERVAL_MS = 250;
38
38
 
39
- export async function prepareDatabaseRuntime(config) {
39
+ export async function prepareDatabaseRuntime(config, options = {}) {
40
40
  const db = config.testkit.database;
41
41
  if (!db) return;
42
42
 
43
43
  fs.mkdirSync(config.stateDir, { recursive: true });
44
44
  if (db.provider === "local") {
45
- await prepareLocalDatabase(config);
45
+ await prepareLocalDatabase(config, options);
46
46
  return;
47
47
  }
48
48
 
49
49
  throw new Error(`Unsupported database provider "${db.provider}"`);
50
50
  }
51
51
 
52
- export async function captureDatabaseTemplateSnapshot(config, outputPath) {
52
+ export async function captureDatabaseTemplateSnapshot(config, outputPath, options = {}) {
53
53
  if (!config.testkit.database || config.testkit.database.provider !== "local") {
54
54
  throw new Error(`Service "${config.name}" does not use a local testkit database`);
55
55
  }
56
56
 
57
- await prepareDatabaseRuntime(config);
57
+ await prepareDatabaseRuntime(config, options);
58
58
  const cacheDir = getLocalServiceCacheDir(config.productDir, config.name);
59
59
  const templateDbName = readStateValue(path.join(cacheDir, "template_database_name"));
60
60
  if (!templateDbName) {
@@ -66,7 +66,41 @@ export async function captureDatabaseTemplateSnapshot(config, outputPath) {
66
66
  throw new Error(`Missing local database container for service "${config.name}"`);
67
67
  }
68
68
 
69
- return captureTemplateSnapshot(config, outputPath, buildDatabaseUrl(infra, templateDbName));
69
+ const snapshotOperation = options.setupRegistry?.start({
70
+ config,
71
+ stage: "template:snapshot",
72
+ kind: "database-snapshot",
73
+ summary: `snapshot: ${path.relative(config.productDir, outputPath)}`,
74
+ });
75
+ try {
76
+ const output = await captureTemplateSnapshot(
77
+ config,
78
+ outputPath,
79
+ buildDatabaseUrl(infra, templateDbName),
80
+ {
81
+ reporter: options.reporter || null,
82
+ logRecord: snapshotOperation?._logRecord || null,
83
+ }
84
+ );
85
+ const finished = snapshotOperation
86
+ ? options.setupRegistry.finish(snapshotOperation, {
87
+ status: "passed",
88
+ summary: `snapshot: ${path.relative(config.productDir, outputPath)}`,
89
+ })
90
+ : null;
91
+ if (finished) options.reporter?.setupOperationFinished?.(finished);
92
+ return output;
93
+ } catch (error) {
94
+ const finished = snapshotOperation
95
+ ? options.setupRegistry.finish(snapshotOperation, {
96
+ status: "failed",
97
+ summary: `snapshot: ${path.relative(config.productDir, outputPath)}`,
98
+ error: error?.message || error,
99
+ })
100
+ : null;
101
+ if (finished) options.reporter?.setupOperationFinished?.(finished);
102
+ throw error;
103
+ }
70
104
  }
71
105
 
72
106
  export async function destroyRuntimeDatabase({ productDir, stateDir }) {
@@ -138,7 +172,7 @@ export function showServiceDatabaseStatus(productDir, serviceName) {
138
172
  return true;
139
173
  }
140
174
 
141
- async function prepareLocalDatabase(config) {
175
+ async function prepareLocalDatabase(config, options = {}) {
142
176
  const db = config.testkit.database;
143
177
  const productDir = config.productDir;
144
178
  const serviceName = config.name;
@@ -154,7 +188,7 @@ async function prepareLocalDatabase(config) {
154
188
  );
155
189
 
156
190
  await withLock(path.join(lockDir, `template-${serviceName}.lock`), async () => {
157
- await ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint);
191
+ await ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint, options);
158
192
  });
159
193
 
160
194
  await withLock(path.join(lockDir, `runtime-${serviceName}-${hashString(bindingKey, 10)}.lock`), async () => {
@@ -162,7 +196,7 @@ async function prepareLocalDatabase(config) {
162
196
  });
163
197
  }
164
198
 
165
- async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint) {
199
+ async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint, options = {}) {
166
200
  const serviceName = config.name;
167
201
  const existingFingerprint = readStateValue(path.join(cacheDir, "template_fingerprint"));
168
202
  const existingDbName = readStateValue(path.join(cacheDir, "template_database_name"));
@@ -173,6 +207,12 @@ async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerpri
173
207
  existingDbName &&
174
208
  (await databaseExists(infra, existingDbName))
175
209
  ) {
210
+ options.setupRegistry?.recordCached({
211
+ config,
212
+ stage: "template",
213
+ kind: "database-template",
214
+ summary: "template cache hit",
215
+ });
176
216
  writeLocalCacheState(cacheDir, infra, existingDbName, templateFingerprint);
177
217
  return;
178
218
  }
@@ -186,11 +226,45 @@ async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerpri
186
226
 
187
227
  const templateUrl = buildDatabaseUrl(infra, desiredDbName);
188
228
  await createEmptyDatabase(infra, desiredDbName);
229
+ const templateOperation = options.setupRegistry?.start({
230
+ config,
231
+ stage: "template",
232
+ kind: "database-template",
233
+ summary: "template rebuild",
234
+ recordLog: false,
235
+ });
189
236
  try {
190
- await runTemplateStage(config, "migrate", templateUrl);
191
- await runTemplateStage(config, "seed", templateUrl);
192
- await runTemplateStage(config, "verify", templateUrl);
237
+ await runTemplateStage(config, "migrate", templateUrl, {
238
+ reporter: options.reporter || null,
239
+ setupRegistry: options.setupRegistry || null,
240
+ parentOperation: templateOperation,
241
+ });
242
+ await runTemplateStage(config, "seed", templateUrl, {
243
+ reporter: options.reporter || null,
244
+ setupRegistry: options.setupRegistry || null,
245
+ parentOperation: templateOperation,
246
+ });
247
+ await runTemplateStage(config, "verify", templateUrl, {
248
+ reporter: options.reporter || null,
249
+ setupRegistry: options.setupRegistry || null,
250
+ parentOperation: templateOperation,
251
+ });
252
+ const finished = templateOperation
253
+ ? options.setupRegistry.finish(templateOperation, {
254
+ status: "passed",
255
+ summary: "template rebuild",
256
+ })
257
+ : null;
258
+ if (finished) options.reporter?.setupOperationFinished?.(finished);
193
259
  } catch (error) {
260
+ const finished = templateOperation
261
+ ? options.setupRegistry.finish(templateOperation, {
262
+ status: "failed",
263
+ summary: "template rebuild",
264
+ error: error?.message || error,
265
+ })
266
+ : null;
267
+ if (finished) options.reporter?.setupOperationFinished?.(finished);
194
268
  await dropDatabaseIfExists(infra, desiredDbName);
195
269
  throw error;
196
270
  }
@@ -6,8 +6,9 @@ import {
6
6
  collectConfiguredInputs,
7
7
  runConfiguredSteps,
8
8
  } from "../runner/template-steps.mjs";
9
+ import { captureOutput } from "../runner/processes.mjs";
9
10
 
10
- export async function runTemplateStage(config, stageName, databaseUrl) {
11
+ export async function runTemplateStage(config, stageName, databaseUrl, options = {}) {
11
12
  const steps = config.testkit.database?.template?.[stageName] || [];
12
13
  if (steps.length === 0) return;
13
14
 
@@ -21,6 +22,9 @@ export async function runTemplateStage(config, stageName, databaseUrl) {
21
22
  steps,
22
23
  env,
23
24
  labelPrefix: `template:${stageName}`,
25
+ reporter: options.reporter || null,
26
+ setupRegistry: options.setupRegistry || null,
27
+ parentOperation: options.parentOperation || null,
24
28
  });
25
29
  }
26
30
 
@@ -32,12 +36,12 @@ export function collectTemplateInputs(productDir, template = {}) {
32
36
  });
33
37
  }
34
38
 
35
- export async function captureTemplateSnapshot(config, outputPath, databaseUrl) {
39
+ export async function captureTemplateSnapshot(config, outputPath, databaseUrl, options = {}) {
36
40
  const templateDbUrl = databaseUrl;
37
41
  const absoluteOutputPath = path.resolve(config.productDir, outputPath);
38
42
  fs.mkdirSync(path.dirname(absoluteOutputPath), { recursive: true });
39
43
 
40
- await execa(
44
+ const child = execa(
41
45
  "pg_dump",
42
46
  [
43
47
  "--schema-only",
@@ -53,19 +57,54 @@ export async function captureTemplateSnapshot(config, outputPath, databaseUrl) {
53
57
  ...buildExecutionEnv(config, {}, process.env),
54
58
  DATABASE_URL: templateDbUrl,
55
59
  },
56
- stdio: "inherit",
60
+ stdout: "pipe",
61
+ stderr: "pipe",
62
+ reject: false,
57
63
  }
58
64
  );
65
+ const liveWriter =
66
+ options.reporter?.outputMode === "debug"
67
+ ? (line) => options.reporter.writeDebugLine?.(line)
68
+ : null;
69
+ const logRecord = options.logRecord || null;
70
+ const drains = [
71
+ captureOutput(child.stdout, {
72
+ livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
73
+ liveWriter,
74
+ onLine(line) {
75
+ if (logRecord) logRecord.stream.write(`${new Date().toISOString()} [stdout] ${line}\n`);
76
+ },
77
+ }),
78
+ captureOutput(child.stderr, {
79
+ livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
80
+ liveWriter,
81
+ onLine(line) {
82
+ if (logRecord) logRecord.stream.write(`${new Date().toISOString()} [stderr] ${line}\n`);
83
+ },
84
+ }),
85
+ ];
86
+ const result = await child;
87
+ await Promise.all(drains);
88
+ if (result.exitCode !== 0) {
89
+ throw new Error(result.shortMessage || result.stderr || result.stdout || "pg_dump failed");
90
+ }
59
91
 
60
92
  sanitizeSnapshotFile(absoluteOutputPath);
61
93
  return absoluteOutputPath;
62
94
  }
63
95
 
64
- function sanitizeSnapshotFile(filePath) {
96
+ export function sanitizeSnapshotFile(filePath) {
65
97
  const dump = fs.readFileSync(filePath, "utf8");
66
98
  const sanitized = dump
67
99
  .split("\n")
68
- .filter((line) => line.trim() !== "SET transaction_timeout = 0;")
100
+ .filter((line) => {
101
+ const trimmed = line.trim();
102
+ return (
103
+ trimmed !== "SET transaction_timeout = 0;" &&
104
+ !trimmed.startsWith("\\restrict ") &&
105
+ !trimmed.startsWith("\\unrestrict ")
106
+ );
107
+ })
69
108
  .join("\n");
70
109
 
71
110
  if (sanitized !== dump) {
@@ -0,0 +1,43 @@
1
+ import fs from "fs";
2
+ import os from "os";
3
+ import path from "path";
4
+ import { afterEach, describe, expect, it } from "vitest";
5
+ import { sanitizeSnapshotFile } from "./template-steps.mjs";
6
+
7
+ const tempDirs = [];
8
+
9
+ afterEach(() => {
10
+ while (tempDirs.length > 0) {
11
+ fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
12
+ }
13
+ });
14
+
15
+ function makeTempDir(prefix) {
16
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
17
+ tempDirs.push(dir);
18
+ return dir;
19
+ }
20
+
21
+ describe("template snapshot sanitization", () => {
22
+ it("removes volatile pg_dump control lines", () => {
23
+ const dir = makeTempDir("testkit-template-snapshot-");
24
+ const filePath = path.join(dir, "schema.sql");
25
+ fs.writeFileSync(
26
+ filePath,
27
+ [
28
+ "SET statement_timeout = 0;",
29
+ "SET transaction_timeout = 0;",
30
+ "\\restrict abc123",
31
+ "CREATE TABLE public.widgets (id integer);",
32
+ "\\unrestrict abc123",
33
+ "",
34
+ ].join("\n")
35
+ );
36
+
37
+ sanitizeSnapshotFile(filePath);
38
+
39
+ expect(fs.readFileSync(filePath, "utf8")).toBe(
40
+ ["SET statement_timeout = 0;", "CREATE TABLE public.widgets (id integer);", ""].join("\n")
41
+ );
42
+ });
43
+ });
package/lib/index.d.ts CHANGED
@@ -38,6 +38,55 @@ export interface HttpSuiteContext<TSetup = unknown> {
38
38
  session: TSetup | null;
39
39
  }
40
40
 
41
+ export interface ScenarioStepResult {
42
+ name: string;
43
+ status: "passed" | "failed";
44
+ startedAt?: string;
45
+ finishedAt?: string;
46
+ durationMs?: number;
47
+ failureCount?: number;
48
+ error?: string;
49
+ }
50
+
51
+ export interface ScenarioResource<TValue = unknown> {
52
+ get(): TValue;
53
+ }
54
+
55
+ export interface ScenarioRuntime {
56
+ readonly seed: string;
57
+ readonly scenarioName: string | null;
58
+ choose<TChoice extends unknown[]>(
59
+ name: string,
60
+ choices: TChoice
61
+ ): TChoice[number];
62
+ choose<TShape extends Record<string, unknown>>(
63
+ name: string,
64
+ shape: TShape
65
+ ): TShape;
66
+ maybe(name: string, probability?: number): boolean;
67
+ note<TValue = unknown>(name: string, value: TValue): TValue;
68
+ pick<TChoice extends unknown[]>(name: string, choices: TChoice): TChoice[number];
69
+ resource<TValue = unknown>(
70
+ name: string,
71
+ factory: () => TValue,
72
+ options?: { scope?: "file" | "scenario" | "step" }
73
+ ): ScenarioResource<TValue>;
74
+ step<TValue = unknown>(name: string, fn: () => TValue): TValue;
75
+ snapshot(): {
76
+ schemaVersion: number;
77
+ seed: string;
78
+ scenarioName: string | null;
79
+ choices: Record<string, unknown>;
80
+ notes: Record<string, unknown>;
81
+ resources: Array<{ name: string; scope: "file" | "scenario" | "step" }>;
82
+ steps: ScenarioStepResult[];
83
+ };
84
+ }
85
+
86
+ export interface ScenarioSuiteContext<TSetup = unknown> extends HttpSuiteContext<TSetup> {
87
+ scenario: ScenarioRuntime;
88
+ }
89
+
41
90
  export interface HttpSuiteConfig<TSetup = unknown> {
42
91
  auth?: AuthAdapter<TSetup> | null;
43
92
  env?: RuntimeEnv;
@@ -68,6 +117,15 @@ export declare function defineHttpSuite<TSetup = unknown>(
68
117
  run: (context: HttpSuiteContext<TSetup>) => unknown
69
118
  ): TestkitSuite<TSetup>;
70
119
 
120
+ export declare function defineScenarioSuite<TSetup = unknown>(
121
+ run: (context: ScenarioSuiteContext<TSetup>) => unknown
122
+ ): TestkitSuite<TSetup>;
123
+
124
+ export declare function defineScenarioSuite<TSetup = unknown>(
125
+ config: HttpSuiteConfig<TSetup>,
126
+ run: (context: ScenarioSuiteContext<TSetup>) => unknown
127
+ ): TestkitSuite<TSetup>;
128
+
71
129
  export declare function defineDalSuite<TSetup = unknown>(
72
130
  run: (context: DalSuiteContext<TSetup>) => unknown
73
131
  ): TestkitSuite<TSetup>;
package/lib/index.mjs CHANGED
@@ -4,6 +4,9 @@ export {
4
4
  export {
5
5
  defineHttpSuite,
6
6
  } from "./runtime-src/k6/suite.js";
7
+ export {
8
+ defineScenarioSuite,
9
+ } from "./runtime-src/k6/scenario-suite.js";
7
10
 
8
11
  export function createAuthAdapter({ setup, headers } = {}) {
9
12
  return {
@@ -9,11 +9,20 @@ import {
9
9
  const TIMINGS_FILENAME = "timings.json";
10
10
  const RESULT_ARTIFACTS_DIRNAME = "artifacts";
11
11
  const RESULT_LOGS_DIRNAME = "logs";
12
+ const RESULT_SETUP_DIRNAME = "setup";
13
+ const LIVE_ARTIFACT_FILENAME = "live.json";
12
14
 
13
15
  export function writeRunArtifact(productDir, artifact) {
14
16
  const resultsDir = path.join(productDir, ".testkit", "results");
15
17
  fs.mkdirSync(resultsDir, { recursive: true });
16
18
  fs.writeFileSync(path.join(resultsDir, "latest.json"), JSON.stringify(artifact, null, 2));
19
+ fs.rmSync(path.join(resultsDir, LIVE_ARTIFACT_FILENAME), { force: true });
20
+ }
21
+
22
+ export function writeLiveRunArtifact(productDir, artifact) {
23
+ const resultsDir = path.join(productDir, ".testkit", "results");
24
+ fs.mkdirSync(resultsDir, { recursive: true });
25
+ fs.writeFileSync(path.join(resultsDir, LIVE_ARTIFACT_FILENAME), JSON.stringify(artifact, null, 2));
17
26
  }
18
27
 
19
28
  export function writeStatusArtifact(productDir, artifact) {
@@ -32,6 +41,13 @@ export function resetResultArtifacts(productDir) {
32
41
  recursive: true,
33
42
  force: true,
34
43
  });
44
+ fs.rmSync(path.join(productDir, ".testkit", "results", RESULT_SETUP_DIRNAME), {
45
+ recursive: true,
46
+ force: true,
47
+ });
48
+ fs.rmSync(path.join(productDir, ".testkit", "results", LIVE_ARTIFACT_FILENAME), {
49
+ force: true,
50
+ });
35
51
  }
36
52
 
37
53
  export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
@@ -95,7 +95,10 @@ export async function runDefaultRuntimeTask(
95
95
  env: buildTaskExecutionEnv(
96
96
  targetConfig,
97
97
  lease,
98
- buildFileTimeoutEnv(fileTimeoutSeconds, startedAt),
98
+ {
99
+ ...buildFileTimeoutEnv(fileTimeoutSeconds, startedAt),
100
+ ...(task.scenarioSeed ? { TESTKIT_SCENARIO_SEED: task.scenarioSeed } : {}),
101
+ },
99
102
  process.env
100
103
  ),
101
104
  reject: false,
@@ -2,21 +2,26 @@ import fs from "fs";
2
2
  import path from "path";
3
3
 
4
4
  const RESULT_LOGS_DIRNAME = "logs";
5
+ const RESULT_SETUP_DIRNAME = "setup";
5
6
 
6
7
  export function createRunLogRegistry(productDir) {
7
- const records = new Map();
8
+ const serviceRecords = new Map();
9
+ const setupRecords = new Map();
8
10
 
9
11
  return {
10
12
  ensureServiceLogRecord(config) {
11
13
  const key = `${config.runtimeLabel || config.name}:${config.name}`;
12
- const existing = records.get(key);
14
+ const existing = serviceRecords.get(key);
13
15
  if (existing) return existing;
14
16
 
15
17
  const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}.log`;
16
18
  const relativePath = path.join(".testkit", "results", RESULT_LOGS_DIRNAME, fileName);
17
19
  const absolutePath = path.join(productDir, relativePath);
18
20
  fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
19
- const stream = fs.createWriteStream(absolutePath, { flags: "a" });
21
+ const stream = fs.createWriteStream(absolutePath, {
22
+ fd: fs.openSync(absolutePath, "a"),
23
+ flags: "a",
24
+ });
20
25
  const record = {
21
26
  key,
22
27
  serviceName: config.name,
@@ -25,7 +30,32 @@ export function createRunLogRegistry(productDir) {
25
30
  absolutePath,
26
31
  stream,
27
32
  };
28
- records.set(key, record);
33
+ serviceRecords.set(key, record);
34
+ return record;
35
+ },
36
+ ensureSetupLogRecord(config, stage) {
37
+ const key = `${config.runtimeLabel || config.name}:${config.name}:${stage}`;
38
+ const existing = setupRecords.get(key);
39
+ if (existing) return existing;
40
+
41
+ const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}__${sanitizePathSegment(stage)}.log`;
42
+ const relativePath = path.join(".testkit", "results", RESULT_SETUP_DIRNAME, fileName);
43
+ const absolutePath = path.join(productDir, relativePath);
44
+ fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
45
+ const stream = fs.createWriteStream(absolutePath, {
46
+ fd: fs.openSync(absolutePath, "a"),
47
+ flags: "a",
48
+ });
49
+ const record = {
50
+ key,
51
+ serviceName: config.name,
52
+ runtimeLabel: config.runtimeLabel || config.name,
53
+ stage,
54
+ path: normalizePath(relativePath),
55
+ absolutePath,
56
+ stream,
57
+ };
58
+ setupRecords.set(key, record);
29
59
  return record;
30
60
  },
31
61
  append(record, streamName, line) {
@@ -33,7 +63,7 @@ export function createRunLogRegistry(productDir) {
33
63
  record.stream.write(`${new Date().toISOString()} [${streamName}] ${line}\n`);
34
64
  },
35
65
  listServiceLogs() {
36
- return [...records.values()]
66
+ return [...serviceRecords.values()]
37
67
  .map((record) => ({
38
68
  serviceName: record.serviceName,
39
69
  runtimeLabel: record.runtimeLabel,
@@ -45,8 +75,26 @@ export function createRunLogRegistry(productDir) {
45
75
  left.runtimeLabel.localeCompare(right.runtimeLabel)
46
76
  );
47
77
  },
78
+ listSetupLogs() {
79
+ return [...setupRecords.values()]
80
+ .map((record) => ({
81
+ serviceName: record.serviceName,
82
+ runtimeLabel: record.runtimeLabel,
83
+ stage: record.stage,
84
+ path: record.path,
85
+ }))
86
+ .sort(
87
+ (left, right) =>
88
+ left.serviceName.localeCompare(right.serviceName) ||
89
+ left.runtimeLabel.localeCompare(right.runtimeLabel) ||
90
+ left.stage.localeCompare(right.stage)
91
+ );
92
+ },
48
93
  closeAll() {
49
- for (const record of records.values()) {
94
+ for (const record of serviceRecords.values()) {
95
+ record.stream.end();
96
+ }
97
+ for (const record of setupRecords.values()) {
50
98
  record.stream.end();
51
99
  }
52
100
  },