@elench/testkit 0.1.54 → 0.1.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/lib/bundler/index.mjs +1 -1
- package/lib/bundler/index.test.mjs +29 -0
- package/lib/cli/args.mjs +2 -2
- package/lib/cli/args.test.mjs +8 -2
- package/lib/cli/command-helpers.mjs +5 -1
- package/lib/cli/commands/artifacts.mjs +2 -2
- package/lib/cli/commands/logs.mjs +2 -2
- package/lib/cli/commands/run.mjs +2 -2
- package/lib/cli/commands/show.mjs +2 -2
- package/lib/cli/db.mjs +17 -2
- package/lib/cli/entrypoint.mjs +2 -1
- package/lib/cli/presentation/run-reporter.mjs +25 -0
- package/lib/cli/presentation/run-reporter.test.mjs +80 -0
- package/lib/cli/tui/watch-app.mjs +134 -18
- package/lib/cli/viewer.mjs +67 -0
- package/lib/config/discovery.mjs +1 -0
- package/lib/config/discovery.test.mjs +8 -0
- package/lib/database/index.mjs +85 -11
- package/lib/database/template-steps.mjs +45 -6
- package/lib/database/template-steps.test.mjs +43 -0
- package/lib/index.d.ts +58 -0
- package/lib/index.mjs +3 -0
- package/lib/runner/artifacts.mjs +16 -0
- package/lib/runner/default-runtime-runner.mjs +4 -1
- package/lib/runner/logs.mjs +54 -6
- package/lib/runner/orchestrator.mjs +67 -14
- package/lib/runner/planning.mjs +1 -1
- package/lib/runner/reporting.mjs +58 -2
- package/lib/runner/reporting.test.mjs +85 -2
- package/lib/runner/runtime-contexts.mjs +3 -3
- package/lib/runner/runtime-preparation.mjs +31 -0
- package/lib/runner/setup-operations.mjs +115 -0
- package/lib/runner/setup-operations.test.mjs +94 -0
- package/lib/runner/suite-selection.mjs +4 -4
- package/lib/runner/suite-selection.test.mjs +9 -2
- package/lib/runner/template-steps.mjs +129 -11
- package/lib/runner/worker-loop.mjs +1 -1
- package/lib/runtime-src/k6/checks.js +9 -0
- package/lib/runtime-src/k6/scenario-runtime.js +234 -0
- package/lib/runtime-src/k6/scenario-suite.js +179 -0
- package/lib/toolchains/index.mjs +0 -4
- package/package.json +1 -1
package/lib/cli/viewer.mjs
CHANGED
|
@@ -16,6 +16,14 @@ export function loadLatestRunArtifact(productDir) {
|
|
|
16
16
|
return JSON.parse(fs.readFileSync(artifactPath, "utf8"));
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
export function loadCurrentRunArtifact(productDir) {
|
|
20
|
+
const livePath = path.join(productDir, ".testkit", "results", "live.json");
|
|
21
|
+
if (fs.existsSync(livePath)) {
|
|
22
|
+
return JSON.parse(fs.readFileSync(livePath, "utf8"));
|
|
23
|
+
}
|
|
24
|
+
return loadLatestRunArtifact(productDir);
|
|
25
|
+
}
|
|
26
|
+
|
|
19
27
|
export function resolveFileSubject(runArtifact, selector = null, serviceFilter = null) {
|
|
20
28
|
const files = collectFiles(runArtifact, serviceFilter);
|
|
21
29
|
if (files.length === 0) {
|
|
@@ -108,6 +116,25 @@ export function formatFileDetail(productDir, runArtifact, subject, options = {})
|
|
|
108
116
|
for (const line of triageLines) lines.push(` ${line}`);
|
|
109
117
|
}
|
|
110
118
|
|
|
119
|
+
const setupOperations = getSetupOperationsForService(runArtifact, subject.service.name);
|
|
120
|
+
if (setupOperations.length > 0) {
|
|
121
|
+
lines.push("");
|
|
122
|
+
lines.push("Setup:");
|
|
123
|
+
for (const operation of setupOperations.slice(0, 8)) {
|
|
124
|
+
const duration = operation.durationMs == null ? "" : ` ${formatDuration(operation.durationMs)}`;
|
|
125
|
+
const suffix = operation.summary ? ` ${operation.summary}` : "";
|
|
126
|
+
lines.push(` ${operation.status} ${operation.stage}${duration}${suffix}`);
|
|
127
|
+
if (operation.logRef?.path) lines.push(` ${operation.logRef.path}`);
|
|
128
|
+
if (operation.error) lines.push(` ${operation.error}`);
|
|
129
|
+
if (operation.logRef?.path) {
|
|
130
|
+
const setupLogPath = path.join(productDir, operation.logRef.path);
|
|
131
|
+
for (const line of readLogTail(setupLogPath, 4).slice(-4)) {
|
|
132
|
+
lines.push(` ${line}`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
111
138
|
const artifacts = collectArtifactEntries(productDir, runArtifact, subject.file.path, subject.service.name);
|
|
112
139
|
if (artifacts.length > 0) {
|
|
113
140
|
lines.push("");
|
|
@@ -148,11 +175,24 @@ export function getServiceLogRefs(runArtifact, serviceName) {
|
|
|
148
175
|
return (runArtifact.logs?.services || []).filter((entry) => entry.serviceName === serviceName);
|
|
149
176
|
}
|
|
150
177
|
|
|
178
|
+
export function getSetupOperationsForService(runArtifact, serviceName) {
|
|
179
|
+
return (runArtifact.setup?.operations || [])
|
|
180
|
+
.filter((entry) => entry.serviceName === serviceName)
|
|
181
|
+
.sort(
|
|
182
|
+
(left, right) =>
|
|
183
|
+
String(left.startedAt || "").localeCompare(String(right.startedAt || "")) ||
|
|
184
|
+
String(left.stage || "").localeCompare(String(right.stage || ""))
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
|
|
151
188
|
export function formatArtifactPreview(payload, maxLines = 6) {
|
|
152
189
|
if (!payload) return ["artifact payload missing"];
|
|
153
190
|
if (payload.kind === "agentic-query") {
|
|
154
191
|
return formatAgenticArtifact(payload, maxLines);
|
|
155
192
|
}
|
|
193
|
+
if (payload.kind === "testkit.scenario") {
|
|
194
|
+
return formatScenarioArtifact(payload, maxLines);
|
|
195
|
+
}
|
|
156
196
|
if (payload.kind === "testkit.http-traces") {
|
|
157
197
|
return formatHttpTraceArtifact(payload, maxLines);
|
|
158
198
|
}
|
|
@@ -193,12 +233,39 @@ function formatHttpTraceArtifact(payload, maxLines) {
|
|
|
193
233
|
return lines;
|
|
194
234
|
}
|
|
195
235
|
|
|
236
|
+
function formatScenarioArtifact(payload, maxLines) {
|
|
237
|
+
const artifact = payload.data || {};
|
|
238
|
+
const lines = [];
|
|
239
|
+
if (artifact.scenarioName) lines.push(`Scenario: ${artifact.scenarioName}`);
|
|
240
|
+
if (artifact.seed) lines.push(`Seed: ${artifact.seed}`);
|
|
241
|
+
const choiceEntries = Object.entries(artifact.choices || {});
|
|
242
|
+
if (choiceEntries.length > 0) {
|
|
243
|
+
lines.push(
|
|
244
|
+
`Choices: ${choiceEntries
|
|
245
|
+
.map(([key, value]) => `${key}=${formatScenarioChoiceValue(value)}`)
|
|
246
|
+
.join(", ")}`
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
const failedStep = (artifact.steps || []).find((step) => step.status === "failed");
|
|
250
|
+
if (failedStep) {
|
|
251
|
+
lines.push(`Failed Step: ${failedStep.name}`);
|
|
252
|
+
} else if (Array.isArray(artifact.steps) && artifact.steps.length > 0) {
|
|
253
|
+
lines.push(`Steps: ${artifact.steps.map((step) => step.name).join(" -> ")}`);
|
|
254
|
+
}
|
|
255
|
+
return lines.slice(0, maxLines);
|
|
256
|
+
}
|
|
257
|
+
|
|
196
258
|
function rankFailureDetails(details) {
|
|
197
259
|
return [...(Array.isArray(details) ? details : [])].sort((left, right) => {
|
|
198
260
|
return failureDetailRank(left) - failureDetailRank(right) || String(left?.key || "").localeCompare(String(right?.key || ""));
|
|
199
261
|
});
|
|
200
262
|
}
|
|
201
263
|
|
|
264
|
+
function formatScenarioChoiceValue(value) {
|
|
265
|
+
if (typeof value === "string") return value;
|
|
266
|
+
return JSON.stringify(value);
|
|
267
|
+
}
|
|
268
|
+
|
|
202
269
|
function failureDetailRank(detail) {
|
|
203
270
|
if (detail?.kind === "http-assertion") return 1;
|
|
204
271
|
if (detail?.request && detail?.response) return 2;
|
package/lib/config/discovery.mjs
CHANGED
|
@@ -5,6 +5,7 @@ const TESTKIT_DIRNAME = "__testkit__";
|
|
|
5
5
|
const DISCOVERY_RULES = [
|
|
6
6
|
{ suffix: ".int.testkit.ts", type: "integration", framework: "k6" },
|
|
7
7
|
{ suffix: ".e2e.testkit.ts", type: "e2e", framework: "k6" },
|
|
8
|
+
{ suffix: ".scenario.testkit.ts", type: "scenario", framework: "k6" },
|
|
8
9
|
{ suffix: ".dal.testkit.ts", type: "dal", framework: "k6" },
|
|
9
10
|
{ suffix: ".load.testkit.ts", type: "load", framework: "k6" },
|
|
10
11
|
{ suffix: ".pw.testkit.ts", type: "e2e", framework: "playwright" },
|
|
@@ -19,6 +19,7 @@ describe("filesystem-discovery", () => {
|
|
|
19
19
|
|
|
20
20
|
writeFile(productDir, "src/api/routes/__testkit__/auth/me.int.testkit.ts");
|
|
21
21
|
writeFile(productDir, "src/api/routes/__testkit__/health/ready.int.testkit.ts");
|
|
22
|
+
writeFile(productDir, "src/api/routes/__testkit__/journeys/smoke.scenario.testkit.ts");
|
|
22
23
|
writeFile(productDir, "frontend/app/__testkit__/homepage/homepage.pw.testkit.ts");
|
|
23
24
|
|
|
24
25
|
const suites = discoverSuites(productDir, {
|
|
@@ -46,6 +47,13 @@ describe("filesystem-discovery", () => {
|
|
|
46
47
|
framework: "k6",
|
|
47
48
|
},
|
|
48
49
|
]);
|
|
50
|
+
expect(suites.api.scenario).toEqual([
|
|
51
|
+
{
|
|
52
|
+
name: "journeys",
|
|
53
|
+
files: ["src/api/routes/__testkit__/journeys/smoke.scenario.testkit.ts"],
|
|
54
|
+
framework: "k6",
|
|
55
|
+
},
|
|
56
|
+
]);
|
|
49
57
|
expect(suites.frontend.e2e).toEqual([
|
|
50
58
|
{
|
|
51
59
|
name: "homepage",
|
package/lib/database/index.mjs
CHANGED
|
@@ -36,25 +36,25 @@ const LOCAL_POLL_INTERVAL_MS = 1_000;
|
|
|
36
36
|
const LOCAL_DROP_DATABASE_TIMEOUT_MS = 15_000;
|
|
37
37
|
const LOCAL_DROP_DATABASE_POLL_INTERVAL_MS = 250;
|
|
38
38
|
|
|
39
|
-
export async function prepareDatabaseRuntime(config) {
|
|
39
|
+
export async function prepareDatabaseRuntime(config, options = {}) {
|
|
40
40
|
const db = config.testkit.database;
|
|
41
41
|
if (!db) return;
|
|
42
42
|
|
|
43
43
|
fs.mkdirSync(config.stateDir, { recursive: true });
|
|
44
44
|
if (db.provider === "local") {
|
|
45
|
-
await prepareLocalDatabase(config);
|
|
45
|
+
await prepareLocalDatabase(config, options);
|
|
46
46
|
return;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
49
|
throw new Error(`Unsupported database provider "${db.provider}"`);
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
export async function captureDatabaseTemplateSnapshot(config, outputPath) {
|
|
52
|
+
export async function captureDatabaseTemplateSnapshot(config, outputPath, options = {}) {
|
|
53
53
|
if (!config.testkit.database || config.testkit.database.provider !== "local") {
|
|
54
54
|
throw new Error(`Service "${config.name}" does not use a local testkit database`);
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
await prepareDatabaseRuntime(config);
|
|
57
|
+
await prepareDatabaseRuntime(config, options);
|
|
58
58
|
const cacheDir = getLocalServiceCacheDir(config.productDir, config.name);
|
|
59
59
|
const templateDbName = readStateValue(path.join(cacheDir, "template_database_name"));
|
|
60
60
|
if (!templateDbName) {
|
|
@@ -66,7 +66,41 @@ export async function captureDatabaseTemplateSnapshot(config, outputPath) {
|
|
|
66
66
|
throw new Error(`Missing local database container for service "${config.name}"`);
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
-
|
|
69
|
+
const snapshotOperation = options.setupRegistry?.start({
|
|
70
|
+
config,
|
|
71
|
+
stage: "template:snapshot",
|
|
72
|
+
kind: "database-snapshot",
|
|
73
|
+
summary: `snapshot: ${path.relative(config.productDir, outputPath)}`,
|
|
74
|
+
});
|
|
75
|
+
try {
|
|
76
|
+
const output = await captureTemplateSnapshot(
|
|
77
|
+
config,
|
|
78
|
+
outputPath,
|
|
79
|
+
buildDatabaseUrl(infra, templateDbName),
|
|
80
|
+
{
|
|
81
|
+
reporter: options.reporter || null,
|
|
82
|
+
logRecord: snapshotOperation?._logRecord || null,
|
|
83
|
+
}
|
|
84
|
+
);
|
|
85
|
+
const finished = snapshotOperation
|
|
86
|
+
? options.setupRegistry.finish(snapshotOperation, {
|
|
87
|
+
status: "passed",
|
|
88
|
+
summary: `snapshot: ${path.relative(config.productDir, outputPath)}`,
|
|
89
|
+
})
|
|
90
|
+
: null;
|
|
91
|
+
if (finished) options.reporter?.setupOperationFinished?.(finished);
|
|
92
|
+
return output;
|
|
93
|
+
} catch (error) {
|
|
94
|
+
const finished = snapshotOperation
|
|
95
|
+
? options.setupRegistry.finish(snapshotOperation, {
|
|
96
|
+
status: "failed",
|
|
97
|
+
summary: `snapshot: ${path.relative(config.productDir, outputPath)}`,
|
|
98
|
+
error: error?.message || error,
|
|
99
|
+
})
|
|
100
|
+
: null;
|
|
101
|
+
if (finished) options.reporter?.setupOperationFinished?.(finished);
|
|
102
|
+
throw error;
|
|
103
|
+
}
|
|
70
104
|
}
|
|
71
105
|
|
|
72
106
|
export async function destroyRuntimeDatabase({ productDir, stateDir }) {
|
|
@@ -138,7 +172,7 @@ export function showServiceDatabaseStatus(productDir, serviceName) {
|
|
|
138
172
|
return true;
|
|
139
173
|
}
|
|
140
174
|
|
|
141
|
-
async function prepareLocalDatabase(config) {
|
|
175
|
+
async function prepareLocalDatabase(config, options = {}) {
|
|
142
176
|
const db = config.testkit.database;
|
|
143
177
|
const productDir = config.productDir;
|
|
144
178
|
const serviceName = config.name;
|
|
@@ -154,7 +188,7 @@ async function prepareLocalDatabase(config) {
|
|
|
154
188
|
);
|
|
155
189
|
|
|
156
190
|
await withLock(path.join(lockDir, `template-${serviceName}.lock`), async () => {
|
|
157
|
-
await ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint);
|
|
191
|
+
await ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint, options);
|
|
158
192
|
});
|
|
159
193
|
|
|
160
194
|
await withLock(path.join(lockDir, `runtime-${serviceName}-${hashString(bindingKey, 10)}.lock`), async () => {
|
|
@@ -162,7 +196,7 @@ async function prepareLocalDatabase(config) {
|
|
|
162
196
|
});
|
|
163
197
|
}
|
|
164
198
|
|
|
165
|
-
async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint) {
|
|
199
|
+
async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerprint, options = {}) {
|
|
166
200
|
const serviceName = config.name;
|
|
167
201
|
const existingFingerprint = readStateValue(path.join(cacheDir, "template_fingerprint"));
|
|
168
202
|
const existingDbName = readStateValue(path.join(cacheDir, "template_database_name"));
|
|
@@ -173,6 +207,12 @@ async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerpri
|
|
|
173
207
|
existingDbName &&
|
|
174
208
|
(await databaseExists(infra, existingDbName))
|
|
175
209
|
) {
|
|
210
|
+
options.setupRegistry?.recordCached({
|
|
211
|
+
config,
|
|
212
|
+
stage: "template",
|
|
213
|
+
kind: "database-template",
|
|
214
|
+
summary: "template cache hit",
|
|
215
|
+
});
|
|
176
216
|
writeLocalCacheState(cacheDir, infra, existingDbName, templateFingerprint);
|
|
177
217
|
return;
|
|
178
218
|
}
|
|
@@ -186,11 +226,45 @@ async function ensureTemplateDatabase(config, infra, cacheDir, templateFingerpri
|
|
|
186
226
|
|
|
187
227
|
const templateUrl = buildDatabaseUrl(infra, desiredDbName);
|
|
188
228
|
await createEmptyDatabase(infra, desiredDbName);
|
|
229
|
+
const templateOperation = options.setupRegistry?.start({
|
|
230
|
+
config,
|
|
231
|
+
stage: "template",
|
|
232
|
+
kind: "database-template",
|
|
233
|
+
summary: "template rebuild",
|
|
234
|
+
recordLog: false,
|
|
235
|
+
});
|
|
189
236
|
try {
|
|
190
|
-
await runTemplateStage(config, "migrate", templateUrl
|
|
191
|
-
|
|
192
|
-
|
|
237
|
+
await runTemplateStage(config, "migrate", templateUrl, {
|
|
238
|
+
reporter: options.reporter || null,
|
|
239
|
+
setupRegistry: options.setupRegistry || null,
|
|
240
|
+
parentOperation: templateOperation,
|
|
241
|
+
});
|
|
242
|
+
await runTemplateStage(config, "seed", templateUrl, {
|
|
243
|
+
reporter: options.reporter || null,
|
|
244
|
+
setupRegistry: options.setupRegistry || null,
|
|
245
|
+
parentOperation: templateOperation,
|
|
246
|
+
});
|
|
247
|
+
await runTemplateStage(config, "verify", templateUrl, {
|
|
248
|
+
reporter: options.reporter || null,
|
|
249
|
+
setupRegistry: options.setupRegistry || null,
|
|
250
|
+
parentOperation: templateOperation,
|
|
251
|
+
});
|
|
252
|
+
const finished = templateOperation
|
|
253
|
+
? options.setupRegistry.finish(templateOperation, {
|
|
254
|
+
status: "passed",
|
|
255
|
+
summary: "template rebuild",
|
|
256
|
+
})
|
|
257
|
+
: null;
|
|
258
|
+
if (finished) options.reporter?.setupOperationFinished?.(finished);
|
|
193
259
|
} catch (error) {
|
|
260
|
+
const finished = templateOperation
|
|
261
|
+
? options.setupRegistry.finish(templateOperation, {
|
|
262
|
+
status: "failed",
|
|
263
|
+
summary: "template rebuild",
|
|
264
|
+
error: error?.message || error,
|
|
265
|
+
})
|
|
266
|
+
: null;
|
|
267
|
+
if (finished) options.reporter?.setupOperationFinished?.(finished);
|
|
194
268
|
await dropDatabaseIfExists(infra, desiredDbName);
|
|
195
269
|
throw error;
|
|
196
270
|
}
|
|
@@ -6,8 +6,9 @@ import {
|
|
|
6
6
|
collectConfiguredInputs,
|
|
7
7
|
runConfiguredSteps,
|
|
8
8
|
} from "../runner/template-steps.mjs";
|
|
9
|
+
import { captureOutput } from "../runner/processes.mjs";
|
|
9
10
|
|
|
10
|
-
export async function runTemplateStage(config, stageName, databaseUrl) {
|
|
11
|
+
export async function runTemplateStage(config, stageName, databaseUrl, options = {}) {
|
|
11
12
|
const steps = config.testkit.database?.template?.[stageName] || [];
|
|
12
13
|
if (steps.length === 0) return;
|
|
13
14
|
|
|
@@ -21,6 +22,9 @@ export async function runTemplateStage(config, stageName, databaseUrl) {
|
|
|
21
22
|
steps,
|
|
22
23
|
env,
|
|
23
24
|
labelPrefix: `template:${stageName}`,
|
|
25
|
+
reporter: options.reporter || null,
|
|
26
|
+
setupRegistry: options.setupRegistry || null,
|
|
27
|
+
parentOperation: options.parentOperation || null,
|
|
24
28
|
});
|
|
25
29
|
}
|
|
26
30
|
|
|
@@ -32,12 +36,12 @@ export function collectTemplateInputs(productDir, template = {}) {
|
|
|
32
36
|
});
|
|
33
37
|
}
|
|
34
38
|
|
|
35
|
-
export async function captureTemplateSnapshot(config, outputPath, databaseUrl) {
|
|
39
|
+
export async function captureTemplateSnapshot(config, outputPath, databaseUrl, options = {}) {
|
|
36
40
|
const templateDbUrl = databaseUrl;
|
|
37
41
|
const absoluteOutputPath = path.resolve(config.productDir, outputPath);
|
|
38
42
|
fs.mkdirSync(path.dirname(absoluteOutputPath), { recursive: true });
|
|
39
43
|
|
|
40
|
-
|
|
44
|
+
const child = execa(
|
|
41
45
|
"pg_dump",
|
|
42
46
|
[
|
|
43
47
|
"--schema-only",
|
|
@@ -53,19 +57,54 @@ export async function captureTemplateSnapshot(config, outputPath, databaseUrl) {
|
|
|
53
57
|
...buildExecutionEnv(config, {}, process.env),
|
|
54
58
|
DATABASE_URL: templateDbUrl,
|
|
55
59
|
},
|
|
56
|
-
|
|
60
|
+
stdout: "pipe",
|
|
61
|
+
stderr: "pipe",
|
|
62
|
+
reject: false,
|
|
57
63
|
}
|
|
58
64
|
);
|
|
65
|
+
const liveWriter =
|
|
66
|
+
options.reporter?.outputMode === "debug"
|
|
67
|
+
? (line) => options.reporter.writeDebugLine?.(line)
|
|
68
|
+
: null;
|
|
69
|
+
const logRecord = options.logRecord || null;
|
|
70
|
+
const drains = [
|
|
71
|
+
captureOutput(child.stdout, {
|
|
72
|
+
livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
|
|
73
|
+
liveWriter,
|
|
74
|
+
onLine(line) {
|
|
75
|
+
if (logRecord) logRecord.stream.write(`${new Date().toISOString()} [stdout] ${line}\n`);
|
|
76
|
+
},
|
|
77
|
+
}),
|
|
78
|
+
captureOutput(child.stderr, {
|
|
79
|
+
livePrefix: `[${config.runtimeLabel || config.name}:${config.name}]`,
|
|
80
|
+
liveWriter,
|
|
81
|
+
onLine(line) {
|
|
82
|
+
if (logRecord) logRecord.stream.write(`${new Date().toISOString()} [stderr] ${line}\n`);
|
|
83
|
+
},
|
|
84
|
+
}),
|
|
85
|
+
];
|
|
86
|
+
const result = await child;
|
|
87
|
+
await Promise.all(drains);
|
|
88
|
+
if (result.exitCode !== 0) {
|
|
89
|
+
throw new Error(result.shortMessage || result.stderr || result.stdout || "pg_dump failed");
|
|
90
|
+
}
|
|
59
91
|
|
|
60
92
|
sanitizeSnapshotFile(absoluteOutputPath);
|
|
61
93
|
return absoluteOutputPath;
|
|
62
94
|
}
|
|
63
95
|
|
|
64
|
-
function sanitizeSnapshotFile(filePath) {
|
|
96
|
+
export function sanitizeSnapshotFile(filePath) {
|
|
65
97
|
const dump = fs.readFileSync(filePath, "utf8");
|
|
66
98
|
const sanitized = dump
|
|
67
99
|
.split("\n")
|
|
68
|
-
.filter((line) =>
|
|
100
|
+
.filter((line) => {
|
|
101
|
+
const trimmed = line.trim();
|
|
102
|
+
return (
|
|
103
|
+
trimmed !== "SET transaction_timeout = 0;" &&
|
|
104
|
+
!trimmed.startsWith("\\restrict ") &&
|
|
105
|
+
!trimmed.startsWith("\\unrestrict ")
|
|
106
|
+
);
|
|
107
|
+
})
|
|
69
108
|
.join("\n");
|
|
70
109
|
|
|
71
110
|
if (sanitized !== dump) {
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import os from "os";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
5
|
+
import { sanitizeSnapshotFile } from "./template-steps.mjs";
|
|
6
|
+
|
|
7
|
+
const tempDirs = [];
|
|
8
|
+
|
|
9
|
+
afterEach(() => {
|
|
10
|
+
while (tempDirs.length > 0) {
|
|
11
|
+
fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
function makeTempDir(prefix) {
|
|
16
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
|
17
|
+
tempDirs.push(dir);
|
|
18
|
+
return dir;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
describe("template snapshot sanitization", () => {
|
|
22
|
+
it("removes volatile pg_dump control lines", () => {
|
|
23
|
+
const dir = makeTempDir("testkit-template-snapshot-");
|
|
24
|
+
const filePath = path.join(dir, "schema.sql");
|
|
25
|
+
fs.writeFileSync(
|
|
26
|
+
filePath,
|
|
27
|
+
[
|
|
28
|
+
"SET statement_timeout = 0;",
|
|
29
|
+
"SET transaction_timeout = 0;",
|
|
30
|
+
"\\restrict abc123",
|
|
31
|
+
"CREATE TABLE public.widgets (id integer);",
|
|
32
|
+
"\\unrestrict abc123",
|
|
33
|
+
"",
|
|
34
|
+
].join("\n")
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
sanitizeSnapshotFile(filePath);
|
|
38
|
+
|
|
39
|
+
expect(fs.readFileSync(filePath, "utf8")).toBe(
|
|
40
|
+
["SET statement_timeout = 0;", "CREATE TABLE public.widgets (id integer);", ""].join("\n")
|
|
41
|
+
);
|
|
42
|
+
});
|
|
43
|
+
});
|
package/lib/index.d.ts
CHANGED
|
@@ -38,6 +38,55 @@ export interface HttpSuiteContext<TSetup = unknown> {
|
|
|
38
38
|
session: TSetup | null;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
export interface ScenarioStepResult {
|
|
42
|
+
name: string;
|
|
43
|
+
status: "passed" | "failed";
|
|
44
|
+
startedAt?: string;
|
|
45
|
+
finishedAt?: string;
|
|
46
|
+
durationMs?: number;
|
|
47
|
+
failureCount?: number;
|
|
48
|
+
error?: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface ScenarioResource<TValue = unknown> {
|
|
52
|
+
get(): TValue;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface ScenarioRuntime {
|
|
56
|
+
readonly seed: string;
|
|
57
|
+
readonly scenarioName: string | null;
|
|
58
|
+
choose<TChoice extends unknown[]>(
|
|
59
|
+
name: string,
|
|
60
|
+
choices: TChoice
|
|
61
|
+
): TChoice[number];
|
|
62
|
+
choose<TShape extends Record<string, unknown>>(
|
|
63
|
+
name: string,
|
|
64
|
+
shape: TShape
|
|
65
|
+
): TShape;
|
|
66
|
+
maybe(name: string, probability?: number): boolean;
|
|
67
|
+
note<TValue = unknown>(name: string, value: TValue): TValue;
|
|
68
|
+
pick<TChoice extends unknown[]>(name: string, choices: TChoice): TChoice[number];
|
|
69
|
+
resource<TValue = unknown>(
|
|
70
|
+
name: string,
|
|
71
|
+
factory: () => TValue,
|
|
72
|
+
options?: { scope?: "file" | "scenario" | "step" }
|
|
73
|
+
): ScenarioResource<TValue>;
|
|
74
|
+
step<TValue = unknown>(name: string, fn: () => TValue): TValue;
|
|
75
|
+
snapshot(): {
|
|
76
|
+
schemaVersion: number;
|
|
77
|
+
seed: string;
|
|
78
|
+
scenarioName: string | null;
|
|
79
|
+
choices: Record<string, unknown>;
|
|
80
|
+
notes: Record<string, unknown>;
|
|
81
|
+
resources: Array<{ name: string; scope: "file" | "scenario" | "step" }>;
|
|
82
|
+
steps: ScenarioStepResult[];
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface ScenarioSuiteContext<TSetup = unknown> extends HttpSuiteContext<TSetup> {
|
|
87
|
+
scenario: ScenarioRuntime;
|
|
88
|
+
}
|
|
89
|
+
|
|
41
90
|
export interface HttpSuiteConfig<TSetup = unknown> {
|
|
42
91
|
auth?: AuthAdapter<TSetup> | null;
|
|
43
92
|
env?: RuntimeEnv;
|
|
@@ -68,6 +117,15 @@ export declare function defineHttpSuite<TSetup = unknown>(
|
|
|
68
117
|
run: (context: HttpSuiteContext<TSetup>) => unknown
|
|
69
118
|
): TestkitSuite<TSetup>;
|
|
70
119
|
|
|
120
|
+
export declare function defineScenarioSuite<TSetup = unknown>(
|
|
121
|
+
run: (context: ScenarioSuiteContext<TSetup>) => unknown
|
|
122
|
+
): TestkitSuite<TSetup>;
|
|
123
|
+
|
|
124
|
+
export declare function defineScenarioSuite<TSetup = unknown>(
|
|
125
|
+
config: HttpSuiteConfig<TSetup>,
|
|
126
|
+
run: (context: ScenarioSuiteContext<TSetup>) => unknown
|
|
127
|
+
): TestkitSuite<TSetup>;
|
|
128
|
+
|
|
71
129
|
export declare function defineDalSuite<TSetup = unknown>(
|
|
72
130
|
run: (context: DalSuiteContext<TSetup>) => unknown
|
|
73
131
|
): TestkitSuite<TSetup>;
|
package/lib/index.mjs
CHANGED
package/lib/runner/artifacts.mjs
CHANGED
|
@@ -9,11 +9,20 @@ import {
|
|
|
9
9
|
const TIMINGS_FILENAME = "timings.json";
|
|
10
10
|
const RESULT_ARTIFACTS_DIRNAME = "artifacts";
|
|
11
11
|
const RESULT_LOGS_DIRNAME = "logs";
|
|
12
|
+
const RESULT_SETUP_DIRNAME = "setup";
|
|
13
|
+
const LIVE_ARTIFACT_FILENAME = "live.json";
|
|
12
14
|
|
|
13
15
|
export function writeRunArtifact(productDir, artifact) {
|
|
14
16
|
const resultsDir = path.join(productDir, ".testkit", "results");
|
|
15
17
|
fs.mkdirSync(resultsDir, { recursive: true });
|
|
16
18
|
fs.writeFileSync(path.join(resultsDir, "latest.json"), JSON.stringify(artifact, null, 2));
|
|
19
|
+
fs.rmSync(path.join(resultsDir, LIVE_ARTIFACT_FILENAME), { force: true });
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function writeLiveRunArtifact(productDir, artifact) {
|
|
23
|
+
const resultsDir = path.join(productDir, ".testkit", "results");
|
|
24
|
+
fs.mkdirSync(resultsDir, { recursive: true });
|
|
25
|
+
fs.writeFileSync(path.join(resultsDir, LIVE_ARTIFACT_FILENAME), JSON.stringify(artifact, null, 2));
|
|
17
26
|
}
|
|
18
27
|
|
|
19
28
|
export function writeStatusArtifact(productDir, artifact) {
|
|
@@ -32,6 +41,13 @@ export function resetResultArtifacts(productDir) {
|
|
|
32
41
|
recursive: true,
|
|
33
42
|
force: true,
|
|
34
43
|
});
|
|
44
|
+
fs.rmSync(path.join(productDir, ".testkit", "results", RESULT_SETUP_DIRNAME), {
|
|
45
|
+
recursive: true,
|
|
46
|
+
force: true,
|
|
47
|
+
});
|
|
48
|
+
fs.rmSync(path.join(productDir, ".testkit", "results", LIVE_ARTIFACT_FILENAME), {
|
|
49
|
+
force: true,
|
|
50
|
+
});
|
|
35
51
|
}
|
|
36
52
|
|
|
37
53
|
export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
|
|
@@ -95,7 +95,10 @@ export async function runDefaultRuntimeTask(
|
|
|
95
95
|
env: buildTaskExecutionEnv(
|
|
96
96
|
targetConfig,
|
|
97
97
|
lease,
|
|
98
|
-
|
|
98
|
+
{
|
|
99
|
+
...buildFileTimeoutEnv(fileTimeoutSeconds, startedAt),
|
|
100
|
+
...(task.scenarioSeed ? { TESTKIT_SCENARIO_SEED: task.scenarioSeed } : {}),
|
|
101
|
+
},
|
|
99
102
|
process.env
|
|
100
103
|
),
|
|
101
104
|
reject: false,
|
package/lib/runner/logs.mjs
CHANGED
|
@@ -2,21 +2,26 @@ import fs from "fs";
|
|
|
2
2
|
import path from "path";
|
|
3
3
|
|
|
4
4
|
const RESULT_LOGS_DIRNAME = "logs";
|
|
5
|
+
const RESULT_SETUP_DIRNAME = "setup";
|
|
5
6
|
|
|
6
7
|
export function createRunLogRegistry(productDir) {
|
|
7
|
-
const
|
|
8
|
+
const serviceRecords = new Map();
|
|
9
|
+
const setupRecords = new Map();
|
|
8
10
|
|
|
9
11
|
return {
|
|
10
12
|
ensureServiceLogRecord(config) {
|
|
11
13
|
const key = `${config.runtimeLabel || config.name}:${config.name}`;
|
|
12
|
-
const existing =
|
|
14
|
+
const existing = serviceRecords.get(key);
|
|
13
15
|
if (existing) return existing;
|
|
14
16
|
|
|
15
17
|
const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}.log`;
|
|
16
18
|
const relativePath = path.join(".testkit", "results", RESULT_LOGS_DIRNAME, fileName);
|
|
17
19
|
const absolutePath = path.join(productDir, relativePath);
|
|
18
20
|
fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
|
|
19
|
-
const stream = fs.createWriteStream(absolutePath, {
|
|
21
|
+
const stream = fs.createWriteStream(absolutePath, {
|
|
22
|
+
fd: fs.openSync(absolutePath, "a"),
|
|
23
|
+
flags: "a",
|
|
24
|
+
});
|
|
20
25
|
const record = {
|
|
21
26
|
key,
|
|
22
27
|
serviceName: config.name,
|
|
@@ -25,7 +30,32 @@ export function createRunLogRegistry(productDir) {
|
|
|
25
30
|
absolutePath,
|
|
26
31
|
stream,
|
|
27
32
|
};
|
|
28
|
-
|
|
33
|
+
serviceRecords.set(key, record);
|
|
34
|
+
return record;
|
|
35
|
+
},
|
|
36
|
+
ensureSetupLogRecord(config, stage) {
|
|
37
|
+
const key = `${config.runtimeLabel || config.name}:${config.name}:${stage}`;
|
|
38
|
+
const existing = setupRecords.get(key);
|
|
39
|
+
if (existing) return existing;
|
|
40
|
+
|
|
41
|
+
const fileName = `${sanitizePathSegment(config.runtimeLabel || config.name)}__${sanitizePathSegment(config.name)}__${sanitizePathSegment(stage)}.log`;
|
|
42
|
+
const relativePath = path.join(".testkit", "results", RESULT_SETUP_DIRNAME, fileName);
|
|
43
|
+
const absolutePath = path.join(productDir, relativePath);
|
|
44
|
+
fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
|
|
45
|
+
const stream = fs.createWriteStream(absolutePath, {
|
|
46
|
+
fd: fs.openSync(absolutePath, "a"),
|
|
47
|
+
flags: "a",
|
|
48
|
+
});
|
|
49
|
+
const record = {
|
|
50
|
+
key,
|
|
51
|
+
serviceName: config.name,
|
|
52
|
+
runtimeLabel: config.runtimeLabel || config.name,
|
|
53
|
+
stage,
|
|
54
|
+
path: normalizePath(relativePath),
|
|
55
|
+
absolutePath,
|
|
56
|
+
stream,
|
|
57
|
+
};
|
|
58
|
+
setupRecords.set(key, record);
|
|
29
59
|
return record;
|
|
30
60
|
},
|
|
31
61
|
append(record, streamName, line) {
|
|
@@ -33,7 +63,7 @@ export function createRunLogRegistry(productDir) {
|
|
|
33
63
|
record.stream.write(`${new Date().toISOString()} [${streamName}] ${line}\n`);
|
|
34
64
|
},
|
|
35
65
|
listServiceLogs() {
|
|
36
|
-
return [...
|
|
66
|
+
return [...serviceRecords.values()]
|
|
37
67
|
.map((record) => ({
|
|
38
68
|
serviceName: record.serviceName,
|
|
39
69
|
runtimeLabel: record.runtimeLabel,
|
|
@@ -45,8 +75,26 @@ export function createRunLogRegistry(productDir) {
|
|
|
45
75
|
left.runtimeLabel.localeCompare(right.runtimeLabel)
|
|
46
76
|
);
|
|
47
77
|
},
|
|
78
|
+
listSetupLogs() {
|
|
79
|
+
return [...setupRecords.values()]
|
|
80
|
+
.map((record) => ({
|
|
81
|
+
serviceName: record.serviceName,
|
|
82
|
+
runtimeLabel: record.runtimeLabel,
|
|
83
|
+
stage: record.stage,
|
|
84
|
+
path: record.path,
|
|
85
|
+
}))
|
|
86
|
+
.sort(
|
|
87
|
+
(left, right) =>
|
|
88
|
+
left.serviceName.localeCompare(right.serviceName) ||
|
|
89
|
+
left.runtimeLabel.localeCompare(right.runtimeLabel) ||
|
|
90
|
+
left.stage.localeCompare(right.stage)
|
|
91
|
+
);
|
|
92
|
+
},
|
|
48
93
|
closeAll() {
|
|
49
|
-
for (const record of
|
|
94
|
+
for (const record of serviceRecords.values()) {
|
|
95
|
+
record.stream.end();
|
|
96
|
+
}
|
|
97
|
+
for (const record of setupRecords.values()) {
|
|
50
98
|
record.stream.end();
|
|
51
99
|
}
|
|
52
100
|
},
|