@elench/testkit 0.1.51 → 0.1.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -7
- package/bin/testkit.mjs +4 -6
- package/lib/cli/command-helpers.mjs +170 -0
- package/lib/cli/commands/artifacts.mjs +45 -0
- package/lib/cli/commands/cleanup.mjs +15 -0
- package/lib/cli/commands/db/snapshot/capture.mjs +22 -0
- package/lib/cli/commands/destroy.mjs +15 -0
- package/lib/cli/commands/known-failures/render.mjs +19 -0
- package/lib/cli/commands/known-failures/validate.mjs +20 -0
- package/lib/cli/commands/logs.mjs +47 -0
- package/lib/cli/commands/run.mjs +23 -0
- package/lib/cli/commands/show.mjs +47 -0
- package/lib/cli/commands/status.mjs +15 -0
- package/lib/cli/commands/watch.mjs +23 -0
- package/lib/cli/entrypoint.mjs +83 -0
- package/lib/cli/index.mjs +6 -116
- package/lib/cli/presentation/run-reporter.mjs +91 -0
- package/lib/cli/tui/watch-app.mjs +104 -0
- package/lib/cli/viewer.mjs +163 -0
- package/lib/runner/artifacts.mjs +35 -0
- package/lib/runner/default-runtime-runner.mjs +44 -10
- package/lib/runner/formatting.mjs +97 -0
- package/lib/runner/formatting.test.mjs +4 -6
- package/lib/runner/logs.mjs +72 -0
- package/lib/runner/orchestrator.mjs +41 -19
- package/lib/runner/playwright-runner.mjs +15 -7
- package/lib/runner/processes.mjs +9 -11
- package/lib/runner/reporting.mjs +5 -1
- package/lib/runner/reporting.test.mjs +4 -1
- package/lib/runner/runtime-contexts.mjs +7 -3
- package/lib/runner/runtime-manager.mjs +8 -2
- package/lib/runner/runtime-preparation.mjs +9 -4
- package/lib/runner/services.mjs +25 -8
- package/lib/runner/template-steps.mjs +4 -3
- package/lib/runner/worker-loop.mjs +8 -7
- package/lib/setup/index.d.ts +46 -13
- package/lib/setup/index.mjs +47 -0
- package/lib/setup/index.test.mjs +109 -1
- package/lib/toolchains/index.mjs +6 -3
- package/package.json +11 -3
package/lib/cli/index.mjs
CHANGED
|
@@ -1,119 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
runKnownFailuresRenderCommand,
|
|
5
|
-
runKnownFailuresValidateCommand,
|
|
6
|
-
} from "./known-failures.mjs";
|
|
7
|
-
import { runDatabaseSnapshotCaptureCommand } from "./db.mjs";
|
|
8
|
-
import {
|
|
9
|
-
parseFileTimeoutOption,
|
|
10
|
-
parseShardOption,
|
|
11
|
-
parseSuiteOption,
|
|
12
|
-
parseTypeOption,
|
|
13
|
-
parseWorkersOption,
|
|
14
|
-
resolveRequestedFiles,
|
|
15
|
-
resolveCliSelection,
|
|
16
|
-
} from "./args.mjs";
|
|
17
|
-
import * as runner from "../runner/index.mjs";
|
|
1
|
+
import { execute } from "@oclif/core";
|
|
2
|
+
import { normalizeCliArgs } from "./entrypoint.mjs";
|
|
18
3
|
|
|
19
4
|
export async function run(argv = process.argv) {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
.option("--service <name>", "Run only one service")
|
|
25
|
-
.option("-t, --type <name>", "Run specific suite type(s): int, e2e, dal, load, pw, all", {
|
|
26
|
-
default: [],
|
|
27
|
-
})
|
|
28
|
-
.option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
|
|
29
|
-
.option("-f, --file <path>", "Run specific file(s)", { default: [] })
|
|
30
|
-
.option("--dir <path>", "Explicit product directory")
|
|
31
|
-
.option("--workers <n>", "Number of test executors for the whole run")
|
|
32
|
-
.option("--file-timeout-seconds <n>", "Per-file wall-clock timeout in seconds")
|
|
33
|
-
.option("--shard <i/n>", "Run only shard i of n at suite granularity")
|
|
34
|
-
.option("--write-status", "Write a deterministic testkit.status.json snapshot")
|
|
35
|
-
.option("--allow-partial-status", "Allow --write-status for filtered runs")
|
|
36
|
-
.option("--input <path>", "Known failures JSON path (repo-relative)")
|
|
37
|
-
.option("--output <path>", "Output path for render/snapshot commands")
|
|
38
|
-
.option("--status <path>", "Status artifact path for validation commands")
|
|
39
|
-
.option("--issue-mode <mode>", "Issue validation mode override: off, warn, error")
|
|
40
|
-
.option(
|
|
41
|
-
"--ignore-skip-rules",
|
|
42
|
-
"Run files even if testkit.setup.ts marks them skipped"
|
|
43
|
-
)
|
|
44
|
-
.action(async (first, second, third, options) => {
|
|
45
|
-
const { lifecycle, positionalType, knownFailuresAction, dbAction } = resolveCliSelection({
|
|
46
|
-
first,
|
|
47
|
-
second,
|
|
48
|
-
third,
|
|
49
|
-
});
|
|
50
|
-
if (knownFailuresAction === "render") {
|
|
51
|
-
await runKnownFailuresRenderCommand(options);
|
|
52
|
-
return;
|
|
53
|
-
}
|
|
54
|
-
if (knownFailuresAction === "validate") {
|
|
55
|
-
await runKnownFailuresValidateCommand(options);
|
|
56
|
-
return;
|
|
57
|
-
}
|
|
58
|
-
if (dbAction === "snapshot-capture") {
|
|
59
|
-
await runDatabaseSnapshotCaptureCommand(options);
|
|
60
|
-
return;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
const allConfigs = await loadConfigs({ dir: options.dir });
|
|
64
|
-
const configs = options.service
|
|
65
|
-
? allConfigs.filter((config) => config.name === options.service)
|
|
66
|
-
: allConfigs;
|
|
67
|
-
if (options.service && configs.length === 0) {
|
|
68
|
-
const available = allConfigs.map((config) => config.name).join(", ");
|
|
69
|
-
throw new Error(`Service "${options.service}" not found. Available: ${available}`);
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
// Lifecycle commands
|
|
73
|
-
if (lifecycle === "cleanup") {
|
|
74
|
-
await runner.cleanup(allConfigs[0]?.productDir || process.cwd());
|
|
75
|
-
return;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
if (lifecycle === "status" || lifecycle === "destroy") {
|
|
79
|
-
for (const config of configs) {
|
|
80
|
-
if (configs.length > 1) console.log(`\n── ${config.name} ──`);
|
|
81
|
-
if (lifecycle === "status") runner.showStatus(config);
|
|
82
|
-
else await runner.destroy(config);
|
|
83
|
-
}
|
|
84
|
-
return;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
const workers = options.workers == null ? null : parseWorkersOption(options.workers);
|
|
88
|
-
const fileTimeoutSeconds =
|
|
89
|
-
options.fileTimeoutSeconds == null
|
|
90
|
-
? null
|
|
91
|
-
: parseFileTimeoutOption(options.fileTimeoutSeconds);
|
|
92
|
-
const shard = parseShardOption(options.shard);
|
|
93
|
-
const typeValues = parseTypeOption(options.type, positionalType);
|
|
94
|
-
const suiteSelectors = parseSuiteOption(options.suite);
|
|
95
|
-
const rawFileNames = Array.isArray(options.file) ? options.file : [options.file].filter(Boolean);
|
|
96
|
-
const productDir = allConfigs[0]?.productDir || process.cwd();
|
|
97
|
-
const fileNames = resolveRequestedFiles(rawFileNames, productDir, process.cwd());
|
|
98
|
-
await runner.runAll(
|
|
99
|
-
configs,
|
|
100
|
-
typeValues,
|
|
101
|
-
suiteSelectors,
|
|
102
|
-
{
|
|
103
|
-
...options,
|
|
104
|
-
typeValues,
|
|
105
|
-
fileNames,
|
|
106
|
-
workers,
|
|
107
|
-
fileTimeoutSeconds,
|
|
108
|
-
shard,
|
|
109
|
-
serviceFilter: options.service || null,
|
|
110
|
-
},
|
|
111
|
-
allConfigs
|
|
112
|
-
);
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
cli.help();
|
|
116
|
-
const parsed = cli.parse(argv, { run: false });
|
|
117
|
-
await cli.runMatchedCommand();
|
|
118
|
-
return parsed;
|
|
5
|
+
return execute({
|
|
6
|
+
args: normalizeCliArgs(argv.slice(2)),
|
|
7
|
+
dir: import.meta.url,
|
|
8
|
+
});
|
|
119
9
|
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import {
|
|
3
|
+
buildCompactRunSummaryLines,
|
|
4
|
+
buildDebugRunSummaryLines,
|
|
5
|
+
formatDuration,
|
|
6
|
+
} from "../../runner/formatting.mjs";
|
|
7
|
+
|
|
8
|
+
export function createRunReporter({ outputMode = "compact", stdout = process.stdout, stderr = process.stderr } = {}) {
|
|
9
|
+
const mode = outputMode || "compact";
|
|
10
|
+
return {
|
|
11
|
+
outputMode: mode,
|
|
12
|
+
writeLine(line = "") {
|
|
13
|
+
stdout.write(`${line}\n`);
|
|
14
|
+
},
|
|
15
|
+
writeDebugLine(line = "") {
|
|
16
|
+
stdout.write(`${line}\n`);
|
|
17
|
+
},
|
|
18
|
+
phaseStarted(label) {
|
|
19
|
+
if (mode !== "debug") return;
|
|
20
|
+
stdout.write(`\n── ${label} ──\n`);
|
|
21
|
+
},
|
|
22
|
+
toolchainResolved(config, resolvedToolchain) {
|
|
23
|
+
if (mode !== "debug") return;
|
|
24
|
+
stdout.write(
|
|
25
|
+
`[testkit] ${config.runtimeLabel || config.name}:${config.name} toolchain ${resolvedToolchain.summary}\n`
|
|
26
|
+
);
|
|
27
|
+
},
|
|
28
|
+
localServiceStarting(config, command) {
|
|
29
|
+
if (mode !== "debug") return;
|
|
30
|
+
stdout.write(`Starting ${config.runtimeLabel}:${config.name}: ${command}\n`);
|
|
31
|
+
},
|
|
32
|
+
serviceSkipped(config, reason) {
|
|
33
|
+
stdout.write(`SKIP ${config.name} ${reason}\n`);
|
|
34
|
+
},
|
|
35
|
+
plannedSkip(entry) {
|
|
36
|
+
stdout.write(
|
|
37
|
+
`SKIP ${entry.serviceName} ${entry.type} ${normalizePath(entry.file)} 0s ${shortenMessage(entry.reason || "skipped")}\n`
|
|
38
|
+
);
|
|
39
|
+
},
|
|
40
|
+
taskStarted(task, targetConfig) {
|
|
41
|
+
if (mode !== "debug") return;
|
|
42
|
+
stdout.write(`RUN ${targetConfig.name} ${task.type} ${task.file}\n`);
|
|
43
|
+
},
|
|
44
|
+
taskFinished(task, outcome) {
|
|
45
|
+
if (mode === "json") return;
|
|
46
|
+
const status = outcome.status === "skipped" ? "SKIP" : outcome.failed ? "FAIL" : "PASS";
|
|
47
|
+
const duration = formatDuration(outcome.durationMs || 0);
|
|
48
|
+
const detail =
|
|
49
|
+
status === "FAIL"
|
|
50
|
+
? ` ${shortenMessage(outcome.error || firstFailureDetail(outcome) || "failed")}`
|
|
51
|
+
: outcome.status === "not_run"
|
|
52
|
+
? ` ${shortenMessage(outcome.reason || "not run")}`
|
|
53
|
+
: "";
|
|
54
|
+
stdout.write(
|
|
55
|
+
`${status} ${task.serviceName} ${displayTaskType(task)} ${normalizePath(task.file)} ${duration}${detail}\n`
|
|
56
|
+
);
|
|
57
|
+
},
|
|
58
|
+
telemetry(message) {
|
|
59
|
+
if (mode === "json") return;
|
|
60
|
+
stdout.write(`${message}\n`);
|
|
61
|
+
},
|
|
62
|
+
runSummary(results, durationMs, knownFailureIssueValidation = null) {
|
|
63
|
+
const lines =
|
|
64
|
+
mode === "debug"
|
|
65
|
+
? buildDebugRunSummaryLines(results, durationMs, knownFailureIssueValidation)
|
|
66
|
+
: buildCompactRunSummaryLines(results, durationMs, knownFailureIssueValidation);
|
|
67
|
+
for (const line of lines) stdout.write(`${line}\n`);
|
|
68
|
+
},
|
|
69
|
+
error(message) {
|
|
70
|
+
stderr.write(`${message}\n`);
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function displayTaskType(task) {
|
|
76
|
+
if (task.framework === "playwright") return "pw";
|
|
77
|
+
if (task.type === "integration") return "int";
|
|
78
|
+
return task.type;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function shortenMessage(message) {
|
|
82
|
+
return String(message).replace(/\s+/g, " ").trim().slice(0, 180);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function firstFailureDetail(outcome) {
|
|
86
|
+
return outcome.failureDetails?.[0]?.message || outcome.failureDetails?.[0]?.title || null;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function normalizePath(filePath) {
|
|
90
|
+
return String(filePath).split(path.sep).join("/");
|
|
91
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import React, { createElement, useEffect, useMemo, useState } from "react";
|
|
2
|
+
import { Box, Text, useApp, useInput } from "ink";
|
|
3
|
+
import { formatDuration } from "../../runner/formatting.mjs";
|
|
4
|
+
import { formatFileDetail, loadLatestRunArtifact, resolveFileSubject } from "../viewer.mjs";
|
|
5
|
+
|
|
6
|
+
export function WatchApp({ productDir, serviceFilter = null }) {
|
|
7
|
+
const { exit } = useApp();
|
|
8
|
+
const [artifact, setArtifact] = useState(() => safeLoadArtifact(productDir));
|
|
9
|
+
const [selectedIndex, setSelectedIndex] = useState(0);
|
|
10
|
+
|
|
11
|
+
const files = useMemo(() => collectFiles(artifact, serviceFilter), [artifact, serviceFilter]);
|
|
12
|
+
const selected = files[Math.min(selectedIndex, Math.max(0, files.length - 1))] || null;
|
|
13
|
+
|
|
14
|
+
useEffect(() => {
|
|
15
|
+
const timer = setInterval(() => {
|
|
16
|
+
setArtifact(safeLoadArtifact(productDir));
|
|
17
|
+
}, 1_000);
|
|
18
|
+
return () => clearInterval(timer);
|
|
19
|
+
}, [productDir]);
|
|
20
|
+
|
|
21
|
+
useInput((input, key) => {
|
|
22
|
+
if (input === "q") {
|
|
23
|
+
exit();
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
if (input === "r") {
|
|
27
|
+
setArtifact(safeLoadArtifact(productDir));
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (key.downArrow) {
|
|
31
|
+
setSelectedIndex((current) => Math.min(current + 1, Math.max(0, files.length - 1)));
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (key.upArrow) {
|
|
35
|
+
setSelectedIndex((current) => Math.max(0, current - 1));
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
if (!artifact) {
|
|
40
|
+
return createElement(Text, null, `No run artifact found in ${productDir}`);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return createElement(
|
|
44
|
+
Box,
|
|
45
|
+
{ flexDirection: "column" },
|
|
46
|
+
createElement(
|
|
47
|
+
Text,
|
|
48
|
+
null,
|
|
49
|
+
`testkit watch · q quit · r reload · ${artifact.run.status} · ${formatDuration(artifact.run.durationMs)}`
|
|
50
|
+
),
|
|
51
|
+
createElement(
|
|
52
|
+
Box,
|
|
53
|
+
{ marginTop: 1 },
|
|
54
|
+
createElement(
|
|
55
|
+
Box,
|
|
56
|
+
{ width: "40%", flexDirection: "column", marginRight: 2 },
|
|
57
|
+
createElement(Text, null, "Files"),
|
|
58
|
+
...files.map((entry, index) => {
|
|
59
|
+
const prefix = index === selectedIndex ? ">" : " ";
|
|
60
|
+
return createElement(
|
|
61
|
+
Text,
|
|
62
|
+
{ key: `${entry.service.name}:${entry.file.path}` },
|
|
63
|
+
`${prefix} ${entry.file.status.toUpperCase().padEnd(7)} ${entry.file.path}`
|
|
64
|
+
);
|
|
65
|
+
})
|
|
66
|
+
),
|
|
67
|
+
createElement(
|
|
68
|
+
Box,
|
|
69
|
+
{ width: "60%", flexDirection: "column" },
|
|
70
|
+
createElement(Text, null, "Details"),
|
|
71
|
+
...(selected
|
|
72
|
+
? formatFileDetail(productDir, artifact, selected, { logTail: 8 })
|
|
73
|
+
.slice(0, 28)
|
|
74
|
+
.map((line, index) => createElement(Text, { key: `${index}:${line}` }, line))
|
|
75
|
+
: [createElement(Text, { key: "empty" }, "No file results")])
|
|
76
|
+
)
|
|
77
|
+
)
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function safeLoadArtifact(productDir) {
|
|
82
|
+
try {
|
|
83
|
+
return loadLatestRunArtifact(productDir);
|
|
84
|
+
} catch {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function collectFiles(runArtifact, serviceFilter) {
|
|
90
|
+
if (!runArtifact) return [];
|
|
91
|
+
const entries = [];
|
|
92
|
+
for (const service of runArtifact.services || []) {
|
|
93
|
+
if (serviceFilter && service.name !== serviceFilter) continue;
|
|
94
|
+
for (const suite of service.suites || []) {
|
|
95
|
+
for (const file of suite.files || []) {
|
|
96
|
+
entries.push({ service, suite, file });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const failed = entries.filter((entry) => entry.file.status === "failed");
|
|
101
|
+
return (failed.length > 0 ? failed : entries).sort((left, right) =>
|
|
102
|
+
left.file.path.localeCompare(right.file.path)
|
|
103
|
+
);
|
|
104
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { formatDuration } from "../runner/formatting.mjs";
|
|
4
|
+
import { readLogTail } from "../runner/logs.mjs";
|
|
5
|
+
|
|
6
|
+
export function loadLatestRunArtifact(productDir) {
|
|
7
|
+
const artifactPath = path.join(productDir, ".testkit", "results", "latest.json");
|
|
8
|
+
if (!fs.existsSync(artifactPath)) {
|
|
9
|
+
throw new Error(`No run artifact found at ${path.relative(productDir, artifactPath)}`);
|
|
10
|
+
}
|
|
11
|
+
return JSON.parse(fs.readFileSync(artifactPath, "utf8"));
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function resolveFileSubject(runArtifact, selector = null, serviceFilter = null) {
|
|
15
|
+
const files = collectFiles(runArtifact, serviceFilter);
|
|
16
|
+
if (files.length === 0) {
|
|
17
|
+
throw new Error("No file results were found in the latest run artifact.");
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
if (!selector) {
|
|
21
|
+
return files.find((entry) => entry.file.status === "failed") || files[0];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const normalizedSelector = normalizePath(selector);
|
|
25
|
+
const exact = files.filter((entry) => entry.file.path === normalizedSelector);
|
|
26
|
+
if (exact.length === 1) return exact[0];
|
|
27
|
+
if (exact.length > 1) {
|
|
28
|
+
throw new Error(`Multiple files matched "${selector}". Re-run with --service to disambiguate.`);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const suffixMatches = files.filter((entry) => entry.file.path.endsWith(normalizedSelector));
|
|
32
|
+
if (suffixMatches.length === 1) return suffixMatches[0];
|
|
33
|
+
if (suffixMatches.length > 1) {
|
|
34
|
+
throw new Error(`Multiple files matched "${selector}". Re-run with --service to disambiguate.`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
throw new Error(`No file matched "${selector}".`);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function collectArtifactEntries(productDir, runArtifact, selector = null, serviceFilter = null) {
|
|
41
|
+
const files = selector
|
|
42
|
+
? [resolveFileSubject(runArtifact, selector, serviceFilter)]
|
|
43
|
+
: collectFiles(runArtifact, serviceFilter);
|
|
44
|
+
const entries = [];
|
|
45
|
+
|
|
46
|
+
for (const entry of files) {
|
|
47
|
+
for (const artifactRef of entry.file.artifacts || []) {
|
|
48
|
+
const absolutePath = path.join(productDir, artifactRef.path);
|
|
49
|
+
const payload = fs.existsSync(absolutePath)
|
|
50
|
+
? JSON.parse(fs.readFileSync(absolutePath, "utf8"))
|
|
51
|
+
: null;
|
|
52
|
+
entries.push({
|
|
53
|
+
...entry,
|
|
54
|
+
artifactRef,
|
|
55
|
+
absolutePath,
|
|
56
|
+
payload,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return entries;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function formatFileDetail(productDir, runArtifact, subject, options = {}) {
|
|
65
|
+
const lines = [];
|
|
66
|
+
lines.push(`File: ${subject.file.path}`);
|
|
67
|
+
lines.push(`Service: ${subject.service.name}`);
|
|
68
|
+
lines.push(`Suite: ${subject.suite.type}:${subject.suite.name}`);
|
|
69
|
+
lines.push(`Status: ${subject.file.status}`);
|
|
70
|
+
lines.push(`Duration: ${formatDuration(subject.file.durationMs || 0)}`);
|
|
71
|
+
if (subject.file.error) lines.push(`Error: ${subject.file.error}`);
|
|
72
|
+
|
|
73
|
+
if ((subject.file.failureDetails || []).length > 0) {
|
|
74
|
+
lines.push("");
|
|
75
|
+
lines.push("Failure Details:");
|
|
76
|
+
for (const detail of subject.file.failureDetails.slice(0, options.failureLimit || 5)) {
|
|
77
|
+
lines.push(` ${detail.title}`);
|
|
78
|
+
if (detail.message) lines.push(` ${detail.message}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const artifacts = collectArtifactEntries(productDir, runArtifact, subject.file.path, subject.service.name);
|
|
83
|
+
if (artifacts.length > 0) {
|
|
84
|
+
lines.push("");
|
|
85
|
+
lines.push("Artifacts:");
|
|
86
|
+
for (const entry of artifacts) {
|
|
87
|
+
lines.push(` ${entry.artifactRef.name}${entry.artifactRef.kind ? ` [${entry.artifactRef.kind}]` : ""}`);
|
|
88
|
+
if (entry.artifactRef.summary) lines.push(` ${entry.artifactRef.summary}`);
|
|
89
|
+
for (const previewLine of formatArtifactPreview(entry.payload, options.previewLength || 6)) {
|
|
90
|
+
lines.push(` ${previewLine}`);
|
|
91
|
+
}
|
|
92
|
+
lines.push(` ${entry.artifactRef.path}`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const logRefs = getServiceLogRefs(runArtifact, subject.service.name);
|
|
97
|
+
if (logRefs.length > 0) {
|
|
98
|
+
lines.push("");
|
|
99
|
+
lines.push("Backend Logs:");
|
|
100
|
+
for (const logRef of logRefs) {
|
|
101
|
+
lines.push(` ${logRef.runtimeLabel}`);
|
|
102
|
+
lines.push(` ${logRef.path}`);
|
|
103
|
+
const tail = readLogTail(path.join(productDir, logRef.path), options.logTail || 12);
|
|
104
|
+
for (const line of tail.slice(-Math.max(0, options.logTail || 12))) {
|
|
105
|
+
lines.push(` ${line}`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return lines;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function getServiceLogRefs(runArtifact, serviceName) {
|
|
114
|
+
return (runArtifact.logs?.services || []).filter((entry) => entry.serviceName === serviceName);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function formatArtifactPreview(payload, maxLines = 6) {
|
|
118
|
+
if (!payload) return ["artifact payload missing"];
|
|
119
|
+
if (payload.kind === "agentic-query") {
|
|
120
|
+
return formatAgenticArtifact(payload, maxLines);
|
|
121
|
+
}
|
|
122
|
+
if (payload.contentType === "text/plain" && typeof payload.data?.text === "string") {
|
|
123
|
+
return payload.data.text
|
|
124
|
+
.split(/\r?\n/)
|
|
125
|
+
.filter((line) => line.trim().length > 0)
|
|
126
|
+
.slice(0, maxLines);
|
|
127
|
+
}
|
|
128
|
+
const preview = JSON.stringify(payload.data, null, 2)
|
|
129
|
+
.split(/\r?\n/)
|
|
130
|
+
.slice(0, maxLines);
|
|
131
|
+
return preview;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function formatAgenticArtifact(payload, maxLines) {
|
|
135
|
+
const artifact = payload.data || {};
|
|
136
|
+
const lines = [];
|
|
137
|
+
if (artifact.query?.text) lines.push(`Query: ${artifact.query.text}`);
|
|
138
|
+
if (artifact.ui?.latestAssistantMessage?.content) {
|
|
139
|
+
lines.push(`Answer: ${String(artifact.ui.latestAssistantMessage.content).replace(/\s+/g, " ").trim()}`);
|
|
140
|
+
}
|
|
141
|
+
if (Array.isArray(artifact.ui?.resultTables) && artifact.ui.resultTables.length > 0) {
|
|
142
|
+
const table = artifact.ui.resultTables[0];
|
|
143
|
+
lines.push(`Table: ${table.rowCount} rows · columns ${table.columns.join(", ")}`);
|
|
144
|
+
}
|
|
145
|
+
return lines.slice(0, maxLines);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function collectFiles(runArtifact, serviceFilter = null) {
|
|
149
|
+
const files = [];
|
|
150
|
+
for (const service of runArtifact.services || []) {
|
|
151
|
+
if (serviceFilter && service.name !== serviceFilter) continue;
|
|
152
|
+
for (const suite of service.suites || []) {
|
|
153
|
+
for (const file of suite.files || []) {
|
|
154
|
+
files.push({ service, suite, file });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return files;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function normalizePath(filePath) {
|
|
162
|
+
return String(filePath).split(path.sep).join("/");
|
|
163
|
+
}
|
package/lib/runner/artifacts.mjs
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
|
|
9
9
|
const TIMINGS_FILENAME = "timings.json";
|
|
10
10
|
const RESULT_ARTIFACTS_DIRNAME = "artifacts";
|
|
11
|
+
const RESULT_LOGS_DIRNAME = "logs";
|
|
11
12
|
|
|
12
13
|
export function writeRunArtifact(productDir, artifact) {
|
|
13
14
|
const resultsDir = path.join(productDir, ".testkit", "results");
|
|
@@ -27,6 +28,10 @@ export function resetResultArtifacts(productDir) {
|
|
|
27
28
|
recursive: true,
|
|
28
29
|
force: true,
|
|
29
30
|
});
|
|
31
|
+
fs.rmSync(path.join(productDir, ".testkit", "results", RESULT_LOGS_DIRNAME), {
|
|
32
|
+
recursive: true,
|
|
33
|
+
force: true,
|
|
34
|
+
});
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
|
|
@@ -81,6 +86,27 @@ export function persistTaskArtifacts(productDir, task, emittedArtifacts) {
|
|
|
81
86
|
});
|
|
82
87
|
}
|
|
83
88
|
|
|
89
|
+
export function persistTaskOutputArtifacts(productDir, task, outputs) {
|
|
90
|
+
if (!Array.isArray(outputs) || outputs.length === 0) return [];
|
|
91
|
+
return persistTaskArtifacts(
|
|
92
|
+
productDir,
|
|
93
|
+
task,
|
|
94
|
+
outputs
|
|
95
|
+
.filter((entry) => typeof entry?.text === "string" && entry.text.trim().length > 0)
|
|
96
|
+
.map((entry) => ({
|
|
97
|
+
name: entry.name || "task-output",
|
|
98
|
+
kind: entry.kind || "runtime.output",
|
|
99
|
+
summary: entry.summary || summarizeOutput(entry.text),
|
|
100
|
+
contentType: entry.contentType || "text/plain",
|
|
101
|
+
emittedAt: entry.emittedAt || new Date().toISOString(),
|
|
102
|
+
data: {
|
|
103
|
+
stream: entry.stream || null,
|
|
104
|
+
text: entry.text,
|
|
105
|
+
},
|
|
106
|
+
}))
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
|
|
84
110
|
export function loadTimings(productDir) {
|
|
85
111
|
const filePath = path.join(productDir, ".testkit", TIMINGS_FILENAME);
|
|
86
112
|
if (!fs.existsSync(filePath)) {
|
|
@@ -113,3 +139,12 @@ function sanitizePathSegment(value) {
|
|
|
113
139
|
function normalizePath(filePath) {
|
|
114
140
|
return filePath.split(path.sep).join("/");
|
|
115
141
|
}
|
|
142
|
+
|
|
143
|
+
function summarizeOutput(text) {
|
|
144
|
+
const firstLine = String(text)
|
|
145
|
+
.split(/\r?\n/)
|
|
146
|
+
.map((line) => line.trim())
|
|
147
|
+
.find(Boolean);
|
|
148
|
+
if (!firstLine) return "captured task output";
|
|
149
|
+
return firstLine.slice(0, 120);
|
|
150
|
+
}
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
buildFileTimeoutEnv,
|
|
8
8
|
formatFileTimeoutBudgetError,
|
|
9
9
|
} from "../shared/file-timeout.mjs";
|
|
10
|
-
import { persistTaskArtifacts } from "./artifacts.mjs";
|
|
10
|
+
import { persistTaskArtifacts, persistTaskOutputArtifacts } from "./artifacts.mjs";
|
|
11
11
|
import { determineDefaultRuntimeFailure } from "./default-runtime-errors.mjs";
|
|
12
12
|
import { collectFailureDetailsFromRuntimeArtifacts } from "./failure-details.mjs";
|
|
13
13
|
import { RUNTIME_ARTIFACT_MARKER } from "../runtime-src/k6/artifacts.js";
|
|
@@ -15,7 +15,7 @@ import { readDatabaseUrl } from "./state-io.mjs";
|
|
|
15
15
|
import { buildTaskExecutionEnv } from "./template.mjs";
|
|
16
16
|
import { killChildProcess } from "./processes.mjs";
|
|
17
17
|
|
|
18
|
-
export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
|
|
18
|
+
export async function runHttpK6Task(targetConfig, task, lifecycle, lease, reporter = null) {
|
|
19
19
|
const baseUrl = targetConfig.testkit.local?.baseUrl;
|
|
20
20
|
if (!baseUrl) {
|
|
21
21
|
throw new Error(`Service "${targetConfig.name}" requires local.baseUrl for HTTP suites`);
|
|
@@ -34,11 +34,12 @@ export async function runHttpK6Task(targetConfig, task, lifecycle, lease) {
|
|
|
34
34
|
task,
|
|
35
35
|
lease,
|
|
36
36
|
["run", "--address", "127.0.0.1:0", "-e", `BASE_URL=${baseUrl}`, bundledFile],
|
|
37
|
-
lifecycle
|
|
37
|
+
lifecycle,
|
|
38
|
+
reporter
|
|
38
39
|
);
|
|
39
40
|
}
|
|
40
41
|
|
|
41
|
-
export async function runDalTask(targetConfig, task, lifecycle, lease) {
|
|
42
|
+
export async function runDalTask(targetConfig, task, lifecycle, lease, reporter = null) {
|
|
42
43
|
const databaseUrl = readDatabaseUrl(targetConfig.stateDir);
|
|
43
44
|
if (!databaseUrl) {
|
|
44
45
|
throw new Error(`Service "${targetConfig.name}" requires a database for DAL suites`);
|
|
@@ -57,12 +58,27 @@ export async function runDalTask(targetConfig, task, lifecycle, lease) {
|
|
|
57
58
|
task,
|
|
58
59
|
lease,
|
|
59
60
|
["run", "--address", "127.0.0.1:0", "-e", `DATABASE_URL=${databaseUrl}`, bundledFile],
|
|
60
|
-
lifecycle
|
|
61
|
+
lifecycle,
|
|
62
|
+
reporter
|
|
61
63
|
);
|
|
62
64
|
}
|
|
63
65
|
|
|
64
|
-
export async function runDefaultRuntimeTask(
|
|
66
|
+
export async function runDefaultRuntimeTask(
|
|
67
|
+
targetConfig,
|
|
68
|
+
task,
|
|
69
|
+
lease,
|
|
70
|
+
args,
|
|
71
|
+
lifecycle,
|
|
72
|
+
reporterOrFirstLine,
|
|
73
|
+
maybeFirstLine
|
|
74
|
+
) {
|
|
65
75
|
const k6Binary = resolveK6Binary();
|
|
76
|
+
const reporter =
|
|
77
|
+
reporterOrFirstLine && typeof reporterOrFirstLine === "object"
|
|
78
|
+
? reporterOrFirstLine
|
|
79
|
+
: null;
|
|
80
|
+
const firstLine =
|
|
81
|
+
typeof reporterOrFirstLine === "function" ? reporterOrFirstLine : maybeFirstLine;
|
|
66
82
|
const getFirstLine = firstLine || defaultFirstLine;
|
|
67
83
|
const summaryFile = buildDefaultRuntimeSummaryPath(lease, task);
|
|
68
84
|
fs.mkdirSync(path.dirname(summaryFile), { recursive: true });
|
|
@@ -91,7 +107,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
91
107
|
if (subprocess.pid) interruptSubprocess();
|
|
92
108
|
else subprocess.once?.("spawn", interruptSubprocess);
|
|
93
109
|
}
|
|
94
|
-
|
|
110
|
+
reporter?.taskStarted?.(task, targetConfig);
|
|
95
111
|
let result;
|
|
96
112
|
let timedOut;
|
|
97
113
|
try {
|
|
@@ -102,8 +118,6 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
102
118
|
|
|
103
119
|
const stdout = parseDefaultRuntimeOutput(result.stdout || "");
|
|
104
120
|
const stderr = parseDefaultRuntimeOutput(result.stderr || "");
|
|
105
|
-
if (stdout.visibleOutput) process.stdout.write(stdout.visibleOutput);
|
|
106
|
-
if (stderr.visibleOutput) process.stderr.write(stderr.visibleOutput);
|
|
107
121
|
|
|
108
122
|
const summary = readDefaultRuntimeSummary(summaryFile);
|
|
109
123
|
const rawRuntimeArtifacts = [...stdout.artifacts, ...stderr.artifacts];
|
|
@@ -112,6 +126,26 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
112
126
|
task,
|
|
113
127
|
rawRuntimeArtifacts
|
|
114
128
|
);
|
|
129
|
+
const outputArtifacts = persistTaskOutputArtifacts(targetConfig.productDir, task, [
|
|
130
|
+
stdout.visibleOutput
|
|
131
|
+
? {
|
|
132
|
+
name: "default-runtime-stdout",
|
|
133
|
+
kind: "runtime.output",
|
|
134
|
+
summary: defaultFirstLine(stdout.visibleOutput) || "captured stdout",
|
|
135
|
+
stream: "stdout",
|
|
136
|
+
text: stdout.visibleOutput,
|
|
137
|
+
}
|
|
138
|
+
: null,
|
|
139
|
+
stderr.visibleOutput
|
|
140
|
+
? {
|
|
141
|
+
name: "default-runtime-stderr",
|
|
142
|
+
kind: "runtime.output",
|
|
143
|
+
summary: defaultFirstLine(stderr.visibleOutput) || "captured stderr",
|
|
144
|
+
stream: "stderr",
|
|
145
|
+
text: stderr.visibleOutput,
|
|
146
|
+
}
|
|
147
|
+
: null,
|
|
148
|
+
]);
|
|
115
149
|
const failureDetails = collectFailureDetailsFromRuntimeArtifacts(rawRuntimeArtifacts);
|
|
116
150
|
const runtimeError = timedOut
|
|
117
151
|
? formatFileTimeoutBudgetError(fileTimeoutSeconds)
|
|
@@ -125,7 +159,7 @@ export async function runDefaultRuntimeTask(targetConfig, task, lease, args, lif
|
|
|
125
159
|
durationMs: finishedAt - startedAt,
|
|
126
160
|
startedAt,
|
|
127
161
|
finishedAt,
|
|
128
|
-
artifacts: runtimeArtifacts,
|
|
162
|
+
artifacts: [...runtimeArtifacts, ...outputArtifacts],
|
|
129
163
|
failureDetails,
|
|
130
164
|
};
|
|
131
165
|
}
|