@elench/testkit 0.1.34 → 0.1.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -11
- package/lib/cli/args.mjs +28 -20
- package/lib/cli/args.test.mjs +22 -17
- package/lib/cli/index.mjs +23 -38
- package/lib/runner/orchestrator.mjs +13 -16
- package/lib/runner/planning.mjs +11 -10
- package/lib/runner/planning.test.mjs +6 -5
- package/lib/runner/reporting.mjs +13 -21
- package/lib/runner/reporting.test.mjs +12 -16
- package/lib/runner/results.mjs +2 -1
- package/lib/runner/selection.mjs +7 -7
- package/lib/runner/selection.test.mjs +5 -6
- package/lib/runner/suite-selection.mjs +91 -0
- package/lib/runner/suite-selection.test.mjs +42 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -15,13 +15,11 @@ cd my-product
|
|
|
15
15
|
npx @elench/testkit
|
|
16
16
|
|
|
17
17
|
# Filter by type
|
|
18
|
-
npx @elench/testkit int
|
|
19
|
-
npx @elench/testkit dal
|
|
20
|
-
npx @elench/testkit e2e
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
npx @elench/testkit --framework playwright
|
|
24
|
-
npx @elench/testkit --framework default
|
|
18
|
+
npx @elench/testkit --type int
|
|
19
|
+
npx @elench/testkit --type dal
|
|
20
|
+
npx @elench/testkit --type e2e
|
|
21
|
+
npx @elench/testkit --type int,e2e,dal
|
|
22
|
+
npx @elench/testkit --type pw
|
|
25
23
|
|
|
26
24
|
# Parallelize with isolated worker stacks
|
|
27
25
|
npx @elench/testkit --jobs 3
|
|
@@ -30,14 +28,15 @@ npx @elench/testkit --jobs 3
|
|
|
30
28
|
npx @elench/testkit --shard 1/3
|
|
31
29
|
|
|
32
30
|
# Specific service / suite
|
|
33
|
-
npx @elench/testkit frontend
|
|
34
|
-
npx @elench/testkit api int -s health
|
|
31
|
+
npx @elench/testkit --service frontend --type pw -s navigation
|
|
32
|
+
npx @elench/testkit --service api --type int -s health
|
|
33
|
+
npx @elench/testkit --type int,e2e,dal -s dal:queries
|
|
35
34
|
|
|
36
35
|
# Exact file
|
|
37
|
-
npx @elench/testkit int --file __testkit__/health/health.int.testkit.ts
|
|
36
|
+
npx @elench/testkit --type int --file __testkit__/health/health.int.testkit.ts
|
|
38
37
|
|
|
39
38
|
# Deterministic git-trackable status snapshot
|
|
40
|
-
npx @elench/testkit int --write-status
|
|
39
|
+
npx @elench/testkit --type int --write-status
|
|
41
40
|
|
|
42
41
|
# Lifecycle
|
|
43
42
|
npx @elench/testkit status
|
package/lib/cli/args.mjs
CHANGED
|
@@ -1,35 +1,43 @@
|
|
|
1
1
|
import path from "path";
|
|
2
|
+
import { normalizeTypeValues, parseSuiteSelectors } from "../runner/suite-selection.mjs";
|
|
2
3
|
|
|
3
|
-
export const
|
|
4
|
+
export const POSITIONAL_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
|
|
4
5
|
export const LIFECYCLE = new Set(["status", "destroy", "cleanup"]);
|
|
5
|
-
export const RESERVED = new Set([...
|
|
6
|
+
export const RESERVED = new Set([...POSITIONAL_TYPES, ...LIFECYCLE]);
|
|
6
7
|
|
|
7
|
-
export function resolveCliSelection({ first, second,
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
export function resolveCliSelection({ first, second, third }) {
|
|
9
|
+
if (second || third) {
|
|
10
|
+
throw new Error(`Unexpected extra positional arguments. Use --service and --type instead.`);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
let lifecycle = null;
|
|
14
|
+
let positionalType = null;
|
|
10
15
|
|
|
11
|
-
if (first &&
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
type = first;
|
|
16
|
+
if (first && LIFECYCLE.has(first)) {
|
|
17
|
+
lifecycle = first;
|
|
18
|
+
} else if (first && POSITIONAL_TYPES.has(first)) {
|
|
19
|
+
positionalType = first;
|
|
16
20
|
} else if (first) {
|
|
17
21
|
throw new Error(
|
|
18
|
-
`Unknown argument "${first}". Expected a
|
|
19
|
-
`or suite type (int, e2e, dal, all).`
|
|
22
|
+
`Unknown argument "${first}". Expected a lifecycle command (status, destroy, cleanup) ` +
|
|
23
|
+
`or suite type (int, e2e, dal, load, pw, all).`
|
|
20
24
|
);
|
|
21
25
|
}
|
|
22
26
|
|
|
23
|
-
return {
|
|
27
|
+
return { lifecycle, positionalType };
|
|
24
28
|
}
|
|
25
29
|
|
|
26
|
-
export function
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
export function parseTypeOption(values, positionalType = null) {
|
|
31
|
+
const input = [];
|
|
32
|
+
if (positionalType) input.push(positionalType);
|
|
33
|
+
if (Array.isArray(values)) input.push(...values);
|
|
34
|
+
else if (values) input.push(values);
|
|
35
|
+
return normalizeTypeValues(input);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function parseSuiteOption(values) {
|
|
39
|
+
const input = Array.isArray(values) ? values : values ? [values] : [];
|
|
40
|
+
return parseSuiteSelectors(input);
|
|
33
41
|
}
|
|
34
42
|
|
|
35
43
|
export function parseJobsOption(value) {
|
package/lib/cli/args.test.mjs
CHANGED
|
@@ -2,35 +2,36 @@ import { describe, expect, it } from "vitest";
|
|
|
2
2
|
import {
|
|
3
3
|
parseJobsOption,
|
|
4
4
|
parseShardOption,
|
|
5
|
+
parseSuiteOption,
|
|
6
|
+
parseTypeOption,
|
|
5
7
|
resolveRequestedFiles,
|
|
6
8
|
resolveCliSelection,
|
|
7
|
-
validateFrameworkOption,
|
|
8
9
|
} from "./args.mjs";
|
|
9
10
|
|
|
10
11
|
describe("cli-args", () => {
|
|
11
|
-
it("resolves a
|
|
12
|
+
it("resolves a positional suite type", () => {
|
|
12
13
|
expect(
|
|
13
14
|
resolveCliSelection({
|
|
14
|
-
first: "
|
|
15
|
-
second:
|
|
16
|
-
|
|
15
|
+
first: "int",
|
|
16
|
+
second: null,
|
|
17
|
+
third: null,
|
|
17
18
|
})
|
|
18
19
|
).toEqual({
|
|
19
|
-
|
|
20
|
-
|
|
20
|
+
lifecycle: null,
|
|
21
|
+
positionalType: "int",
|
|
21
22
|
});
|
|
22
23
|
});
|
|
23
24
|
|
|
24
|
-
it("resolves
|
|
25
|
+
it("resolves lifecycle commands", () => {
|
|
25
26
|
expect(
|
|
26
27
|
resolveCliSelection({
|
|
27
|
-
first: "
|
|
28
|
+
first: "status",
|
|
28
29
|
second: null,
|
|
29
|
-
|
|
30
|
+
third: null,
|
|
30
31
|
})
|
|
31
32
|
).toEqual({
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
lifecycle: "status",
|
|
34
|
+
positionalType: null,
|
|
34
35
|
});
|
|
35
36
|
});
|
|
36
37
|
|
|
@@ -39,15 +40,19 @@ describe("cli-args", () => {
|
|
|
39
40
|
resolveCliSelection({
|
|
40
41
|
first: "mystery",
|
|
41
42
|
second: null,
|
|
42
|
-
|
|
43
|
+
third: null,
|
|
43
44
|
})
|
|
44
45
|
).toThrow('Unknown argument "mystery"');
|
|
45
46
|
});
|
|
46
47
|
|
|
47
|
-
it("
|
|
48
|
-
expect(
|
|
49
|
-
expect(
|
|
50
|
-
|
|
48
|
+
it("parses types and suite selectors", () => {
|
|
49
|
+
expect(parseTypeOption(["e2e,dal"], "int")).toEqual(["int", "e2e", "dal"]);
|
|
50
|
+
expect(() => parseTypeOption(["all", "int"])).toThrow("cannot be combined");
|
|
51
|
+
|
|
52
|
+
expect(parseSuiteOption(["auth,dal:queries"])).toEqual([
|
|
53
|
+
{ kind: "plain", name: "auth", raw: "auth" },
|
|
54
|
+
{ kind: "typed", type: "dal", name: "queries", raw: "dal:queries" },
|
|
55
|
+
]);
|
|
51
56
|
});
|
|
52
57
|
|
|
53
58
|
it("parses and validates jobs", () => {
|
package/lib/cli/index.mjs
CHANGED
|
@@ -3,9 +3,10 @@ import { loadConfigs } from "../config/index.mjs";
|
|
|
3
3
|
import {
|
|
4
4
|
parseJobsOption,
|
|
5
5
|
parseShardOption,
|
|
6
|
+
parseSuiteOption,
|
|
7
|
+
parseTypeOption,
|
|
6
8
|
resolveRequestedFiles,
|
|
7
9
|
resolveCliSelection,
|
|
8
|
-
validateFrameworkOption,
|
|
9
10
|
} from "./args.mjs";
|
|
10
11
|
import * as runner from "../runner/index.mjs";
|
|
11
12
|
|
|
@@ -13,7 +14,11 @@ export function run() {
|
|
|
13
14
|
const cli = cac("testkit");
|
|
14
15
|
|
|
15
16
|
cli
|
|
16
|
-
.command("[first] [second] [third]", "Run test suites
|
|
17
|
+
.command("[first] [second] [third]", "Run test suites")
|
|
18
|
+
.option("--service <name>", "Run only one service")
|
|
19
|
+
.option("-t, --type <name>", "Run specific suite type(s): int, e2e, dal, load, pw, all", {
|
|
20
|
+
default: [],
|
|
21
|
+
})
|
|
17
22
|
.option("-s, --suite <name>", "Run specific suite(s)", { default: [] })
|
|
18
23
|
.option("-f, --file <path>", "Run specific file(s)", { default: [] })
|
|
19
24
|
.option("--dir <path>", "Explicit product directory")
|
|
@@ -21,76 +26,56 @@ export function run() {
|
|
|
21
26
|
default: "1",
|
|
22
27
|
})
|
|
23
28
|
.option("--shard <i/n>", "Run only shard i of n at suite granularity")
|
|
24
|
-
.option("--framework <name>", "Filter by framework (default, playwright, all)", {
|
|
25
|
-
default: "all",
|
|
26
|
-
})
|
|
27
29
|
.option("--write-status", "Write a deterministic testkit.status.json snapshot")
|
|
28
30
|
.option("--allow-partial-status", "Allow --write-status for filtered runs")
|
|
29
31
|
.action(async (first, second, third, options) => {
|
|
30
|
-
|
|
31
|
-
//
|
|
32
|
-
// From product dir:
|
|
33
|
-
// testkit → all services, all types
|
|
34
|
-
// testkit int -s health → all services, int, health
|
|
35
|
-
// testkit api int → one service, int
|
|
36
|
-
// testkit api → one service, all types
|
|
37
|
-
//
|
|
38
|
-
// From workspace root:
|
|
39
|
-
// testkit --dir my-product int → all services, int
|
|
40
|
-
// testkit --dir my-product api int → one service, int
|
|
41
|
-
|
|
42
|
-
// Now resolve service vs type from remaining args
|
|
43
|
-
const allConfigs = await loadConfigs({ dir: options.dir });
|
|
44
|
-
const serviceNames = new Set(allConfigs.map((config) => config.name));
|
|
45
|
-
const { service, type } = resolveCliSelection({
|
|
32
|
+
const { lifecycle, positionalType } = resolveCliSelection({
|
|
46
33
|
first,
|
|
47
34
|
second,
|
|
48
|
-
|
|
35
|
+
third,
|
|
49
36
|
});
|
|
50
|
-
const
|
|
51
|
-
|
|
37
|
+
const allConfigs = await loadConfigs({ dir: options.dir });
|
|
38
|
+
const configs = options.service
|
|
39
|
+
? allConfigs.filter((config) => config.name === options.service)
|
|
52
40
|
: allConfigs;
|
|
53
|
-
if (service && configs.length === 0) {
|
|
41
|
+
if (options.service && configs.length === 0) {
|
|
54
42
|
const available = allConfigs.map((config) => config.name).join(", ");
|
|
55
|
-
throw new Error(`Service "${service}" not found. Available: ${available}`);
|
|
43
|
+
throw new Error(`Service "${options.service}" not found. Available: ${available}`);
|
|
56
44
|
}
|
|
57
45
|
|
|
58
46
|
// Lifecycle commands
|
|
59
|
-
if (
|
|
47
|
+
if (lifecycle === "cleanup") {
|
|
60
48
|
await runner.cleanup(allConfigs[0]?.productDir || process.cwd());
|
|
61
49
|
return;
|
|
62
50
|
}
|
|
63
51
|
|
|
64
|
-
if (
|
|
52
|
+
if (lifecycle === "status" || lifecycle === "destroy") {
|
|
65
53
|
for (const config of configs) {
|
|
66
54
|
if (configs.length > 1) console.log(`\n── ${config.name} ──`);
|
|
67
|
-
if (
|
|
55
|
+
if (lifecycle === "status") runner.showStatus(config);
|
|
68
56
|
else await runner.destroy(config);
|
|
69
57
|
}
|
|
70
58
|
return;
|
|
71
59
|
}
|
|
72
60
|
|
|
73
|
-
const framework = validateFrameworkOption(options.framework);
|
|
74
|
-
|
|
75
61
|
const jobs = parseJobsOption(options.jobs);
|
|
76
62
|
const shard = parseShardOption(options.shard);
|
|
77
|
-
|
|
78
|
-
const
|
|
79
|
-
const suiteNames = Array.isArray(options.suite) ? options.suite : [options.suite].filter(Boolean);
|
|
63
|
+
const typeValues = parseTypeOption(options.type, positionalType);
|
|
64
|
+
const suiteSelectors = parseSuiteOption(options.suite);
|
|
80
65
|
const rawFileNames = Array.isArray(options.file) ? options.file : [options.file].filter(Boolean);
|
|
81
66
|
const productDir = allConfigs[0]?.productDir || process.cwd();
|
|
82
67
|
const fileNames = resolveRequestedFiles(rawFileNames, productDir, process.cwd());
|
|
83
68
|
await runner.runAll(
|
|
84
69
|
configs,
|
|
85
|
-
|
|
86
|
-
|
|
70
|
+
typeValues,
|
|
71
|
+
suiteSelectors,
|
|
87
72
|
{
|
|
88
73
|
...options,
|
|
89
|
-
|
|
74
|
+
typeValues,
|
|
90
75
|
fileNames,
|
|
91
76
|
jobs,
|
|
92
77
|
shard,
|
|
93
|
-
serviceFilter: service,
|
|
78
|
+
serviceFilter: options.service || null,
|
|
94
79
|
},
|
|
95
80
|
allConfigs
|
|
96
81
|
);
|
|
@@ -33,7 +33,7 @@ import { createWorker, runWorker } from "./worker-loop.mjs";
|
|
|
33
33
|
import { findUnmatchedRequestedFiles, isFullRunSelection } from "./selection.mjs";
|
|
34
34
|
import { uploadTelemetryArtifact } from "../telemetry/index.mjs";
|
|
35
35
|
|
|
36
|
-
export async function runAll(configs,
|
|
36
|
+
export async function runAll(configs, typeValues, suiteSelectors, opts, allConfigs = configs) {
|
|
37
37
|
const configMap = new Map(allConfigs.map((config) => [config.name, config]));
|
|
38
38
|
const startedAt = Date.now();
|
|
39
39
|
const telemetry = configs[0]?.telemetry || null;
|
|
@@ -51,9 +51,8 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
|
|
|
51
51
|
if (requestedFiles.length > 0) {
|
|
52
52
|
const unmatchedFiles = findUnmatchedRequestedFiles(
|
|
53
53
|
configs,
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
opts.framework || "all",
|
|
54
|
+
typeValues,
|
|
55
|
+
suiteSelectors,
|
|
57
56
|
requestedFiles,
|
|
58
57
|
collectSuites,
|
|
59
58
|
normalizePathSeparators
|
|
@@ -69,9 +68,9 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
|
|
|
69
68
|
opts.writeStatus &&
|
|
70
69
|
!opts.allowPartialStatus &&
|
|
71
70
|
!isFullRunSelection(
|
|
72
|
-
|
|
71
|
+
typeValues,
|
|
72
|
+
suiteSelectors,
|
|
73
73
|
requestedFiles,
|
|
74
|
-
opts.framework || "all",
|
|
75
74
|
opts.shard || null,
|
|
76
75
|
opts.serviceFilter || null
|
|
77
76
|
)
|
|
@@ -82,7 +81,7 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
|
|
|
82
81
|
);
|
|
83
82
|
}
|
|
84
83
|
|
|
85
|
-
const servicePlans = collectServicePlans(configs, configMap,
|
|
84
|
+
const servicePlans = collectServicePlans(configs, configMap, typeValues, suiteSelectors, opts);
|
|
86
85
|
const trackers = buildServiceTrackers(servicePlans, startedAt);
|
|
87
86
|
const executedPlans = servicePlans.filter((plan) => !plan.skipped);
|
|
88
87
|
let workerCount = 0;
|
|
@@ -144,10 +143,9 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
|
|
|
144
143
|
finishedAt,
|
|
145
144
|
requestedJobs: opts.jobs || 1,
|
|
146
145
|
workerCount,
|
|
147
|
-
|
|
148
|
-
|
|
146
|
+
typeValues,
|
|
147
|
+
suiteSelectors,
|
|
149
148
|
fileNames: requestedFiles,
|
|
150
|
-
framework: opts.framework || "all",
|
|
151
149
|
shard: opts.shard || null,
|
|
152
150
|
serviceFilter: opts.serviceFilter || null,
|
|
153
151
|
metadata,
|
|
@@ -161,10 +159,9 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
|
|
|
161
159
|
buildStatusArtifact({
|
|
162
160
|
productDir,
|
|
163
161
|
results,
|
|
164
|
-
|
|
165
|
-
|
|
162
|
+
typeValues,
|
|
163
|
+
suiteSelectors,
|
|
166
164
|
fileNames: requestedFiles,
|
|
167
|
-
framework: opts.framework || "all",
|
|
168
165
|
shard: opts.shard || null,
|
|
169
166
|
serviceFilter: opts.serviceFilter || null,
|
|
170
167
|
metadata,
|
|
@@ -188,17 +185,17 @@ export async function runAll(configs, suiteType, suiteNames, opts, allConfigs =
|
|
|
188
185
|
}
|
|
189
186
|
}
|
|
190
187
|
|
|
191
|
-
function collectServicePlans(configs, configMap,
|
|
188
|
+
function collectServicePlans(configs, configMap, typeValues, suiteSelectors, opts) {
|
|
192
189
|
return configs.map((config) => {
|
|
193
190
|
console.log(`\n══ ${config.name} ══`);
|
|
194
191
|
const suites = applyShard(
|
|
195
|
-
collectSuites(config,
|
|
192
|
+
collectSuites(config, typeValues, suiteSelectors, opts.fileNames || []),
|
|
196
193
|
opts.shard
|
|
197
194
|
);
|
|
198
195
|
|
|
199
196
|
if (suites.length === 0) {
|
|
200
197
|
console.log(
|
|
201
|
-
`No test files for ${config.name}
|
|
198
|
+
`No test files for ${config.name} types=${typeValues.join(",") || "all"} suites=${suiteSelectors.map((selector) => selector.raw).join(",") || "all"} files=${(opts.fileNames || []).join(",") || "all"} — skipping`
|
|
202
199
|
);
|
|
203
200
|
return {
|
|
204
201
|
config,
|
package/lib/runner/planning.mjs
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import { buildTimingKey, estimateTaskDuration } from "../timing/index.mjs";
|
|
2
|
+
import {
|
|
3
|
+
matchesSelectedTypes,
|
|
4
|
+
matchesSuiteSelectors,
|
|
5
|
+
suiteSelectionType,
|
|
6
|
+
} from "./suite-selection.mjs";
|
|
2
7
|
|
|
3
8
|
const TYPE_ORDER = ["dal", "integration", "e2e", "load"];
|
|
4
9
|
|
|
@@ -34,26 +39,21 @@ export function resolveRuntimeConfigs(targetConfig, configMap) {
|
|
|
34
39
|
return ordered;
|
|
35
40
|
}
|
|
36
41
|
|
|
37
|
-
export function collectSuites(config,
|
|
38
|
-
const types =
|
|
39
|
-
suiteType === "all"
|
|
40
|
-
? orderedTypes(Object.keys(config.suites))
|
|
41
|
-
: [suiteType === "int" ? "integration" : suiteType];
|
|
42
|
-
|
|
43
|
-
const selectedNames = new Set(suiteNames);
|
|
42
|
+
export function collectSuites(config, typeValues, suiteSelectors, fileNames = []) {
|
|
44
43
|
const selectedFiles = new Set(fileNames.map(normalizePathSeparators));
|
|
45
44
|
const suites = [];
|
|
46
45
|
let orderIndex = 0;
|
|
47
46
|
|
|
48
|
-
for (const type of
|
|
47
|
+
for (const type of orderedTypes(Object.keys(config.suites))) {
|
|
49
48
|
for (const suite of config.suites[type] || []) {
|
|
50
49
|
const framework = suite.framework || "k6";
|
|
50
|
+
const displayType = suiteSelectionType(type, framework);
|
|
51
51
|
const files =
|
|
52
52
|
selectedFiles.size === 0
|
|
53
53
|
? suite.files
|
|
54
54
|
: suite.files.filter((file) => selectedFiles.has(normalizePathSeparators(file)));
|
|
55
|
-
if (
|
|
56
|
-
if (
|
|
55
|
+
if (!matchesSelectedTypes(displayType, typeValues)) continue;
|
|
56
|
+
if (!matchesSuiteSelectors(displayType, suite.name, suiteSelectors)) continue;
|
|
57
57
|
if (files.length === 0) continue;
|
|
58
58
|
|
|
59
59
|
suites.push({
|
|
@@ -61,6 +61,7 @@ export function collectSuites(config, suiteType, suiteNames, frameworkFilter, fi
|
|
|
61
61
|
files,
|
|
62
62
|
framework,
|
|
63
63
|
type,
|
|
64
|
+
displayType,
|
|
64
65
|
orderIndex,
|
|
65
66
|
sortKey: `${type}:${suite.name}`,
|
|
66
67
|
weight:
|
|
@@ -38,7 +38,7 @@ describe("runner-planning", () => {
|
|
|
38
38
|
expect(() => resolveRuntimeConfigs(frontend, configMap)).toThrow("Dependency cycle");
|
|
39
39
|
});
|
|
40
40
|
|
|
41
|
-
it("collects suites with
|
|
41
|
+
it("collects suites with user-facing type selection", () => {
|
|
42
42
|
const config = makeConfig("api", {
|
|
43
43
|
suites: {
|
|
44
44
|
integration: [
|
|
@@ -57,15 +57,17 @@ describe("runner-planning", () => {
|
|
|
57
57
|
},
|
|
58
58
|
});
|
|
59
59
|
|
|
60
|
-
expect(collectSuites(config, "int", [],
|
|
60
|
+
expect(collectSuites(config, ["int"], [], [])[0]).toMatchObject({
|
|
61
61
|
name: "health",
|
|
62
62
|
type: "integration",
|
|
63
|
+
displayType: "int",
|
|
63
64
|
framework: "k6",
|
|
64
65
|
weight: 1,
|
|
65
66
|
});
|
|
66
67
|
|
|
67
|
-
expect(collectSuites(config, "
|
|
68
|
+
expect(collectSuites(config, ["pw"], [], [])[0]).toMatchObject({
|
|
68
69
|
name: "auth",
|
|
70
|
+
displayType: "pw",
|
|
69
71
|
framework: "playwright",
|
|
70
72
|
weight: 2,
|
|
71
73
|
});
|
|
@@ -73,9 +75,8 @@ describe("runner-planning", () => {
|
|
|
73
75
|
expect(
|
|
74
76
|
collectSuites(
|
|
75
77
|
config,
|
|
76
|
-
"all",
|
|
78
|
+
["all"],
|
|
77
79
|
[],
|
|
78
|
-
"all",
|
|
79
80
|
["tests/frontend/e2e/signup.pw.testkit.ts"]
|
|
80
81
|
)[0]
|
|
81
82
|
).toMatchObject({
|
package/lib/runner/reporting.mjs
CHANGED
|
@@ -3,10 +3,9 @@ import path from "path";
|
|
|
3
3
|
export function buildStatusArtifact({
|
|
4
4
|
productDir,
|
|
5
5
|
results,
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
typeValues,
|
|
7
|
+
suiteSelectors,
|
|
8
8
|
fileNames,
|
|
9
|
-
framework,
|
|
10
9
|
shard,
|
|
11
10
|
serviceFilter,
|
|
12
11
|
metadata,
|
|
@@ -49,22 +48,22 @@ export function buildStatusArtifact({
|
|
|
49
48
|
};
|
|
50
49
|
|
|
51
50
|
const scope = {
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
types: [...(typeValues || ["all"])].sort(),
|
|
52
|
+
suiteSelectors: [...(suiteSelectors || [])].map((selector) => selector.raw).sort(),
|
|
54
53
|
fileNames: [...(fileNames || [])].sort(),
|
|
55
|
-
framework: formatFrameworkForArtifact(framework || "all"),
|
|
56
54
|
shard: shard || null,
|
|
57
55
|
serviceFilter: serviceFilter || null,
|
|
58
56
|
};
|
|
59
57
|
scope.isFullRun =
|
|
60
|
-
scope.
|
|
58
|
+
scope.types.length === 1 &&
|
|
59
|
+
scope.types[0] === "all" &&
|
|
60
|
+
scope.suiteSelectors.length === 0 &&
|
|
61
61
|
scope.fileNames.length === 0 &&
|
|
62
|
-
scope.framework === "all" &&
|
|
63
62
|
scope.shard === null &&
|
|
64
63
|
scope.serviceFilter === null;
|
|
65
64
|
|
|
66
65
|
return {
|
|
67
|
-
schemaVersion:
|
|
66
|
+
schemaVersion: 2,
|
|
68
67
|
source: "testkit",
|
|
69
68
|
notice: "Generated file. Do not edit manually.",
|
|
70
69
|
product: {
|
|
@@ -88,10 +87,9 @@ export function buildRunArtifact({
|
|
|
88
87
|
finishedAt,
|
|
89
88
|
requestedJobs,
|
|
90
89
|
workerCount,
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
typeValues,
|
|
91
|
+
suiteSelectors,
|
|
93
92
|
fileNames,
|
|
94
|
-
framework,
|
|
95
93
|
shard,
|
|
96
94
|
serviceFilter,
|
|
97
95
|
metadata,
|
|
@@ -109,7 +107,7 @@ export function buildRunArtifact({
|
|
|
109
107
|
const dbBackend = summarizeDbBackend(results);
|
|
110
108
|
|
|
111
109
|
return {
|
|
112
|
-
schemaVersion:
|
|
110
|
+
schemaVersion: 2,
|
|
113
111
|
source: "testkit",
|
|
114
112
|
generatedAt: new Date(finishedAt).toISOString(),
|
|
115
113
|
product: {
|
|
@@ -126,10 +124,9 @@ export function buildRunArtifact({
|
|
|
126
124
|
requestedJobs,
|
|
127
125
|
workerCount,
|
|
128
126
|
dbBackend,
|
|
129
|
-
|
|
130
|
-
|
|
127
|
+
types: typeValues,
|
|
128
|
+
suiteSelectors: suiteSelectors.map((selector) => selector.raw),
|
|
131
129
|
fileNames,
|
|
132
|
-
framework: formatFrameworkForArtifact(framework),
|
|
133
130
|
shard,
|
|
134
131
|
serviceFilter,
|
|
135
132
|
testkitVersion: metadata.testkitVersion,
|
|
@@ -173,8 +170,3 @@ export function buildRunArtifact({
|
|
|
173
170
|
})),
|
|
174
171
|
};
|
|
175
172
|
}
|
|
176
|
-
|
|
177
|
-
function formatFrameworkForArtifact(framework) {
|
|
178
|
-
if (framework === "k6") return "default";
|
|
179
|
-
return framework;
|
|
180
|
-
}
|
|
@@ -49,10 +49,9 @@ describe("runner reporting", () => {
|
|
|
49
49
|
finishedAt: 4000,
|
|
50
50
|
requestedJobs: 2,
|
|
51
51
|
workerCount: 1,
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
typeValues: ["all"],
|
|
53
|
+
suiteSelectors: [],
|
|
54
54
|
fileNames: [],
|
|
55
|
-
framework: "all",
|
|
56
55
|
shard: null,
|
|
57
56
|
serviceFilter: null,
|
|
58
57
|
metadata: {
|
|
@@ -93,7 +92,7 @@ describe("runner reporting", () => {
|
|
|
93
92
|
suites: [
|
|
94
93
|
{
|
|
95
94
|
name: "health",
|
|
96
|
-
type: "
|
|
95
|
+
type: "int",
|
|
97
96
|
framework: "k6",
|
|
98
97
|
files: [
|
|
99
98
|
{ path: "tests/api/integration/a.int.testkit.ts", status: "passed" },
|
|
@@ -103,10 +102,9 @@ describe("runner reporting", () => {
|
|
|
103
102
|
],
|
|
104
103
|
},
|
|
105
104
|
],
|
|
106
|
-
|
|
107
|
-
|
|
105
|
+
typeValues: ["int"],
|
|
106
|
+
suiteSelectors: [{ kind: "plain", name: "health", raw: "health" }],
|
|
108
107
|
fileNames: ["tests/api/integration/b.int.testkit.ts"],
|
|
109
|
-
framework: "default",
|
|
110
108
|
shard: null,
|
|
111
109
|
serviceFilter: "api",
|
|
112
110
|
metadata: {
|
|
@@ -119,7 +117,7 @@ describe("runner reporting", () => {
|
|
|
119
117
|
});
|
|
120
118
|
|
|
121
119
|
expect(status).toEqual({
|
|
122
|
-
schemaVersion:
|
|
120
|
+
schemaVersion: 2,
|
|
123
121
|
source: "testkit",
|
|
124
122
|
notice: "Generated file. Do not edit manually.",
|
|
125
123
|
product: {
|
|
@@ -131,10 +129,9 @@ describe("runner reporting", () => {
|
|
|
131
129
|
},
|
|
132
130
|
testkitVersion: "0.1.20",
|
|
133
131
|
scope: {
|
|
134
|
-
|
|
135
|
-
|
|
132
|
+
types: ["int"],
|
|
133
|
+
suiteSelectors: ["health"],
|
|
136
134
|
fileNames: ["tests/api/integration/b.int.testkit.ts"],
|
|
137
|
-
framework: "default",
|
|
138
135
|
shard: null,
|
|
139
136
|
serviceFilter: "api",
|
|
140
137
|
isFullRun: false,
|
|
@@ -155,13 +152,13 @@ describe("runner reporting", () => {
|
|
|
155
152
|
tests: [
|
|
156
153
|
{
|
|
157
154
|
service: "api",
|
|
158
|
-
type: "
|
|
155
|
+
type: "int",
|
|
159
156
|
path: "tests/api/integration/a.int.testkit.ts",
|
|
160
157
|
status: "passed",
|
|
161
158
|
},
|
|
162
159
|
{
|
|
163
160
|
service: "api",
|
|
164
|
-
type: "
|
|
161
|
+
type: "int",
|
|
165
162
|
path: "tests/api/integration/b.int.testkit.ts",
|
|
166
163
|
status: "failed",
|
|
167
164
|
},
|
|
@@ -173,10 +170,9 @@ describe("runner reporting", () => {
|
|
|
173
170
|
const status = buildStatusArtifact({
|
|
174
171
|
productDir: "/tmp/my-product",
|
|
175
172
|
results: [],
|
|
176
|
-
|
|
177
|
-
|
|
173
|
+
typeValues: ["all"],
|
|
174
|
+
suiteSelectors: [],
|
|
178
175
|
fileNames: [],
|
|
179
|
-
framework: "all",
|
|
180
176
|
shard: null,
|
|
181
177
|
serviceFilter: null,
|
|
182
178
|
metadata: {
|
package/lib/runner/results.mjs
CHANGED
|
@@ -26,6 +26,7 @@ export function buildServiceTrackers(servicePlans, startedAt) {
|
|
|
26
26
|
key: `${suite.type}:${suite.name}`,
|
|
27
27
|
name: suite.name,
|
|
28
28
|
type: suite.type,
|
|
29
|
+
displayType: suite.displayType || suite.type,
|
|
29
30
|
framework: suite.framework,
|
|
30
31
|
orderIndex: suite.orderIndex,
|
|
31
32
|
fileCount: suite.files.length,
|
|
@@ -215,7 +216,7 @@ function finalizeSuite(suite) {
|
|
|
215
216
|
|
|
216
217
|
return {
|
|
217
218
|
name: suite.name,
|
|
218
|
-
type: suite.
|
|
219
|
+
type: suite.displayType,
|
|
219
220
|
framework: formatFrameworkForArtifact(suite.framework),
|
|
220
221
|
failed: suite.failedFiles.length > 0,
|
|
221
222
|
fileCount: suite.fileCount,
|
package/lib/runner/selection.mjs
CHANGED
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
export function findUnmatchedRequestedFiles(
|
|
2
2
|
configs,
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
framework,
|
|
3
|
+
typeValues,
|
|
4
|
+
suiteSelectors,
|
|
6
5
|
fileNames,
|
|
7
6
|
collectSuites,
|
|
8
7
|
normalizePathSeparators
|
|
9
8
|
) {
|
|
10
9
|
const matchedFiles = new Set();
|
|
11
10
|
for (const config of configs) {
|
|
12
|
-
const suites = collectSuites(config,
|
|
11
|
+
const suites = collectSuites(config, typeValues, suiteSelectors, []);
|
|
13
12
|
for (const suite of suites) {
|
|
14
13
|
for (const file of suite.files) {
|
|
15
14
|
matchedFiles.add(normalizePathSeparators(file));
|
|
@@ -22,11 +21,12 @@ export function findUnmatchedRequestedFiles(
|
|
|
22
21
|
);
|
|
23
22
|
}
|
|
24
23
|
|
|
25
|
-
export function isFullRunSelection(
|
|
24
|
+
export function isFullRunSelection(typeValues, suiteSelectors, fileNames, shard, serviceFilter) {
|
|
26
25
|
return (
|
|
27
|
-
(
|
|
26
|
+
(typeValues || []).length === 1 &&
|
|
27
|
+
typeValues[0] === "all" &&
|
|
28
|
+
(suiteSelectors || []).length === 0 &&
|
|
28
29
|
(fileNames || []).length === 0 &&
|
|
29
|
-
(framework || "all") === "all" &&
|
|
30
30
|
(shard || null) === null &&
|
|
31
31
|
(serviceFilter || null) === null
|
|
32
32
|
);
|
|
@@ -5,9 +5,8 @@ describe("runner selection", () => {
|
|
|
5
5
|
it("finds unmatched requested files", () => {
|
|
6
6
|
const unmatched = findUnmatchedRequestedFiles(
|
|
7
7
|
[{ name: "api" }],
|
|
8
|
-
"int",
|
|
8
|
+
["int"],
|
|
9
9
|
[],
|
|
10
|
-
"all",
|
|
11
10
|
["tests/a.int.testkit.ts", "tests/missing.int.testkit.ts"],
|
|
12
11
|
() => [{ files: ["tests/a.int.testkit.ts"] }],
|
|
13
12
|
(value) => value
|
|
@@ -17,9 +16,9 @@ describe("runner selection", () => {
|
|
|
17
16
|
});
|
|
18
17
|
|
|
19
18
|
it("detects a full run selection", () => {
|
|
20
|
-
expect(isFullRunSelection([], [],
|
|
21
|
-
expect(isFullRunSelection(["
|
|
22
|
-
expect(isFullRunSelection([], [
|
|
23
|
-
expect(isFullRunSelection([], [],
|
|
19
|
+
expect(isFullRunSelection(["all"], [], [], null, null)).toBe(true);
|
|
20
|
+
expect(isFullRunSelection(["all"], [{ raw: "auth" }], [], null, null)).toBe(false);
|
|
21
|
+
expect(isFullRunSelection(["all"], [], ["a"], null, null)).toBe(false);
|
|
22
|
+
expect(isFullRunSelection(["int"], [], [], null, null)).toBe(false);
|
|
24
23
|
});
|
|
25
24
|
});
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
const USER_TYPES = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
|
|
2
|
+
|
|
3
|
+
export function normalizeTypeValues(values = []) {
|
|
4
|
+
const expanded = [];
|
|
5
|
+
for (const rawValue of values) {
|
|
6
|
+
if (rawValue == null) continue;
|
|
7
|
+
for (const part of String(rawValue).split(",")) {
|
|
8
|
+
const value = part.trim();
|
|
9
|
+
if (!value) continue;
|
|
10
|
+
if (!USER_TYPES.has(value)) {
|
|
11
|
+
throw new Error(
|
|
12
|
+
`Unknown type "${value}". Expected one of: int, e2e, dal, load, pw, all.`
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
expanded.push(value);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
if (expanded.length === 0) {
|
|
20
|
+
return ["all"];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const deduped = [...new Set(expanded)];
|
|
24
|
+
if (deduped.includes("all") && deduped.length > 1) {
|
|
25
|
+
throw new Error(`"--type all" cannot be combined with other types.`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const order = ["int", "e2e", "dal", "load", "pw", "all"];
|
|
29
|
+
return deduped.sort((left, right) => order.indexOf(left) - order.indexOf(right));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function isAllTypeSelection(typeValues = []) {
|
|
33
|
+
return typeValues.length === 0 || (typeValues.length === 1 && typeValues[0] === "all");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function parseSuiteSelectors(values = []) {
|
|
37
|
+
const selectors = [];
|
|
38
|
+
|
|
39
|
+
for (const rawValue of values) {
|
|
40
|
+
if (rawValue == null) continue;
|
|
41
|
+
for (const part of String(rawValue).split(",")) {
|
|
42
|
+
const value = part.trim();
|
|
43
|
+
if (!value) continue;
|
|
44
|
+
|
|
45
|
+
const typeMatch = value.match(/^([a-z]+):(.*)$/);
|
|
46
|
+
if (!typeMatch) {
|
|
47
|
+
selectors.push({ kind: "plain", name: value, raw: value });
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const type = typeMatch[1];
|
|
52
|
+
const name = typeMatch[2].trim();
|
|
53
|
+
if (!USER_TYPES.has(type) || type === "all") {
|
|
54
|
+
throw new Error(
|
|
55
|
+
`Unknown suite selector type "${type}". Expected one of: int, e2e, dal, load, pw.`
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
if (!name) {
|
|
59
|
+
throw new Error(`Invalid suite selector "${value}". Expected "<type>:<suite-name>".`);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
selectors.push({
|
|
63
|
+
kind: "typed",
|
|
64
|
+
type,
|
|
65
|
+
name,
|
|
66
|
+
raw: `${type}:${name}`,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return selectors;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function suiteSelectionType(type, framework) {
|
|
75
|
+
if ((framework || "k6") === "playwright") return "pw";
|
|
76
|
+
if (type === "integration") return "int";
|
|
77
|
+
return type;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function matchesSelectedTypes(selectionType, typeValues) {
|
|
81
|
+
return isAllTypeSelection(typeValues) || typeValues.includes(selectionType);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function matchesSuiteSelectors(selectionType, suiteName, selectors = []) {
|
|
85
|
+
if (!selectors || selectors.length === 0) return true;
|
|
86
|
+
|
|
87
|
+
return selectors.some((selector) => {
|
|
88
|
+
if (selector.kind === "plain") return selector.name === suiteName;
|
|
89
|
+
return selector.type === selectionType && selector.name === suiteName;
|
|
90
|
+
});
|
|
91
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
isAllTypeSelection,
|
|
4
|
+
matchesSelectedTypes,
|
|
5
|
+
matchesSuiteSelectors,
|
|
6
|
+
normalizeTypeValues,
|
|
7
|
+
parseSuiteSelectors,
|
|
8
|
+
suiteSelectionType,
|
|
9
|
+
} from "./suite-selection.mjs";
|
|
10
|
+
|
|
11
|
+
describe("runner suite selection", () => {
|
|
12
|
+
it("normalizes selected type values", () => {
|
|
13
|
+
expect(normalizeTypeValues([])).toEqual(["all"]);
|
|
14
|
+
expect(normalizeTypeValues(["int,e2e", "dal"])).toEqual(["int", "e2e", "dal"]);
|
|
15
|
+
expect(() => normalizeTypeValues(["all", "int"])).toThrow("cannot be combined");
|
|
16
|
+
expect(() => normalizeTypeValues(["jest"])).toThrow("Unknown type");
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it("parses suite selectors", () => {
|
|
20
|
+
expect(parseSuiteSelectors(["auth,dal:queries"])).toEqual([
|
|
21
|
+
{ kind: "plain", name: "auth", raw: "auth" },
|
|
22
|
+
{ kind: "typed", type: "dal", name: "queries", raw: "dal:queries" },
|
|
23
|
+
]);
|
|
24
|
+
expect(() => parseSuiteSelectors(["all:auth"])).toThrow("Unknown suite selector type");
|
|
25
|
+
expect(() => parseSuiteSelectors(["int:"])).toThrow("Invalid suite selector");
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it("matches suite types and selectors", () => {
|
|
29
|
+
expect(isAllTypeSelection(["all"])).toBe(true);
|
|
30
|
+
expect(matchesSelectedTypes("int", ["int", "dal"])).toBe(true);
|
|
31
|
+
expect(matchesSelectedTypes("pw", ["int", "dal"])).toBe(false);
|
|
32
|
+
expect(matchesSuiteSelectors("int", "auth", parseSuiteSelectors(["auth"]))).toBe(true);
|
|
33
|
+
expect(matchesSuiteSelectors("e2e", "auth", parseSuiteSelectors(["int:auth"]))).toBe(false);
|
|
34
|
+
expect(matchesSuiteSelectors("int", "auth", parseSuiteSelectors(["int:auth"]))).toBe(true);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it("maps discovered suites to user-facing selection types", () => {
|
|
38
|
+
expect(suiteSelectionType("integration", "k6")).toBe("int");
|
|
39
|
+
expect(suiteSelectionType("e2e", "playwright")).toBe("pw");
|
|
40
|
+
expect(suiteSelectionType("dal", "k6")).toBe("dal");
|
|
41
|
+
});
|
|
42
|
+
});
|