as-test 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -3
- package/README.md +14 -7
- package/as-test.config.schema.json +142 -142
- package/assembly/__fuzz__/math.fuzz.ts +17 -14
- package/assembly/__fuzz__/seed-perf.fuzz.ts +4 -2
- package/assembly/coverage.ts +12 -0
- package/assembly/index.ts +17 -4
- package/assembly/src/expectation.ts +7 -1
- package/assembly/src/fuzz.ts +44 -23
- package/assembly/src/suite.ts +2 -0
- package/assembly/test.ts +85 -0
- package/assembly/util/format.ts +10 -1
- package/assembly/util/wipc.ts +1 -2
- package/bin/build-worker-pool.js +7 -1
- package/bin/commands/build-core.js +6 -1
- package/bin/commands/build.js +1 -1
- package/bin/commands/clean-core.js +3 -1
- package/bin/commands/clean.js +0 -37
- package/bin/commands/fuzz-core.js +2 -2
- package/bin/commands/run-core.js +45 -24
- package/bin/commands/run.js +3 -1
- package/bin/commands/test.js +3 -1
- package/bin/commands/web-runner-source.js +14 -14
- package/bin/commands/web-session.js +6 -1
- package/bin/coverage-points.js +207 -2
- package/bin/crash-store.js +3 -1
- package/bin/index.js +357 -125
- package/bin/reporters/default.js +295 -64
- package/bin/util.js +36 -11
- package/lib/build/index.js +74 -24
- package/lib/src/index.ts +96 -35
- package/package.json +2 -1
- package/transform/lib/coverage.js +222 -71
package/bin/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import chalk from "chalk";
|
|
3
|
-
import { build, formatInvocation as formatBuildInvocation, getBuildInvocationPreview, } from "./commands/build.js";
|
|
3
|
+
import { build, BuildFailureError, formatInvocation as formatBuildInvocation, getBuildInvocationPreview, } from "./commands/build.js";
|
|
4
4
|
import { createRunReporter, run } from "./commands/run.js";
|
|
5
5
|
import { executeBuildCommand } from "./commands/build.js";
|
|
6
6
|
import { executeRunCommand } from "./commands/run.js";
|
|
@@ -10,7 +10,7 @@ import { executeInitCommand } from "./commands/init.js";
|
|
|
10
10
|
import { executeDoctorCommand } from "./commands/doctor.js";
|
|
11
11
|
import { executeCleanCommand } from "./commands/clean.js";
|
|
12
12
|
import { fuzz } from "./commands/fuzz-core.js";
|
|
13
|
-
import { applyMode,
|
|
13
|
+
import { applyMode, formatTime, formatSpecDisplayPath, getDefaultModeNames, getCliVersion, loadConfig, resolveModeNames, } from "./util.js";
|
|
14
14
|
import * as path from "path";
|
|
15
15
|
import { spawnSync } from "child_process";
|
|
16
16
|
import { glob } from "glob";
|
|
@@ -78,6 +78,7 @@ else if (COMMANDS.includes(args[0])) {
|
|
|
78
78
|
resolveParallelJobs,
|
|
79
79
|
resolveBrowserOverride,
|
|
80
80
|
resolveReporterOverride,
|
|
81
|
+
resolveShowCoverageMode,
|
|
81
82
|
resolveExecutionModes,
|
|
82
83
|
listExecutionPlan,
|
|
83
84
|
runRuntimeModes,
|
|
@@ -96,6 +97,7 @@ else if (COMMANDS.includes(args[0])) {
|
|
|
96
97
|
resolveParallelJobs,
|
|
97
98
|
resolveBrowserOverride,
|
|
98
99
|
resolveReporterOverride,
|
|
100
|
+
resolveShowCoverageMode,
|
|
99
101
|
resolveFuzzOverrides,
|
|
100
102
|
resolveExecutionModes,
|
|
101
103
|
listExecutionPlan,
|
|
@@ -277,7 +279,7 @@ function printCommandHelp(command) {
|
|
|
277
279
|
process.stdout.write(" --create-snapshots Create missing snapshot entries\n");
|
|
278
280
|
process.stdout.write(" --overwrite-snapshots Overwrite existing snapshot entries on mismatch\n");
|
|
279
281
|
process.stdout.write(" --no-snapshot Disable snapshot assertions for this run\n");
|
|
280
|
-
process.stdout.write(" --show-coverage
|
|
282
|
+
process.stdout.write(" --show-coverage[=all] Print uncovered coverage point details; use =all to expand nested gaps\n");
|
|
281
283
|
process.stdout.write(" --suite <name[,name...]> Filter results to matching suite names or suite slug paths\n");
|
|
282
284
|
process.stdout.write(" --suites <name[,name...]> Alias for --suite\n");
|
|
283
285
|
process.stdout.write(" --enable <feature> Enable feature (coverage|try-as)\n");
|
|
@@ -305,7 +307,7 @@ function printCommandHelp(command) {
|
|
|
305
307
|
process.stdout.write(" --create-snapshots Create missing snapshot entries\n");
|
|
306
308
|
process.stdout.write(" --overwrite-snapshots Overwrite existing snapshot entries on mismatch\n");
|
|
307
309
|
process.stdout.write(" --no-snapshot Disable snapshot assertions for this run\n");
|
|
308
|
-
process.stdout.write(" --show-coverage
|
|
310
|
+
process.stdout.write(" --show-coverage[=all] Print uncovered coverage point details; use =all to expand nested gaps\n");
|
|
309
311
|
process.stdout.write(" --suite <name[,name...]> Filter results to matching suite names or suite slug paths\n");
|
|
310
312
|
process.stdout.write(" --suites <name[,name...]> Alias for --suite\n");
|
|
311
313
|
process.stdout.write(" --enable <feature> Enable feature (coverage|try-as)\n");
|
|
@@ -374,7 +376,6 @@ function printCommandHelp(command) {
|
|
|
374
376
|
process.stdout.write(chalk.bold("Flags:\n"));
|
|
375
377
|
process.stdout.write(" --config <path> Use a specific config file\n");
|
|
376
378
|
process.stdout.write(" --mode <name[,name...]> Clean one or multiple named modes\n");
|
|
377
|
-
process.stdout.write(" -f, --force Skip the full-clean confirmation prompt\n");
|
|
378
379
|
process.stdout.write(" --help, -h Show this help\n");
|
|
379
380
|
return;
|
|
380
381
|
}
|
|
@@ -463,8 +464,15 @@ function resolveCommandArgs(rawArgs, command) {
|
|
|
463
464
|
arg == "--build-jobs" ||
|
|
464
465
|
arg == "--run-jobs" ||
|
|
465
466
|
arg == "--browser" ||
|
|
467
|
+
arg == "--show-coverage" ||
|
|
466
468
|
arg == "--fuzz-runs" ||
|
|
467
469
|
arg == "--fuzz-seed") {
|
|
470
|
+
if (arg == "--show-coverage") {
|
|
471
|
+
const next = rawArgs[i + 1];
|
|
472
|
+
if (next == "all")
|
|
473
|
+
i++;
|
|
474
|
+
continue;
|
|
475
|
+
}
|
|
468
476
|
i++;
|
|
469
477
|
continue;
|
|
470
478
|
}
|
|
@@ -477,6 +485,7 @@ function resolveCommandArgs(rawArgs, command) {
|
|
|
477
485
|
arg.startsWith("--build-jobs=") ||
|
|
478
486
|
arg.startsWith("--run-jobs=") ||
|
|
479
487
|
arg.startsWith("--browser=") ||
|
|
488
|
+
arg.startsWith("--show-coverage=") ||
|
|
480
489
|
arg.startsWith("--fuzz-runs=") ||
|
|
481
490
|
arg.startsWith("--fuzz-seed=")) {
|
|
482
491
|
continue;
|
|
@@ -763,6 +772,34 @@ function resolveReporterOverride(rawArgs, command) {
|
|
|
763
772
|
}
|
|
764
773
|
return undefined;
|
|
765
774
|
}
|
|
775
|
+
function resolveShowCoverageMode(rawArgs, command) {
|
|
776
|
+
let seenCommand = false;
|
|
777
|
+
for (let i = 0; i < rawArgs.length; i++) {
|
|
778
|
+
const arg = rawArgs[i];
|
|
779
|
+
if (!seenCommand) {
|
|
780
|
+
if (arg == command)
|
|
781
|
+
seenCommand = true;
|
|
782
|
+
continue;
|
|
783
|
+
}
|
|
784
|
+
if (arg == "--show-coverage") {
|
|
785
|
+
const next = rawArgs[i + 1];
|
|
786
|
+
if (next == "all")
|
|
787
|
+
return "all";
|
|
788
|
+
return "collapsed";
|
|
789
|
+
}
|
|
790
|
+
if (arg.startsWith("--show-coverage=")) {
|
|
791
|
+
const value = arg.slice("--show-coverage=".length).trim();
|
|
792
|
+
if (!value.length) {
|
|
793
|
+
throw new Error("--show-coverage requires a value when using =");
|
|
794
|
+
}
|
|
795
|
+
if (value != "all") {
|
|
796
|
+
throw new Error(`--show-coverage only supports "all" when given a value`);
|
|
797
|
+
}
|
|
798
|
+
return "all";
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
return undefined;
|
|
802
|
+
}
|
|
766
803
|
function resolveJobs(rawArgs, command) {
|
|
767
804
|
let seenCommand = false;
|
|
768
805
|
let parallel = false;
|
|
@@ -1070,10 +1107,12 @@ function resolveCommandTokens(rawArgs, command) {
|
|
|
1070
1107
|
}
|
|
1071
1108
|
async function buildFileForMode(args) {
|
|
1072
1109
|
if (args.buildPool) {
|
|
1110
|
+
const buildInvocation = await getBuildInvocationPreview(args.configPath, args.file, args.modeName, args.buildFeatureToggles);
|
|
1073
1111
|
await args.buildPool.buildFileMode({
|
|
1074
1112
|
configPath: args.configPath,
|
|
1075
1113
|
file: args.file,
|
|
1076
1114
|
modeName: args.modeName,
|
|
1115
|
+
buildCommand: formatBuildInvocation(buildInvocation),
|
|
1077
1116
|
featureToggles: args.buildFeatureToggles,
|
|
1078
1117
|
});
|
|
1079
1118
|
}
|
|
@@ -1104,21 +1143,30 @@ async function runTestSequential(runFlags, configPath, selectors, suiteSelectors
|
|
|
1104
1143
|
const duplicateSpecBasenames = resolveDuplicateSpecBasenames(files);
|
|
1105
1144
|
for (const file of files) {
|
|
1106
1145
|
const buildStartedAt = Date.now();
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1146
|
+
let result;
|
|
1147
|
+
try {
|
|
1148
|
+
await build(configPath, [file], modeName, buildFeatureToggles);
|
|
1149
|
+
buildIntervals.push({ start: buildStartedAt, end: Date.now() });
|
|
1150
|
+
const buildInvocation = await getBuildInvocationPreview(configPath, file, modeName, buildFeatureToggles);
|
|
1151
|
+
const artifactKey = resolvePerFileArtifactKey(file, duplicateSpecBasenames);
|
|
1152
|
+
result = await run(runFlags, configPath, [file], false, {
|
|
1153
|
+
reporter,
|
|
1154
|
+
webSession,
|
|
1155
|
+
suiteSelectors,
|
|
1156
|
+
emitRunStart: false,
|
|
1157
|
+
emitRunComplete: false,
|
|
1158
|
+
logFileName: `test.${artifactKey}.log.json`,
|
|
1159
|
+
coverageFileName: `coverage.${artifactKey}.log.json`,
|
|
1160
|
+
buildCommand: formatBuildInvocation(buildInvocation),
|
|
1161
|
+
modeName,
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
catch (error) {
|
|
1165
|
+
const buildFailure = getBuildFailureErrorLike(error);
|
|
1166
|
+
if (!buildFailure)
|
|
1167
|
+
throw error;
|
|
1168
|
+
result = createBuildFailureRunResult(buildFailure);
|
|
1169
|
+
}
|
|
1122
1170
|
results.push(result);
|
|
1123
1171
|
if (result?.failed)
|
|
1124
1172
|
failed = true;
|
|
@@ -1130,6 +1178,8 @@ async function runTestSequential(runFlags, configPath, selectors, suiteSelectors
|
|
|
1130
1178
|
clean: runFlags.clean,
|
|
1131
1179
|
snapshotEnabled,
|
|
1132
1180
|
showCoverage: runFlags.showCoverage,
|
|
1181
|
+
showCoverageAll: runFlags.showCoverageAll,
|
|
1182
|
+
verbose: runFlags.verbose,
|
|
1133
1183
|
buildTime: getMergedIntervalDuration(buildIntervals),
|
|
1134
1184
|
snapshotSummary: summary.snapshotSummary,
|
|
1135
1185
|
coverageSummary: summary.coverageSummary,
|
|
@@ -1305,7 +1355,7 @@ async function runRuntimeMatrix(runFlags, configPath, selectors, suiteSelectors,
|
|
|
1305
1355
|
const buildIntervals = [];
|
|
1306
1356
|
for (let fileIndex = 0; fileIndex < files.length; fileIndex++) {
|
|
1307
1357
|
const file = files[fileIndex];
|
|
1308
|
-
const fileName =
|
|
1358
|
+
const fileName = formatSpecDisplayPath(file);
|
|
1309
1359
|
const fileResults = [];
|
|
1310
1360
|
const modeTimes = modes.map(() => "...");
|
|
1311
1361
|
if (liveMatrix) {
|
|
@@ -1362,6 +1412,8 @@ async function runRuntimeMatrix(runFlags, configPath, selectors, suiteSelectors,
|
|
|
1362
1412
|
clean: runFlags.clean,
|
|
1363
1413
|
snapshotEnabled,
|
|
1364
1414
|
showCoverage: runFlags.showCoverage,
|
|
1415
|
+
showCoverageAll: runFlags.showCoverageAll,
|
|
1416
|
+
verbose: runFlags.verbose,
|
|
1365
1417
|
buildTime: 0,
|
|
1366
1418
|
snapshotSummary: summary.snapshotSummary,
|
|
1367
1419
|
coverageSummary: summary.coverageSummary,
|
|
@@ -1425,6 +1477,8 @@ async function runTestModes(runFlags, configPath, selectors, suiteSelectors, fuz
|
|
|
1425
1477
|
clean: runFlags.clean,
|
|
1426
1478
|
snapshotEnabled: effectiveRunFlags.snapshot !== false,
|
|
1427
1479
|
showCoverage: effectiveRunFlags.showCoverage,
|
|
1480
|
+
showCoverageAll: effectiveRunFlags.showCoverageAll,
|
|
1481
|
+
verbose: effectiveRunFlags.verbose,
|
|
1428
1482
|
buildTime: modeResult.summary.buildTime +
|
|
1429
1483
|
getMergedIntervalDuration(collectFuzzBuildIntervals(fuzzResults)),
|
|
1430
1484
|
snapshotSummary: modeResult.summary.snapshotSummary,
|
|
@@ -1481,7 +1535,7 @@ async function runTestMatrix(runFlags, configPath, selectors, suiteSelectors, fu
|
|
|
1481
1535
|
const buildIntervals = [];
|
|
1482
1536
|
for (let fileIndex = 0; fileIndex < files.length; fileIndex++) {
|
|
1483
1537
|
const file = files[fileIndex];
|
|
1484
|
-
const fileName =
|
|
1538
|
+
const fileName = formatSpecDisplayPath(file);
|
|
1485
1539
|
const fileResults = [];
|
|
1486
1540
|
const modeTimes = modes.map(() => "...");
|
|
1487
1541
|
if (liveMatrix) {
|
|
@@ -1489,6 +1543,7 @@ async function runTestMatrix(runFlags, configPath, selectors, suiteSelectors, fu
|
|
|
1489
1543
|
}
|
|
1490
1544
|
for (let i = 0; i < modes.length; i++) {
|
|
1491
1545
|
const modeName = modes[i];
|
|
1546
|
+
let result;
|
|
1492
1547
|
try {
|
|
1493
1548
|
const buildStartedAt = Date.now();
|
|
1494
1549
|
await buildFileForMode({
|
|
@@ -1500,7 +1555,7 @@ async function runTestMatrix(runFlags, configPath, selectors, suiteSelectors, fu
|
|
|
1500
1555
|
buildIntervals.push({ start: buildStartedAt, end: Date.now() });
|
|
1501
1556
|
const buildInvocation = await getBuildInvocationPreview(configPath, file, modeName, buildFeatureToggles);
|
|
1502
1557
|
const artifactKey = resolvePerFileArtifactKey(file, duplicateSpecBasenames);
|
|
1503
|
-
|
|
1558
|
+
result = await run(runFlags, configPath, [file], false, {
|
|
1504
1559
|
reporter: silentReporter,
|
|
1505
1560
|
reporterKind: "default",
|
|
1506
1561
|
emitRunStart: false,
|
|
@@ -1510,23 +1565,27 @@ async function runTestMatrix(runFlags, configPath, selectors, suiteSelectors, fu
|
|
|
1510
1565
|
buildCommand: formatBuildInvocation(buildInvocation),
|
|
1511
1566
|
modeName,
|
|
1512
1567
|
});
|
|
1513
|
-
modeTimes[i] = formatMatrixModeTime(result.stats.time);
|
|
1514
|
-
if (liveMatrix) {
|
|
1515
|
-
renderMatrixLiveLine(fileName, modeLabels, modeTimes, showPerModeTimes);
|
|
1516
|
-
}
|
|
1517
|
-
if (result.failed) {
|
|
1518
|
-
modeState[i].failed = true;
|
|
1519
|
-
}
|
|
1520
|
-
else if (result.stats.passedFiles > 0) {
|
|
1521
|
-
modeState[i].passed = true;
|
|
1522
|
-
}
|
|
1523
|
-
fileResults.push(result);
|
|
1524
|
-
allResults.push(result);
|
|
1525
1568
|
}
|
|
1526
1569
|
catch (error) {
|
|
1527
|
-
|
|
1528
|
-
|
|
1570
|
+
const buildFailure = getBuildFailureErrorLike(error);
|
|
1571
|
+
if (!buildFailure) {
|
|
1572
|
+
clearLiveLine();
|
|
1573
|
+
throw error;
|
|
1574
|
+
}
|
|
1575
|
+
result = createBuildFailureRunResult(buildFailure);
|
|
1529
1576
|
}
|
|
1577
|
+
modeTimes[i] = formatMatrixModeTime(result.stats.time);
|
|
1578
|
+
if (liveMatrix) {
|
|
1579
|
+
renderMatrixLiveLine(fileName, modeLabels, modeTimes, showPerModeTimes);
|
|
1580
|
+
}
|
|
1581
|
+
if (result.failed) {
|
|
1582
|
+
modeState[i].failed = true;
|
|
1583
|
+
}
|
|
1584
|
+
else if (result.stats.passedFiles > 0) {
|
|
1585
|
+
modeState[i].passed = true;
|
|
1586
|
+
}
|
|
1587
|
+
fileResults.push(result);
|
|
1588
|
+
allResults.push(result);
|
|
1530
1589
|
}
|
|
1531
1590
|
if (reporterSession.reporterKind == "default") {
|
|
1532
1591
|
renderMatrixFileResult(fileName, modeLabels, fileResults, modeTimes, liveMatrix, showPerModeTimes);
|
|
@@ -1557,6 +1616,8 @@ async function runTestMatrix(runFlags, configPath, selectors, suiteSelectors, fu
|
|
|
1557
1616
|
clean: runFlags.clean,
|
|
1558
1617
|
snapshotEnabled,
|
|
1559
1618
|
showCoverage: runFlags.showCoverage,
|
|
1619
|
+
showCoverageAll: runFlags.showCoverageAll,
|
|
1620
|
+
verbose: runFlags.verbose,
|
|
1560
1621
|
buildTime: getMergedIntervalDuration(buildIntervals),
|
|
1561
1622
|
snapshotSummary: summary.snapshotSummary,
|
|
1562
1623
|
coverageSummary: summary.coverageSummary,
|
|
@@ -1611,22 +1672,31 @@ async function runRuntimeSingleParallel(runFlags, configPath, selectors, suiteSe
|
|
|
1611
1672
|
const poolWidth = Math.max(runFlags.buildJobs, runFlags.runJobs);
|
|
1612
1673
|
await runOrderedPool(files, poolWidth, async (file, index) => {
|
|
1613
1674
|
const token = useQueueDisplay
|
|
1614
|
-
? renderQueuedFileStart(queueDisplay,
|
|
1675
|
+
? renderQueuedFileStart(queueDisplay, formatSpecDisplayPath(file))
|
|
1615
1676
|
: null;
|
|
1616
1677
|
const buffered = useQueueDisplay
|
|
1617
1678
|
? await createBufferedReporter(configPath, runFlags.reporterPath, modeName)
|
|
1618
1679
|
: null;
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1680
|
+
let result;
|
|
1681
|
+
try {
|
|
1682
|
+
result = await runLimit(() => run({ ...runFlags, clean: true }, configPath, [file], false, {
|
|
1683
|
+
reporter: buffered?.reporter,
|
|
1684
|
+
reporterKind: buffered?.reporterKind,
|
|
1685
|
+
modeName,
|
|
1686
|
+
suiteSelectors,
|
|
1687
|
+
emitRunComplete: false,
|
|
1688
|
+
fileSummaryTotal: 1,
|
|
1689
|
+
modeSummaryTotal,
|
|
1690
|
+
modeSummaryExecuted: 1,
|
|
1691
|
+
buildCommandsByFile: { [file]: buildCommandsByFile[file] ?? "" },
|
|
1692
|
+
}));
|
|
1693
|
+
}
|
|
1694
|
+
catch (error) {
|
|
1695
|
+
const buildFailure = getBuildFailureErrorLike(error);
|
|
1696
|
+
if (!buildFailure)
|
|
1697
|
+
throw error;
|
|
1698
|
+
result = createBuildFailureRunResult(buildFailure);
|
|
1699
|
+
}
|
|
1630
1700
|
buffered?.reporter.flush?.();
|
|
1631
1701
|
results[index] = result;
|
|
1632
1702
|
if (buffered && token != null) {
|
|
@@ -1640,6 +1710,8 @@ async function runRuntimeSingleParallel(runFlags, configPath, selectors, suiteSe
|
|
|
1640
1710
|
clean: runFlags.clean,
|
|
1641
1711
|
snapshotEnabled,
|
|
1642
1712
|
showCoverage: runFlags.showCoverage,
|
|
1713
|
+
showCoverageAll: runFlags.showCoverageAll,
|
|
1714
|
+
verbose: runFlags.verbose,
|
|
1643
1715
|
buildTime: 0,
|
|
1644
1716
|
snapshotSummary: summary.snapshotSummary,
|
|
1645
1717
|
coverageSummary: summary.coverageSummary,
|
|
@@ -1677,7 +1749,7 @@ async function runRuntimeMatrixParallel(runFlags, configPath, selectors, suiteSe
|
|
|
1677
1749
|
const buildIntervals = [];
|
|
1678
1750
|
try {
|
|
1679
1751
|
await runOrderedPool(files, poolWidth, async (file, fileIndex) => {
|
|
1680
|
-
const fileName =
|
|
1752
|
+
const fileName = formatSpecDisplayPath(file);
|
|
1681
1753
|
const token = useQueueDisplay
|
|
1682
1754
|
? renderQueuedFileStart(queueDisplay, fileName)
|
|
1683
1755
|
: null;
|
|
@@ -1685,28 +1757,37 @@ async function runRuntimeMatrixParallel(runFlags, configPath, selectors, suiteSe
|
|
|
1685
1757
|
const modeTimes = modes.map(() => "...");
|
|
1686
1758
|
for (let i = 0; i < modes.length; i++) {
|
|
1687
1759
|
const modeName = modes[i];
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1760
|
+
let result;
|
|
1761
|
+
try {
|
|
1762
|
+
const buildStartedAt = Date.now();
|
|
1763
|
+
await buildFileForMode({
|
|
1764
|
+
configPath,
|
|
1765
|
+
file,
|
|
1766
|
+
modeName,
|
|
1767
|
+
buildFeatureToggles: {},
|
|
1768
|
+
buildPool,
|
|
1769
|
+
});
|
|
1770
|
+
buildIntervals.push({ start: buildStartedAt, end: Date.now() });
|
|
1771
|
+
const buildInvocation = await getBuildInvocationPreview(configPath, file, modeName, {});
|
|
1772
|
+
const artifactKey = resolvePerFileArtifactKey(file, duplicateSpecBasenames);
|
|
1773
|
+
result = await run(runFlags, configPath, [file], false, {
|
|
1774
|
+
reporter: silentReporter,
|
|
1775
|
+
reporterKind: "default",
|
|
1776
|
+
suiteSelectors,
|
|
1777
|
+
emitRunStart: false,
|
|
1778
|
+
emitRunComplete: false,
|
|
1779
|
+
logFileName: `run.${artifactKey}.log.json`,
|
|
1780
|
+
coverageFileName: `coverage.${artifactKey}.log.json`,
|
|
1781
|
+
buildCommand: formatBuildInvocation(buildInvocation),
|
|
1782
|
+
modeName,
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
catch (error) {
|
|
1786
|
+
const buildFailure = getBuildFailureErrorLike(error);
|
|
1787
|
+
if (!buildFailure)
|
|
1788
|
+
throw error;
|
|
1789
|
+
result = createBuildFailureRunResult(buildFailure);
|
|
1790
|
+
}
|
|
1710
1791
|
modeTimes[i] = formatMatrixModeTime(result.stats.time);
|
|
1711
1792
|
fileResults.push(result);
|
|
1712
1793
|
}
|
|
@@ -1745,6 +1826,8 @@ async function runRuntimeMatrixParallel(runFlags, configPath, selectors, suiteSe
|
|
|
1745
1826
|
clean: runFlags.clean,
|
|
1746
1827
|
snapshotEnabled,
|
|
1747
1828
|
showCoverage: runFlags.showCoverage,
|
|
1829
|
+
showCoverageAll: runFlags.showCoverageAll,
|
|
1830
|
+
verbose: runFlags.verbose,
|
|
1748
1831
|
buildTime: getMergedIntervalDuration(buildIntervals),
|
|
1749
1832
|
snapshotSummary: summary.snapshotSummary,
|
|
1750
1833
|
coverageSummary: summary.coverageSummary,
|
|
@@ -1781,32 +1864,41 @@ async function runTestSingleParallel(runFlags, configPath, selectors, suiteSelec
|
|
|
1781
1864
|
try {
|
|
1782
1865
|
await runOrderedPool(files, poolWidth, async (file, index) => {
|
|
1783
1866
|
const token = useQueueDisplay
|
|
1784
|
-
? renderQueuedFileStart(queueDisplay,
|
|
1867
|
+
? renderQueuedFileStart(queueDisplay, formatSpecDisplayPath(file))
|
|
1785
1868
|
: null;
|
|
1786
|
-
const buildStartedAt = Date.now();
|
|
1787
|
-
await buildFileForMode({
|
|
1788
|
-
configPath,
|
|
1789
|
-
file,
|
|
1790
|
-
modeName,
|
|
1791
|
-
buildFeatureToggles,
|
|
1792
|
-
buildPool,
|
|
1793
|
-
});
|
|
1794
|
-
buildIntervals.push({ start: buildStartedAt, end: Date.now() });
|
|
1795
|
-
const buildInvocation = await getBuildInvocationPreview(configPath, file, modeName, buildFeatureToggles);
|
|
1796
|
-
const artifactKey = resolvePerFileArtifactKey(file, duplicateSpecBasenames);
|
|
1797
1869
|
const buffered = useQueueDisplay
|
|
1798
1870
|
? await createBufferedReporter(configPath, runFlags.reporterPath, modeName)
|
|
1799
1871
|
: null;
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1872
|
+
let result;
|
|
1873
|
+
try {
|
|
1874
|
+
const buildStartedAt = Date.now();
|
|
1875
|
+
await buildFileForMode({
|
|
1876
|
+
configPath,
|
|
1877
|
+
file,
|
|
1878
|
+
modeName,
|
|
1879
|
+
buildFeatureToggles,
|
|
1880
|
+
buildPool,
|
|
1881
|
+
});
|
|
1882
|
+
buildIntervals.push({ start: buildStartedAt, end: Date.now() });
|
|
1883
|
+
const buildInvocation = await getBuildInvocationPreview(configPath, file, modeName, buildFeatureToggles);
|
|
1884
|
+
const artifactKey = resolvePerFileArtifactKey(file, duplicateSpecBasenames);
|
|
1885
|
+
result = await run({ ...runFlags, clean: true }, configPath, [file], false, {
|
|
1886
|
+
reporter: buffered?.reporter,
|
|
1887
|
+
reporterKind: buffered?.reporterKind,
|
|
1888
|
+
suiteSelectors,
|
|
1889
|
+
emitRunComplete: false,
|
|
1890
|
+
logFileName: `test.${artifactKey}.log.json`,
|
|
1891
|
+
coverageFileName: `coverage.${artifactKey}.log.json`,
|
|
1892
|
+
buildCommand: formatBuildInvocation(buildInvocation),
|
|
1893
|
+
modeName,
|
|
1894
|
+
});
|
|
1895
|
+
}
|
|
1896
|
+
catch (error) {
|
|
1897
|
+
const buildFailure = getBuildFailureErrorLike(error);
|
|
1898
|
+
if (!buildFailure)
|
|
1899
|
+
throw error;
|
|
1900
|
+
result = createBuildFailureRunResult(buildFailure);
|
|
1901
|
+
}
|
|
1810
1902
|
buffered?.reporter.flush?.();
|
|
1811
1903
|
results[index] = result;
|
|
1812
1904
|
if (buffered && token != null) {
|
|
@@ -1838,6 +1930,8 @@ async function runTestSingleParallel(runFlags, configPath, selectors, suiteSelec
|
|
|
1838
1930
|
clean: runFlags.clean,
|
|
1839
1931
|
snapshotEnabled,
|
|
1840
1932
|
showCoverage: runFlags.showCoverage,
|
|
1933
|
+
showCoverageAll: runFlags.showCoverageAll,
|
|
1934
|
+
verbose: runFlags.verbose,
|
|
1841
1935
|
buildTime: getMergedIntervalDuration(buildIntervals),
|
|
1842
1936
|
snapshotSummary: summary.snapshotSummary,
|
|
1843
1937
|
coverageSummary: summary.coverageSummary,
|
|
@@ -1882,7 +1976,7 @@ async function runTestMatrixParallel(runFlags, configPath, selectors, suiteSelec
|
|
|
1882
1976
|
const buildIntervals = [];
|
|
1883
1977
|
try {
|
|
1884
1978
|
await runOrderedPool(files, poolWidth, async (file, fileIndex) => {
|
|
1885
|
-
const fileName =
|
|
1979
|
+
const fileName = formatSpecDisplayPath(file);
|
|
1886
1980
|
const token = useQueueDisplay
|
|
1887
1981
|
? renderQueuedFileStart(queueDisplay, fileName)
|
|
1888
1982
|
: null;
|
|
@@ -1890,28 +1984,37 @@ async function runTestMatrixParallel(runFlags, configPath, selectors, suiteSelec
|
|
|
1890
1984
|
const modeTimes = modes.map(() => "...");
|
|
1891
1985
|
for (let i = 0; i < modes.length; i++) {
|
|
1892
1986
|
const modeName = modes[i];
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1987
|
+
let result;
|
|
1988
|
+
try {
|
|
1989
|
+
const buildStartedAt = Date.now();
|
|
1990
|
+
await buildFileForMode({
|
|
1991
|
+
configPath,
|
|
1992
|
+
file,
|
|
1993
|
+
modeName,
|
|
1994
|
+
buildFeatureToggles,
|
|
1995
|
+
buildPool,
|
|
1996
|
+
});
|
|
1997
|
+
buildIntervals.push({ start: buildStartedAt, end: Date.now() });
|
|
1998
|
+
const buildInvocation = await getBuildInvocationPreview(configPath, file, modeName, buildFeatureToggles);
|
|
1999
|
+
const artifactKey = resolvePerFileArtifactKey(file, duplicateSpecBasenames);
|
|
2000
|
+
result = await run(runFlags, configPath, [file], false, {
|
|
2001
|
+
reporter: silentReporter,
|
|
2002
|
+
reporterKind: "default",
|
|
2003
|
+
suiteSelectors,
|
|
2004
|
+
emitRunStart: false,
|
|
2005
|
+
emitRunComplete: false,
|
|
2006
|
+
logFileName: `test.${artifactKey}.log.json`,
|
|
2007
|
+
coverageFileName: `coverage.${artifactKey}.log.json`,
|
|
2008
|
+
buildCommand: formatBuildInvocation(buildInvocation),
|
|
2009
|
+
modeName,
|
|
2010
|
+
});
|
|
2011
|
+
}
|
|
2012
|
+
catch (error) {
|
|
2013
|
+
const buildFailure = getBuildFailureErrorLike(error);
|
|
2014
|
+
if (!buildFailure)
|
|
2015
|
+
throw error;
|
|
2016
|
+
result = createBuildFailureRunResult(buildFailure);
|
|
2017
|
+
}
|
|
1915
2018
|
modeTimes[i] = formatMatrixModeTime(result.stats.time);
|
|
1916
2019
|
fileResults.push(result);
|
|
1917
2020
|
}
|
|
@@ -1962,6 +2065,8 @@ async function runTestMatrixParallel(runFlags, configPath, selectors, suiteSelec
|
|
|
1962
2065
|
clean: runFlags.clean,
|
|
1963
2066
|
snapshotEnabled,
|
|
1964
2067
|
showCoverage: runFlags.showCoverage,
|
|
2068
|
+
showCoverageAll: runFlags.showCoverageAll,
|
|
2069
|
+
verbose: runFlags.verbose,
|
|
1965
2070
|
buildTime: getMergedIntervalDuration(buildIntervals),
|
|
1966
2071
|
snapshotSummary: summary.snapshotSummary,
|
|
1967
2072
|
coverageSummary: summary.coverageSummary,
|
|
@@ -2006,7 +2111,9 @@ async function runFuzzMatrixResultsParallel(configPath, selectors, fuzzerSelecto
|
|
|
2006
2111
|
async function runFuzzMatrixResults(configPath, selectors, fuzzerSelectors, modes, overrides, reporter) {
|
|
2007
2112
|
const results = [];
|
|
2008
2113
|
for (const modeName of modes) {
|
|
2009
|
-
const files = await resolveSelectedFuzzFiles(configPath, selectors, [
|
|
2114
|
+
const files = await resolveSelectedFuzzFiles(configPath, selectors, [
|
|
2115
|
+
modeName,
|
|
2116
|
+
]);
|
|
2010
2117
|
if (!files.length) {
|
|
2011
2118
|
continue;
|
|
2012
2119
|
}
|
|
@@ -2212,6 +2319,133 @@ function buildSingleModeSummary(stats, snapshotSummary, totalModes) {
|
|
|
2212
2319
|
total,
|
|
2213
2320
|
};
|
|
2214
2321
|
}
|
|
2322
|
+
function createBuildFailureRunResult(error) {
|
|
2323
|
+
const message = formatBuildFailureMessage(error);
|
|
2324
|
+
const report = {
|
|
2325
|
+
file: error.file,
|
|
2326
|
+
modeName: error.mode,
|
|
2327
|
+
suites: [
|
|
2328
|
+
{
|
|
2329
|
+
file: error.file,
|
|
2330
|
+
description: formatSpecDisplayPath(error.file),
|
|
2331
|
+
depth: 0,
|
|
2332
|
+
kind: "build-error",
|
|
2333
|
+
verdict: "fail",
|
|
2334
|
+
time: {
|
|
2335
|
+
start: 0,
|
|
2336
|
+
end: 0,
|
|
2337
|
+
},
|
|
2338
|
+
suites: [],
|
|
2339
|
+
logs: [],
|
|
2340
|
+
tests: [
|
|
2341
|
+
{
|
|
2342
|
+
order: 0,
|
|
2343
|
+
type: "build-error",
|
|
2344
|
+
verdict: "fail",
|
|
2345
|
+
left: null,
|
|
2346
|
+
right: null,
|
|
2347
|
+
instr: "build failed before the test could run",
|
|
2348
|
+
message,
|
|
2349
|
+
location: "",
|
|
2350
|
+
},
|
|
2351
|
+
],
|
|
2352
|
+
modeName: error.mode,
|
|
2353
|
+
buildCommand: formatBuildInvocation(error.invocation),
|
|
2354
|
+
runCommand: "",
|
|
2355
|
+
},
|
|
2356
|
+
],
|
|
2357
|
+
coverage: {
|
|
2358
|
+
total: 0,
|
|
2359
|
+
covered: 0,
|
|
2360
|
+
uncovered: 0,
|
|
2361
|
+
percent: 100,
|
|
2362
|
+
points: [],
|
|
2363
|
+
},
|
|
2364
|
+
runCommand: "",
|
|
2365
|
+
buildCommand: formatBuildInvocation(error.invocation),
|
|
2366
|
+
snapshotSummary: {
|
|
2367
|
+
matched: 0,
|
|
2368
|
+
created: 0,
|
|
2369
|
+
updated: 0,
|
|
2370
|
+
failed: 0,
|
|
2371
|
+
},
|
|
2372
|
+
};
|
|
2373
|
+
return {
|
|
2374
|
+
failed: true,
|
|
2375
|
+
buildTime: 0,
|
|
2376
|
+
stats: {
|
|
2377
|
+
passedFiles: 0,
|
|
2378
|
+
failedFiles: 1,
|
|
2379
|
+
skippedFiles: 0,
|
|
2380
|
+
passedSuites: 0,
|
|
2381
|
+
failedSuites: 1,
|
|
2382
|
+
skippedSuites: 0,
|
|
2383
|
+
passedTests: 0,
|
|
2384
|
+
failedTests: 1,
|
|
2385
|
+
skippedTests: 0,
|
|
2386
|
+
time: 0,
|
|
2387
|
+
failedEntries: [
|
|
2388
|
+
{
|
|
2389
|
+
...report.suites[0],
|
|
2390
|
+
file: error.file,
|
|
2391
|
+
modeName: error.mode,
|
|
2392
|
+
buildCommand: report.buildCommand,
|
|
2393
|
+
runCommand: "",
|
|
2394
|
+
},
|
|
2395
|
+
],
|
|
2396
|
+
},
|
|
2397
|
+
snapshotSummary: report.snapshotSummary,
|
|
2398
|
+
coverageSummary: {
|
|
2399
|
+
enabled: false,
|
|
2400
|
+
showPoints: false,
|
|
2401
|
+
total: 0,
|
|
2402
|
+
covered: 0,
|
|
2403
|
+
uncovered: 0,
|
|
2404
|
+
percent: 100,
|
|
2405
|
+
files: [],
|
|
2406
|
+
},
|
|
2407
|
+
reports: [report],
|
|
2408
|
+
};
|
|
2409
|
+
}
|
|
2410
|
+
function formatBuildFailureMessage(error) {
|
|
2411
|
+
const parts = [];
|
|
2412
|
+
const stderr = error.stderr.trim();
|
|
2413
|
+
const stdout = error.stdout.trim();
|
|
2414
|
+
if (stderr.length) {
|
|
2415
|
+
parts.push(`stderr:\n${stderr}`);
|
|
2416
|
+
}
|
|
2417
|
+
if (stdout.length) {
|
|
2418
|
+
parts.push(`stdout:\n${stdout}`);
|
|
2419
|
+
}
|
|
2420
|
+
if (error.crashLogPath.length) {
|
|
2421
|
+
parts.push(`Crash log:\n${error.crashLogPath}`);
|
|
2422
|
+
}
|
|
2423
|
+
return parts.join("\n\n") || "build failed with no compiler output";
|
|
2424
|
+
}
|
|
2425
|
+
function getBuildFailureErrorLike(error) {
|
|
2426
|
+
if (error instanceof BuildFailureError) {
|
|
2427
|
+
return error;
|
|
2428
|
+
}
|
|
2429
|
+
if (!error || typeof error != "object") {
|
|
2430
|
+
return null;
|
|
2431
|
+
}
|
|
2432
|
+
const candidate = error;
|
|
2433
|
+
if (candidate.name != "BuildFailureError") {
|
|
2434
|
+
return null;
|
|
2435
|
+
}
|
|
2436
|
+
if (typeof candidate.file != "string" ||
|
|
2437
|
+
typeof candidate.mode != "string" ||
|
|
2438
|
+
typeof candidate.stdout != "string" ||
|
|
2439
|
+
typeof candidate.stderr != "string" ||
|
|
2440
|
+
typeof candidate.crashLogPath != "string" ||
|
|
2441
|
+
!candidate.invocation ||
|
|
2442
|
+
typeof candidate.invocation != "object" ||
|
|
2443
|
+
typeof candidate.invocation.command != "string" ||
|
|
2444
|
+
!Array.isArray(candidate.invocation.args)) {
|
|
2445
|
+
return null;
|
|
2446
|
+
}
|
|
2447
|
+
return candidate;
|
|
2448
|
+
}
|
|
2215
2449
|
function applyConfiguredFileTotalToStats(stats, fileSummaryTotal) {
|
|
2216
2450
|
const total = Math.max(fileSummaryTotal, 0);
|
|
2217
2451
|
const executed = stats.failedFiles + stats.passedFiles + stats.skippedFiles;
|
|
@@ -2320,7 +2554,7 @@ async function buildNoTestFilesMatchedError(configPath, selectors, includeFuzz =
|
|
|
2320
2554
|
if (configuredFiles.length) {
|
|
2321
2555
|
const sample = configuredFiles
|
|
2322
2556
|
.slice(0, 5)
|
|
2323
|
-
.map((file) =>
|
|
2557
|
+
.map((file) => formatSpecDisplayPath(file))
|
|
2324
2558
|
.join(", ");
|
|
2325
2559
|
lines.push(`Configured specs (${configuredFiles.length}): ${sample}${configuredFiles.length > 5 ? ", ..." : ""}`);
|
|
2326
2560
|
}
|
|
@@ -2628,7 +2862,7 @@ async function handleMissingWebBrowsers(missing) {
|
|
|
2628
2862
|
const details = "no web-capable browser was found in PATH, BROWSER, or Playwright cache";
|
|
2629
2863
|
const selected = choosePreferredBrowserInstall(missing);
|
|
2630
2864
|
const installCommand = selected == "webkit"
|
|
2631
|
-
?
|
|
2865
|
+
? "npx -y playwright install webkit"
|
|
2632
2866
|
: `npx -y playwright install ${selected}`;
|
|
2633
2867
|
if (!canPromptForWebInstall()) {
|
|
2634
2868
|
throw new Error(`web target requires a browser for mode(s) ${scope}; ${details}. Export BROWSER or install one with "${installCommand}".`);
|
|
@@ -2748,10 +2982,10 @@ function isPlaywrightBrowserExecutable(browser) {
|
|
|
2748
2982
|
function extractPlaywrightDepsSummary(result) {
|
|
2749
2983
|
const stdout = typeof result.stdout == "string"
|
|
2750
2984
|
? result.stdout
|
|
2751
|
-
: result.stdout?.toString("utf8") ?? "";
|
|
2985
|
+
: (result.stdout?.toString("utf8") ?? "");
|
|
2752
2986
|
const stderr = typeof result.stderr == "string"
|
|
2753
2987
|
? result.stderr
|
|
2754
|
-
: result.stderr?.toString("utf8") ?? "";
|
|
2988
|
+
: (result.stderr?.toString("utf8") ?? "");
|
|
2755
2989
|
return [stdout.trim(), stderr.trim()].filter(Boolean).join("\n");
|
|
2756
2990
|
}
|
|
2757
2991
|
function canPromptForWebInstall() {
|
|
@@ -2886,9 +3120,7 @@ function resolveSystemBrowserExecutable(browser) {
|
|
|
2886
3120
|
"Firefox.app/Contents/MacOS/firefox",
|
|
2887
3121
|
"Firefox Developer Edition.app/Contents/MacOS/firefox",
|
|
2888
3122
|
],
|
|
2889
|
-
msedge: [
|
|
2890
|
-
"Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
|
2891
|
-
],
|
|
3123
|
+
msedge: ["Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
|
|
2892
3124
|
webkit: [],
|
|
2893
3125
|
};
|
|
2894
3126
|
for (const root of macSearchRoots) {
|