as-test 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +116 -1
- package/README.md +138 -406
- package/as-test.config.schema.json +210 -17
- package/assembly/__fuzz__/array.fuzz.ts +10 -0
- package/assembly/__fuzz__/bytes.fuzz.ts +8 -0
- package/assembly/__fuzz__/math.fuzz.ts +9 -0
- package/assembly/__fuzz__/string.fuzz.ts +21 -0
- package/assembly/index.ts +141 -86
- package/assembly/src/expectation.ts +104 -19
- package/assembly/src/fuzz.ts +723 -0
- package/assembly/src/log.ts +6 -1
- package/assembly/src/suite.ts +45 -3
- package/assembly/util/json.ts +38 -4
- package/assembly/util/wipc.ts +35 -26
- package/bin/build-worker-pool.js +149 -0
- package/bin/build-worker.js +43 -0
- package/bin/commands/build-core.js +221 -29
- package/bin/commands/build.js +1 -0
- package/bin/commands/fuzz-core.js +306 -0
- package/bin/commands/fuzz.js +10 -0
- package/bin/commands/init-core.js +129 -24
- package/bin/commands/run-core.js +525 -123
- package/bin/commands/run.js +4 -1
- package/bin/commands/test.js +8 -3
- package/bin/commands/web-runner-source.js +634 -0
- package/bin/crash-store.js +64 -0
- package/bin/index.js +1484 -169
- package/bin/reporters/default.js +281 -49
- package/bin/reporters/tap.js +83 -2
- package/bin/types.js +19 -2
- package/bin/util.js +315 -33
- package/bin/wipc.js +79 -0
- package/package.json +19 -9
- package/transform/lib/coverage.js +1 -2
- package/transform/lib/index.js +3 -3
- package/transform/lib/log.js +1 -1
package/bin/reporters/default.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import chalk from "chalk";
|
|
2
2
|
import { diff } from "typer-diff";
|
|
3
|
+
import { readFileSync } from "fs";
|
|
4
|
+
import * as path from "path";
|
|
3
5
|
import { formatTime } from "../util.js";
|
|
4
6
|
export const createReporter = (context) => {
|
|
5
7
|
return new DefaultReporter(context);
|
|
@@ -14,6 +16,8 @@ class DefaultReporter {
|
|
|
14
16
|
this.fileHasWarning = false;
|
|
15
17
|
this.verboseMode = false;
|
|
16
18
|
this.cleanMode = false;
|
|
19
|
+
this.hasRenderedTestFiles = false;
|
|
20
|
+
this.hasRenderedFuzzFiles = false;
|
|
17
21
|
}
|
|
18
22
|
canRewriteLine() {
|
|
19
23
|
return (!this.cleanMode &&
|
|
@@ -118,6 +122,8 @@ class DefaultReporter {
|
|
|
118
122
|
onRunStart(event) {
|
|
119
123
|
this.verboseMode = Boolean(event.verbose);
|
|
120
124
|
this.cleanMode = Boolean(event.clean);
|
|
125
|
+
this.hasRenderedTestFiles = false;
|
|
126
|
+
this.hasRenderedFuzzFiles = false;
|
|
121
127
|
}
|
|
122
128
|
onFileStart(event) {
|
|
123
129
|
this.currentFile = event.file;
|
|
@@ -130,13 +136,24 @@ class DefaultReporter {
|
|
|
130
136
|
this.renderVerboseState();
|
|
131
137
|
return;
|
|
132
138
|
}
|
|
133
|
-
if (this.verboseMode
|
|
139
|
+
if (!this.verboseMode) {
|
|
140
|
+
if (!this.canRewriteLine()) {
|
|
141
|
+
this.context.stdout.write(`${this.badgeRunning()} ${event.file}\n`);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
this.clearRenderedBlock();
|
|
145
|
+
this.context.stdout.write(`${this.badgeRunning()} ${event.file}`);
|
|
146
|
+
this.renderedLines = 1;
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
if (!this.canRewriteLine()) {
|
|
134
150
|
this.context.stdout.write(`${this.badgeRunning()} ${event.file}\n`);
|
|
135
151
|
return;
|
|
136
152
|
}
|
|
137
153
|
this.renderLiveState();
|
|
138
154
|
}
|
|
139
155
|
onFileEnd(event) {
|
|
156
|
+
this.hasRenderedTestFiles = true;
|
|
140
157
|
if (this.verboseMode && this.canRewriteLine()) {
|
|
141
158
|
this.renderVerboseState(event);
|
|
142
159
|
this.context.stdout.write("\n");
|
|
@@ -158,6 +175,8 @@ class DefaultReporter {
|
|
|
158
175
|
onSuiteStart(event) {
|
|
159
176
|
if (this.cleanMode)
|
|
160
177
|
return;
|
|
178
|
+
if (!this.verboseMode)
|
|
179
|
+
return;
|
|
161
180
|
const depth = Math.max(event.depth, 0);
|
|
162
181
|
if (this.verboseMode && this.canRewriteLine()) {
|
|
163
182
|
if (this.currentFile !== event.file)
|
|
@@ -183,6 +202,8 @@ class DefaultReporter {
|
|
|
183
202
|
onSuiteEnd(event) {
|
|
184
203
|
if (this.cleanMode)
|
|
185
204
|
return;
|
|
205
|
+
if (!this.verboseMode)
|
|
206
|
+
return;
|
|
186
207
|
const depth = Math.max(event.depth, 0);
|
|
187
208
|
const verdict = String(event.verdict ?? "none");
|
|
188
209
|
if (this.verboseMode && this.canRewriteLine()) {
|
|
@@ -214,7 +235,7 @@ class DefaultReporter {
|
|
|
214
235
|
onAssertionFail(_event) { }
|
|
215
236
|
onSnapshotMissing(event) {
|
|
216
237
|
this.fileHasWarning = true;
|
|
217
|
-
const warnLine = `${chalk.bgYellow.black(" WARN ")} missing snapshot for ${chalk.dim(event.key)}. Re-run with ${chalk.bold("--
|
|
238
|
+
const warnLine = `${chalk.bgYellow.black(" WARN ")} missing snapshot for ${chalk.dim(event.key)}. Re-run with ${chalk.bold("--create-snapshots")} to create it.\n`;
|
|
218
239
|
if (!this.canRewriteLine() || !this.currentFile) {
|
|
219
240
|
this.context.stdout.write(warnLine);
|
|
220
241
|
return;
|
|
@@ -228,13 +249,37 @@ class DefaultReporter {
|
|
|
228
249
|
this.renderLiveState();
|
|
229
250
|
}
|
|
230
251
|
}
|
|
252
|
+
onWarning(event) {
|
|
253
|
+
this.fileHasWarning = true;
|
|
254
|
+
const warnLine = `${chalk.bgYellow.black(" WARN ")} ${event.message}\n`;
|
|
255
|
+
if (!this.canRewriteLine() || !this.currentFile) {
|
|
256
|
+
this.context.stdout.write(warnLine);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
this.clearRenderedBlock();
|
|
260
|
+
this.context.stdout.write(warnLine);
|
|
261
|
+
if (this.verboseMode) {
|
|
262
|
+
this.renderVerboseState();
|
|
263
|
+
}
|
|
264
|
+
else {
|
|
265
|
+
this.renderLiveState();
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
onLog(event) {
|
|
269
|
+
if (this.cleanMode)
|
|
270
|
+
return;
|
|
271
|
+
if (this.verboseMode || !this.canRewriteLine()) {
|
|
272
|
+
const depth = Math.max(event.depth, 0);
|
|
273
|
+
this.context.stdout.write(`${" ".repeat(depth + 1)}${chalk.dim("LOG")} ${event.text}\n`);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
231
276
|
onRunComplete(event) {
|
|
232
277
|
this.clearRenderedBlock();
|
|
233
278
|
if (!event.clean) {
|
|
234
279
|
renderFailedSuites(event.stats.failedEntries);
|
|
235
280
|
}
|
|
236
281
|
if (event.snapshotEnabled) {
|
|
237
|
-
renderSnapshotSummary(event.snapshotSummary);
|
|
282
|
+
renderSnapshotSummary(event.snapshotSummary, true);
|
|
238
283
|
}
|
|
239
284
|
if (event.coverageSummary.enabled) {
|
|
240
285
|
renderCoverageSummary(event.coverageSummary);
|
|
@@ -244,6 +289,153 @@ class DefaultReporter {
|
|
|
244
289
|
}
|
|
245
290
|
renderTotals(event.stats, event);
|
|
246
291
|
}
|
|
292
|
+
onFuzzComplete(event) {
|
|
293
|
+
renderFuzzSummary(this.context, event, this.hasRenderedTestFiles);
|
|
294
|
+
}
|
|
295
|
+
onFuzzFileComplete(event) {
|
|
296
|
+
this.hasRenderedFuzzFiles = true;
|
|
297
|
+
renderFuzzFileSummary(this.context, event.results);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
function renderFuzzFileSummary(context, results) {
|
|
301
|
+
if (!results.length)
|
|
302
|
+
return;
|
|
303
|
+
const file = results[0].file;
|
|
304
|
+
const itemFailed = results.some((mode) => mode.crashes > 0 || mode.fuzzers.some((fuzzer) => fuzzer.failed > 0));
|
|
305
|
+
const itemSkipped = !itemFailed &&
|
|
306
|
+
results.length > 0 &&
|
|
307
|
+
results.every((mode) => mode.fuzzers.length > 0 &&
|
|
308
|
+
mode.fuzzers.every((fuzzer) => fuzzer.skipped > 0));
|
|
309
|
+
const itemBadge = itemFailed
|
|
310
|
+
? chalk.bgRed.white(" FAIL ")
|
|
311
|
+
: itemSkipped
|
|
312
|
+
? chalk.bgBlackBright.white(" SKIP ")
|
|
313
|
+
: chalk.bgGreenBright.black(" PASS ");
|
|
314
|
+
const detail = formatTime(averageFuzzModeTime(results));
|
|
315
|
+
const crashFile = firstFuzzCrashFile(results);
|
|
316
|
+
const crashSuffix = crashFile != null ? ` ${chalk.dim(`-> ${crashFile}`)}` : "";
|
|
317
|
+
context.stdout.write(`${itemBadge} ${path.basename(file)} ${chalk.dim(detail)}${crashSuffix}\n`);
|
|
318
|
+
renderFailedFuzzers(groupFuzzResultsByFile(results));
|
|
319
|
+
}
|
|
320
|
+
function renderFuzzSummary(context, event, hasRenderedTestFiles) {
|
|
321
|
+
context.stdout.write("\n");
|
|
322
|
+
if (!hasRenderedTestFiles) {
|
|
323
|
+
renderStandaloneFuzzTotals(event);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
function renderFailedFuzzers(results) {
|
|
327
|
+
let rendered = false;
|
|
328
|
+
for (const result of results) {
|
|
329
|
+
for (const modeResult of result.modes) {
|
|
330
|
+
const relativeFile = toRelativeResultPath(modeResult.file);
|
|
331
|
+
const repro = buildFuzzReproCommand(relativeFile, modeResult.seed, modeResult.modeName);
|
|
332
|
+
if (modeResult.crashes > 0 && !modeResult.fuzzers.length) {
|
|
333
|
+
if (!rendered) {
|
|
334
|
+
console.log("");
|
|
335
|
+
rendered = true;
|
|
336
|
+
}
|
|
337
|
+
console.log(`${chalk.bgRed(" FAIL ")} ${chalk.dim(path.basename(modeResult.file))} ${chalk.dim("(crash)")}`);
|
|
338
|
+
console.log(chalk.dim(`Mode: ${modeResult.modeName}`));
|
|
339
|
+
console.log(chalk.dim(`Runs: ${modeResult.runs} configured`));
|
|
340
|
+
console.log(chalk.dim(`Repro: ${repro}`));
|
|
341
|
+
console.log(chalk.dim(`Seed: ${modeResult.seed}`));
|
|
342
|
+
if (modeResult.crashFiles.length) {
|
|
343
|
+
console.log(chalk.dim(`Crash: ${modeResult.crashFiles[0]}`));
|
|
344
|
+
}
|
|
345
|
+
console.log("");
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
for (const fuzzer of modeResult.fuzzers) {
|
|
349
|
+
if (fuzzer.failed <= 0 && fuzzer.crashed <= 0)
|
|
350
|
+
continue;
|
|
351
|
+
if (!rendered) {
|
|
352
|
+
console.log("");
|
|
353
|
+
rendered = true;
|
|
354
|
+
}
|
|
355
|
+
console.log(`${chalk.bgRed(" FAIL ")} ${formatFuzzFailureTitle(modeResult.file, fuzzer.name)}`);
|
|
356
|
+
if (fuzzer.failure?.message?.length) {
|
|
357
|
+
console.log(chalk.dim(`Message: ${fuzzer.failure.message}`));
|
|
358
|
+
}
|
|
359
|
+
console.log(chalk.dim(`Mode: ${modeResult.modeName}`));
|
|
360
|
+
console.log(chalk.dim(`Runs: ${fuzzer.passed + fuzzer.failed + fuzzer.crashed} completed (${fuzzer.passed} passed, ${fuzzer.failed} failed, ${fuzzer.crashed} crashed)`));
|
|
361
|
+
console.log(chalk.dim(`Repro: ${repro}`));
|
|
362
|
+
console.log(chalk.dim(`Seed: ${modeResult.seed}`));
|
|
363
|
+
if (modeResult.crashFiles.length) {
|
|
364
|
+
console.log(chalk.dim(`Crash: ${modeResult.crashFiles[0]}`));
|
|
365
|
+
}
|
|
366
|
+
console.log("");
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
return rendered;
|
|
371
|
+
}
|
|
372
|
+
function groupFuzzResultsByFile(results) {
|
|
373
|
+
const grouped = new Map();
|
|
374
|
+
for (const result of results) {
|
|
375
|
+
const current = grouped.get(result.file) ?? [];
|
|
376
|
+
current.push(result);
|
|
377
|
+
grouped.set(result.file, current);
|
|
378
|
+
}
|
|
379
|
+
return [...grouped.entries()]
|
|
380
|
+
.sort((a, b) => a[0].localeCompare(b[0]))
|
|
381
|
+
.map(([file, modes]) => ({ file, modes }));
|
|
382
|
+
}
|
|
383
|
+
function firstFuzzCrashFile(results) {
|
|
384
|
+
for (const result of results) {
|
|
385
|
+
if (result.crashFiles.length)
|
|
386
|
+
return result.crashFiles[0];
|
|
387
|
+
}
|
|
388
|
+
return null;
|
|
389
|
+
}
|
|
390
|
+
function averageFuzzModeTime(results) {
|
|
391
|
+
if (!results.length)
|
|
392
|
+
return 0;
|
|
393
|
+
return results.reduce((sum, result) => sum + result.time, 0) / results.length;
|
|
394
|
+
}
|
|
395
|
+
function buildFuzzReproCommand(file, seed, modeName) {
|
|
396
|
+
const modeArg = modeName != "default" ? ` --mode ${modeName}` : "";
|
|
397
|
+
return `ast fuzz ${file}${modeArg} --seed ${seed}`;
|
|
398
|
+
}
|
|
399
|
+
function toRelativeResultPath(file) {
|
|
400
|
+
const relative = path.relative(process.cwd(), path.resolve(process.cwd(), file));
|
|
401
|
+
return relative.length ? relative : file;
|
|
402
|
+
}
|
|
403
|
+
function formatFuzzFailureTitle(file, name) {
|
|
404
|
+
const location = findFuzzLocation(file, name);
|
|
405
|
+
const suffix = location
|
|
406
|
+
? ` (${path.basename(file)}:${location})`
|
|
407
|
+
: ` (${path.basename(file)})`;
|
|
408
|
+
return `${chalk.dim(name)}${chalk.dim(suffix)}`;
|
|
409
|
+
}
|
|
410
|
+
function findFuzzLocation(file, name) {
|
|
411
|
+
try {
|
|
412
|
+
const source = readFileSync(path.resolve(process.cwd(), file), "utf8");
|
|
413
|
+
const patterns = [`fuzz("${name}"`, `fuzz('${name}'`];
|
|
414
|
+
patterns.push(`xfuzz("${name}"`, `xfuzz('${name}'`);
|
|
415
|
+
let index = -1;
|
|
416
|
+
for (const pattern of patterns) {
|
|
417
|
+
index = source.indexOf(pattern);
|
|
418
|
+
if (index != -1)
|
|
419
|
+
break;
|
|
420
|
+
}
|
|
421
|
+
if (index == -1)
|
|
422
|
+
return null;
|
|
423
|
+
let line = 1;
|
|
424
|
+
let column = 1;
|
|
425
|
+
for (let i = 0; i < index; i++) {
|
|
426
|
+
if (source.charCodeAt(i) == 10) {
|
|
427
|
+
line++;
|
|
428
|
+
column = 1;
|
|
429
|
+
}
|
|
430
|
+
else {
|
|
431
|
+
column++;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
return `${line}:${column}`;
|
|
435
|
+
}
|
|
436
|
+
catch {
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
247
439
|
}
|
|
248
440
|
function renderFailedSuites(failedEntries) {
|
|
249
441
|
if (!failedEntries.length)
|
|
@@ -272,7 +464,8 @@ function collectSuiteFailures(suite, file, path, printed) {
|
|
|
272
464
|
const title = `${nextPath.join(" > ")}#${assertionIndex}`;
|
|
273
465
|
const loc = String(test.location ?? "");
|
|
274
466
|
const where = loc.length ? `${file}:${loc}` : file;
|
|
275
|
-
const
|
|
467
|
+
const modeName = String(suiteAny.modeName ?? "");
|
|
468
|
+
const dedupeKey = `${file}::${modeName}::${title}::${String(test.left)}::${String(test.right)}`;
|
|
276
469
|
if (printed.has(dedupeKey))
|
|
277
470
|
continue;
|
|
278
471
|
printed.add(dedupeKey);
|
|
@@ -300,6 +493,9 @@ function collectSuiteFailures(suite, file, path, printed) {
|
|
|
300
493
|
}
|
|
301
494
|
}
|
|
302
495
|
console.log(`${chalk.bgRed(" FAIL ")} ${chalk.dim(title)} ${chalk.dim("(" + where + ")")}`);
|
|
496
|
+
if (modeName.length) {
|
|
497
|
+
console.log(chalk.dim(`Mode: ${modeName}`));
|
|
498
|
+
}
|
|
303
499
|
console.log(`${chalk.dim("(expected) ->")} ${expected}`);
|
|
304
500
|
console.log(`${chalk.dim("(received) ->")} ${chalk.dim(left)}\n`);
|
|
305
501
|
}
|
|
@@ -310,58 +506,94 @@ function collectSuiteFailures(suite, file, path, printed) {
|
|
|
310
506
|
collectSuiteFailures(sub, file, nextPath, printed);
|
|
311
507
|
}
|
|
312
508
|
}
|
|
313
|
-
function renderSnapshotSummary(snapshotSummary) {
|
|
314
|
-
|
|
509
|
+
function renderSnapshotSummary(snapshotSummary, leadingGap = true) {
|
|
510
|
+
if (leadingGap) {
|
|
511
|
+
console.log("");
|
|
512
|
+
}
|
|
315
513
|
console.log(`${chalk.bold("Snapshots:")} ${chalk.greenBright(snapshotSummary.matched)} matched, ${chalk.blueBright(snapshotSummary.created)} created, ${chalk.blueBright(snapshotSummary.updated)} updated, ${snapshotSummary.failed ? chalk.red(snapshotSummary.failed) : chalk.greenBright("0")} failed`);
|
|
316
514
|
}
|
|
317
515
|
function renderTotals(stats, event) {
|
|
318
516
|
console.log("");
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
:
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
? chalk.gray(stats.skippedTests + " skipped")
|
|
348
|
-
: chalk.gray("0 skipped")));
|
|
349
|
-
process.stdout.write(", " + (stats.failedTests + stats.passedTests + stats.skippedTests) + " total\n");
|
|
517
|
+
const filesSummary = {
|
|
518
|
+
failed: stats.failedFiles,
|
|
519
|
+
skipped: stats.skippedFiles,
|
|
520
|
+
total: stats.failedFiles + stats.passedFiles + stats.skippedFiles,
|
|
521
|
+
};
|
|
522
|
+
const suitesSummary = {
|
|
523
|
+
failed: stats.failedSuites,
|
|
524
|
+
skipped: stats.skippedSuites,
|
|
525
|
+
total: stats.failedSuites + stats.passedSuites + stats.skippedSuites,
|
|
526
|
+
};
|
|
527
|
+
const testsSummary = {
|
|
528
|
+
failed: stats.failedTests,
|
|
529
|
+
skipped: stats.skippedTests,
|
|
530
|
+
total: stats.failedTests + stats.passedTests + stats.skippedTests,
|
|
531
|
+
};
|
|
532
|
+
const layout = createSummaryLayout([
|
|
533
|
+
event.fuzzSummary,
|
|
534
|
+
filesSummary,
|
|
535
|
+
suitesSummary,
|
|
536
|
+
testsSummary,
|
|
537
|
+
event.modeSummary,
|
|
538
|
+
]);
|
|
539
|
+
if (event.fuzzSummary) {
|
|
540
|
+
renderFuzzTotals(event.fuzzSummary, layout);
|
|
541
|
+
}
|
|
542
|
+
renderSummaryLine("Files:", filesSummary, layout);
|
|
543
|
+
renderSummaryLine("Suites:", suitesSummary, layout);
|
|
544
|
+
renderSummaryLine("Tests:", testsSummary, layout);
|
|
350
545
|
if (event.modeSummary) {
|
|
351
|
-
renderModeSummary(event.modeSummary);
|
|
546
|
+
renderModeSummary(event.modeSummary, layout);
|
|
352
547
|
}
|
|
353
|
-
process.stdout.write(chalk.bold("Time:
|
|
548
|
+
process.stdout.write(chalk.bold("Time:".padEnd(9)) +
|
|
549
|
+
formatTime(stats.time) +
|
|
550
|
+
chalk.dim(` (${formatTime(event.buildTime)} build)`) +
|
|
551
|
+
"\n");
|
|
552
|
+
}
|
|
553
|
+
function renderModeSummary(summary, layout) {
|
|
554
|
+
renderSummaryLine("Modes:", summary, layout);
|
|
555
|
+
}
|
|
556
|
+
function renderFuzzTotals(summary, layout) {
|
|
557
|
+
renderSummaryLine("Fuzz:", summary, layout);
|
|
558
|
+
}
|
|
559
|
+
function renderStandaloneFuzzTotals(event) {
|
|
560
|
+
console.log("");
|
|
561
|
+
const layout = createSummaryLayout([
|
|
562
|
+
event.fuzzingSummary,
|
|
563
|
+
event.suiteSummary,
|
|
564
|
+
event.modeSummary,
|
|
565
|
+
]);
|
|
566
|
+
renderSummaryLine("Fuzz:", event.fuzzingSummary, layout);
|
|
567
|
+
renderSummaryLine("Suites:", event.suiteSummary, layout);
|
|
568
|
+
renderSummaryLine("Modes:", event.modeSummary, layout);
|
|
569
|
+
process.stdout.write(chalk.bold("Time:".padEnd(9)) +
|
|
570
|
+
formatTime(event.time) +
|
|
571
|
+
chalk.dim(` (${formatTime(event.buildTime)} build)`) +
|
|
572
|
+
"\n");
|
|
573
|
+
}
|
|
574
|
+
function createSummaryLayout(summaries) {
|
|
575
|
+
return {
|
|
576
|
+
failedWidth: Math.max(...summaries.map((summary) => summary ? `${summary.failed} failed`.length : 0)),
|
|
577
|
+
skippedWidth: Math.max(...summaries.map((summary) => summary ? `${summary.skipped} skipped`.length : 0)),
|
|
578
|
+
totalWidth: Math.max(...summaries.map((summary) => summary ? `${summary.total} total`.length : 0)),
|
|
579
|
+
};
|
|
354
580
|
}
|
|
355
|
-
function
|
|
356
|
-
|
|
581
|
+
function renderSummaryLine(label, summary, layout = {
|
|
582
|
+
failedWidth: `${summary.failed} failed`.length,
|
|
583
|
+
skippedWidth: `${summary.skipped} skipped`.length,
|
|
584
|
+
totalWidth: `${summary.total} total`.length,
|
|
585
|
+
}) {
|
|
586
|
+
const failedText = `${summary.failed} failed`;
|
|
587
|
+
const skippedText = `${summary.skipped} skipped`;
|
|
588
|
+
const totalText = `${summary.total} total`;
|
|
589
|
+
process.stdout.write(chalk.bold(label.padEnd(9)));
|
|
357
590
|
process.stdout.write(summary.failed
|
|
358
|
-
? chalk.bold.red(
|
|
359
|
-
: chalk.bold.greenBright(
|
|
360
|
-
process.stdout.write(", "
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
process.stdout.write(", " + summary.total + " total\n");
|
|
591
|
+
? chalk.bold.red(failedText.padStart(layout.failedWidth))
|
|
592
|
+
: chalk.bold.greenBright(failedText.padStart(layout.failedWidth)));
|
|
593
|
+
process.stdout.write(", ");
|
|
594
|
+
process.stdout.write(chalk.gray(skippedText.padStart(layout.skippedWidth)));
|
|
595
|
+
process.stdout.write(", ");
|
|
596
|
+
process.stdout.write(totalText.padStart(layout.totalWidth) + "\n");
|
|
365
597
|
}
|
|
366
598
|
function renderCoverageSummary(summary) {
|
|
367
599
|
const pct = summary.total
|
|
@@ -373,7 +605,7 @@ function renderCoverageSummary(summary) {
|
|
|
373
605
|
? chalk.yellowBright
|
|
374
606
|
: chalk.redBright;
|
|
375
607
|
console.log("");
|
|
376
|
-
console.log(`${chalk.bold("Coverage:")} ${color(pct + "%")} ${chalk.dim(`(${summary.covered}/${summary.total} points, ${summary.uncovered} uncovered)
|
|
608
|
+
console.log(`${chalk.bold("Coverage:")} ${color(pct + "%")} ${chalk.dim(`(${summary.covered}/${summary.total} points, ${summary.uncovered} uncovered)`)}`);
|
|
377
609
|
}
|
|
378
610
|
function renderCoveragePoints(files) {
|
|
379
611
|
console.log("");
|
package/bin/reporters/tap.js
CHANGED
|
@@ -10,9 +10,39 @@ class TapReporter {
|
|
|
10
10
|
constructor(context, config) {
|
|
11
11
|
this.context = context;
|
|
12
12
|
this.config = config;
|
|
13
|
+
this.pendingRunEvent = null;
|
|
14
|
+
this.pendingFuzzEvent = null;
|
|
15
|
+
this.pendingFuzzPoints = [];
|
|
13
16
|
}
|
|
14
17
|
onRunComplete(event) {
|
|
15
|
-
|
|
18
|
+
this.pendingRunEvent = event;
|
|
19
|
+
}
|
|
20
|
+
onFuzzComplete(event) {
|
|
21
|
+
this.pendingFuzzEvent = event;
|
|
22
|
+
}
|
|
23
|
+
onFuzzFileComplete(event) {
|
|
24
|
+
this.pendingFuzzPoints.push(...collectFuzzTapPoints({
|
|
25
|
+
results: event.results,
|
|
26
|
+
time: event.results.reduce((sum, item) => sum + item.time, 0),
|
|
27
|
+
buildTime: event.results.reduce((sum, item) => sum + item.buildTime, 0),
|
|
28
|
+
fuzzingSummary: { failed: 0, skipped: 0, total: 0 },
|
|
29
|
+
suiteSummary: { failed: 0, skipped: 0, total: 0 },
|
|
30
|
+
modeSummary: { failed: 0, skipped: 0, total: 0 },
|
|
31
|
+
}));
|
|
32
|
+
}
|
|
33
|
+
flush() {
|
|
34
|
+
if (!this.pendingRunEvent && !this.pendingFuzzEvent)
|
|
35
|
+
return;
|
|
36
|
+
const points = [];
|
|
37
|
+
if (this.pendingRunEvent) {
|
|
38
|
+
points.push(...collectTapPoints(this.pendingRunEvent.reports));
|
|
39
|
+
}
|
|
40
|
+
if (this.pendingFuzzPoints.length) {
|
|
41
|
+
points.push(...this.pendingFuzzPoints);
|
|
42
|
+
}
|
|
43
|
+
else if (this.pendingFuzzEvent) {
|
|
44
|
+
points.push(...collectFuzzTapPoints(this.pendingFuzzEvent));
|
|
45
|
+
}
|
|
16
46
|
const output = buildTapDocument(points);
|
|
17
47
|
this.context.stdout.write(output);
|
|
18
48
|
for (const point of points) {
|
|
@@ -21,6 +51,9 @@ class TapReporter {
|
|
|
21
51
|
emitGitHubAnnotation(this.context, point);
|
|
22
52
|
}
|
|
23
53
|
this.writeArtifacts(points, output);
|
|
54
|
+
this.pendingRunEvent = null;
|
|
55
|
+
this.pendingFuzzEvent = null;
|
|
56
|
+
this.pendingFuzzPoints = [];
|
|
24
57
|
}
|
|
25
58
|
writeArtifacts(points, output) {
|
|
26
59
|
if (this.config.mode == "per-file") {
|
|
@@ -108,7 +141,10 @@ function buildTapDocument(points) {
|
|
|
108
141
|
return lines.join("\n") + "\n";
|
|
109
142
|
}
|
|
110
143
|
function buildFailDetails(point) {
|
|
111
|
-
const lines = [
|
|
144
|
+
const lines = [
|
|
145
|
+
" ---",
|
|
146
|
+
` message: ${JSON.stringify(point.message ?? "assertion failed")}`,
|
|
147
|
+
];
|
|
112
148
|
if (point.file) {
|
|
113
149
|
lines.push(` file: ${JSON.stringify(point.file)}`);
|
|
114
150
|
}
|
|
@@ -149,6 +185,51 @@ function collectTapPoints(reports) {
|
|
|
149
185
|
}
|
|
150
186
|
return points;
|
|
151
187
|
}
|
|
188
|
+
function collectFuzzTapPoints(event) {
|
|
189
|
+
const points = [];
|
|
190
|
+
for (const result of event.results) {
|
|
191
|
+
const durationMs = result.time;
|
|
192
|
+
if (!result.fuzzers.length) {
|
|
193
|
+
points.push({
|
|
194
|
+
name: `fuzz ${path.basename(result.file)}`,
|
|
195
|
+
status: result.crashes > 0 ? "fail" : "ok",
|
|
196
|
+
file: result.file,
|
|
197
|
+
message: result.crashes > 0
|
|
198
|
+
? buildFuzzMessage(result.runs, result.seed, result.crashFiles[0])
|
|
199
|
+
: `fuzz passed after ${result.runs} runs (seed ${result.seed})`,
|
|
200
|
+
durationMs,
|
|
201
|
+
});
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
for (const fuzzer of result.fuzzers) {
|
|
205
|
+
const crashed = fuzzer.crashed > 0 || result.crashes > 0;
|
|
206
|
+
const failed = crashed || fuzzer.failed > 0;
|
|
207
|
+
points.push({
|
|
208
|
+
name: `fuzz ${path.basename(result.file)} > ${fuzzer.name}`,
|
|
209
|
+
status: failed ? "fail" : "ok",
|
|
210
|
+
file: result.file,
|
|
211
|
+
message: failed
|
|
212
|
+
? buildFuzzerFailureMessage(result, fuzzer)
|
|
213
|
+
: `fuzz passed after ${fuzzer.runs} runs (seed ${result.seed})`,
|
|
214
|
+
durationMs: fuzzer.time.end - fuzzer.time.start,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return points;
|
|
219
|
+
}
|
|
220
|
+
function buildFuzzerFailureMessage(result, fuzzer) {
|
|
221
|
+
if (fuzzer.crashed > 0 || result.crashes > 0) {
|
|
222
|
+
return buildFuzzMessage(result.runs, result.seed, result.crashFiles[0]);
|
|
223
|
+
}
|
|
224
|
+
if (fuzzer.failure?.message?.length) {
|
|
225
|
+
return `${fuzzer.failure.message} (runs ${fuzzer.runs}, seed ${result.seed})`;
|
|
226
|
+
}
|
|
227
|
+
return `fuzz failed after ${fuzzer.runs} runs (seed ${result.seed})`;
|
|
228
|
+
}
|
|
229
|
+
function buildFuzzMessage(runs, seed, crashFile) {
|
|
230
|
+
const crashSuffix = crashFile?.length ? `, crash ${crashFile}` : "";
|
|
231
|
+
return `fuzz crashed after ${runs} runs (seed ${seed}${crashSuffix})`;
|
|
232
|
+
}
|
|
152
233
|
function collectTapPointsFromSuite(suite, file, pathStack, points) {
|
|
153
234
|
const suiteAny = suite;
|
|
154
235
|
const description = String(suiteAny.description ?? "suite");
|
package/bin/types.js
CHANGED
|
@@ -7,17 +7,20 @@ export class Config {
|
|
|
7
7
|
this.coverageDir = "./.as-test/coverage";
|
|
8
8
|
this.snapshotDir = "./.as-test/snapshots";
|
|
9
9
|
this.config = "none";
|
|
10
|
-
this.coverage =
|
|
10
|
+
this.coverage = false;
|
|
11
11
|
this.env = {};
|
|
12
12
|
this.buildOptions = new BuildOptions();
|
|
13
13
|
this.runOptions = new RunOptions();
|
|
14
|
+
this.fuzz = new FuzzConfig();
|
|
14
15
|
this.modes = {};
|
|
15
16
|
}
|
|
16
17
|
}
|
|
17
18
|
export class CoverageOptions {
|
|
18
19
|
constructor() {
|
|
19
|
-
this.enabled =
|
|
20
|
+
this.enabled = false;
|
|
20
21
|
this.includeSpecs = false;
|
|
22
|
+
this.include = [];
|
|
23
|
+
this.exclude = [];
|
|
21
24
|
}
|
|
22
25
|
}
|
|
23
26
|
export class Suite {
|
|
@@ -30,17 +33,20 @@ export class BuildOptions {
|
|
|
30
33
|
this.cmd = "";
|
|
31
34
|
this.args = [];
|
|
32
35
|
this.target = "wasi";
|
|
36
|
+
this.env = {};
|
|
33
37
|
}
|
|
34
38
|
}
|
|
35
39
|
export class RunOptions {
|
|
36
40
|
constructor() {
|
|
37
41
|
this.runtime = new Runtime();
|
|
38
42
|
this.reporter = "";
|
|
43
|
+
this.env = {};
|
|
39
44
|
}
|
|
40
45
|
}
|
|
41
46
|
export class Runtime {
|
|
42
47
|
constructor() {
|
|
43
48
|
this.cmd = "node ./.as-test/runners/default.wasi.js <file>";
|
|
49
|
+
this.browser = "";
|
|
44
50
|
}
|
|
45
51
|
}
|
|
46
52
|
export class ModeConfig {
|
|
@@ -58,3 +64,14 @@ export class ReporterConfig {
|
|
|
58
64
|
this.outFile = "";
|
|
59
65
|
}
|
|
60
66
|
}
|
|
67
|
+
export class FuzzConfig {
|
|
68
|
+
constructor() {
|
|
69
|
+
this.input = ["./assembly/__fuzz__/*.fuzz.ts"];
|
|
70
|
+
this.runs = 1000;
|
|
71
|
+
this.seed = 1337;
|
|
72
|
+
this.maxInputBytes = 4096;
|
|
73
|
+
this.target = "bindings";
|
|
74
|
+
this.corpusDir = "./.as-test/fuzz/corpus";
|
|
75
|
+
this.crashDir = "./.as-test/crashes";
|
|
76
|
+
}
|
|
77
|
+
}
|