as-test 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -1
- package/README.md +2 -0
- package/as-test.config.schema.json +1 -3
- package/assembly/__fuzz__/seed-perf.fuzz.ts +68 -0
- package/assembly/src/fuzz.ts +156 -55
- package/assembly/util/wipc.ts +30 -1
- package/bin/commands/fuzz-core.js +67 -3
- package/bin/commands/init-core.js +0 -1
- package/bin/commands/run-core.js +177 -4
- package/bin/index.js +2 -2
- package/bin/reporters/default.js +43 -44
- package/bin/types.js +1 -1
- package/bin/util.js +1 -1
- package/package.json +3 -1
- package/transform/lib/coverage.js +3 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
-
##
|
|
3
|
+
## 2026-04-28 - v1.0.12
|
|
4
|
+
|
|
5
|
+
- perf: faster seed generation
|
|
6
|
+
- feat: make fuzz campaigns use a random base seed by default when `fuzz.seed` and CLI seed overrides are not set, while keeping deterministic replay via `--seed` / `--fuzz-seed`.
|
|
7
|
+
|
|
8
|
+
## 2026-04-18 - v1.0.11
|
|
9
|
+
|
|
10
|
+
### Coverage
|
|
11
|
+
|
|
12
|
+
- fix: ignore additional AssemblyScript compile-time builtin calls during coverage instrumentation, including `isVector`, `isVoid`, and `lengthof`.
|
|
4
13
|
|
|
5
14
|
## 2026-04-18 - v1.0.10
|
|
6
15
|
|
package/README.md
CHANGED
|
@@ -276,6 +276,8 @@ If you used `npx ast init` with a fuzzer example, the config is already there. O
|
|
|
276
276
|
|
|
277
277
|
`ast fuzz` runs fuzz files across the selected modes, reports one result per file, and keeps the final summary separate from the normal test totals. If you want one combined command, use `ast test --fuzz`.
|
|
278
278
|
|
|
279
|
+
By default, each fuzz run campaign picks a new random base seed. Pin a seed with `--seed <n>` (or `--fuzz-seed <n>` on `ast test`) when you want deterministic replay.
|
|
280
|
+
|
|
279
281
|
When a fuzzer fails, `as-test` now prints the exact failing seeds and one-run repro commands such as `ast fuzz ... --seed <seed+n> --runs 1`. Crash records in `.as-test/crashes` also include the captured inputs passed to `run(...)`, which helps when the generator itself has side effects.
|
|
280
282
|
|
|
281
283
|
Run only fuzzers:
|
|
@@ -259,8 +259,7 @@
|
|
|
259
259
|
},
|
|
260
260
|
"seed": {
|
|
261
261
|
"type": "number",
|
|
262
|
-
"description": "
|
|
263
|
-
"default": 1337
|
|
262
|
+
"description": "Optional base seed for input mutation. If omitted, a random seed is generated each run."
|
|
264
263
|
},
|
|
265
264
|
"maxInputBytes": {
|
|
266
265
|
"type": "number",
|
|
@@ -287,7 +286,6 @@
|
|
|
287
286
|
"default": {
|
|
288
287
|
"input": ["./assembly/__fuzz__/*.fuzz.ts"],
|
|
289
288
|
"runs": 1000,
|
|
290
|
-
"seed": 1337,
|
|
291
289
|
"maxInputBytes": 4096,
|
|
292
290
|
"target": "bindings",
|
|
293
291
|
"corpusDir": "./.as-test/fuzz/corpus",
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import {
|
|
2
|
+
fuzz,
|
|
3
|
+
FuzzSeed,
|
|
4
|
+
IntegerOptions,
|
|
5
|
+
FloatOptions,
|
|
6
|
+
StringOptions,
|
|
7
|
+
BytesOptions,
|
|
8
|
+
ArrayOptions,
|
|
9
|
+
} from "as-test";
|
|
10
|
+
|
|
11
|
+
const PERF_I32 = new IntegerOptions<i32>();
|
|
12
|
+
PERF_I32.min = -100000;
|
|
13
|
+
PERF_I32.max = 100000;
|
|
14
|
+
const PERF_U32 = new IntegerOptions<u32>();
|
|
15
|
+
PERF_U32.min = 0;
|
|
16
|
+
PERF_U32.max = 1000000;
|
|
17
|
+
const PERF_F64 = new FloatOptions<f64>();
|
|
18
|
+
PERF_F64.min = -1.0;
|
|
19
|
+
PERF_F64.max = 1.0;
|
|
20
|
+
const PERF_ASCII = new StringOptions();
|
|
21
|
+
PERF_ASCII.charset = "ascii";
|
|
22
|
+
PERF_ASCII.min = 4;
|
|
23
|
+
PERF_ASCII.max = 24;
|
|
24
|
+
const PERF_BYTES = new BytesOptions();
|
|
25
|
+
PERF_BYTES.min = 8;
|
|
26
|
+
PERF_BYTES.max = 32;
|
|
27
|
+
const PERF_ARRAY = new ArrayOptions();
|
|
28
|
+
PERF_ARRAY.min = 4;
|
|
29
|
+
PERF_ARRAY.max = 16;
|
|
30
|
+
|
|
31
|
+
fuzz("seed perf i32/u32/f64", (_n: i32): bool => true).generate(
|
|
32
|
+
(seed: FuzzSeed, run: (n: i32) => bool): void => {
|
|
33
|
+
let accI: i32 = 0;
|
|
34
|
+
let accU: u32 = 0;
|
|
35
|
+
let accF: f64 = 0.0;
|
|
36
|
+
for (let i = 0; i < 128; i++) {
|
|
37
|
+
accI += seed.i32(PERF_I32);
|
|
38
|
+
accU ^= seed.u32(PERF_U32);
|
|
39
|
+
accF += seed.f64(PERF_F64);
|
|
40
|
+
}
|
|
41
|
+
run(accI + <i32>accU + <i32>accF);
|
|
42
|
+
},
|
|
43
|
+
20000,
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
fuzz("seed perf strings", (_n: i32): bool => true).generate(
|
|
47
|
+
(seed: FuzzSeed, run: (n: i32) => bool): void => {
|
|
48
|
+
let total = 0;
|
|
49
|
+
for (let i = 0; i < 96; i++) {
|
|
50
|
+
total += seed.string(PERF_ASCII).length;
|
|
51
|
+
}
|
|
52
|
+
run(total);
|
|
53
|
+
},
|
|
54
|
+
20000,
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
fuzz("seed perf bytes/array", (_n: i32): bool => true).generate(
|
|
58
|
+
(seed: FuzzSeed, run: (n: i32) => bool): void => {
|
|
59
|
+
let score = 0;
|
|
60
|
+
for (let i = 0; i < 64; i++) {
|
|
61
|
+
score += seed.bytes(PERF_BYTES).length;
|
|
62
|
+
score += seed.array<i32>((s) => s.i32({ min: -9, max: 9 }), PERF_ARRAY)
|
|
63
|
+
.length;
|
|
64
|
+
}
|
|
65
|
+
run(score);
|
|
66
|
+
},
|
|
67
|
+
20000,
|
|
68
|
+
);
|
package/assembly/src/fuzz.ts
CHANGED
|
@@ -34,8 +34,28 @@ export class ArrayOptions {
|
|
|
34
34
|
max: i32 = 16;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
const DEFAULT_I32_OPTIONS = new IntegerOptions<i32>();
|
|
38
|
+
const DEFAULT_U32_OPTIONS = new IntegerOptions<u32>();
|
|
39
|
+
const DEFAULT_F32_OPTIONS = new FloatOptions<f32>();
|
|
40
|
+
const DEFAULT_F64_OPTIONS = new FloatOptions<f64>();
|
|
41
|
+
const DEFAULT_STRING_OPTIONS = new StringOptions();
|
|
42
|
+
const DEFAULT_BYTES_OPTIONS = new BytesOptions();
|
|
43
|
+
const DEFAULT_ARRAY_OPTIONS = new ArrayOptions();
|
|
44
|
+
|
|
37
45
|
export class FuzzSeed {
|
|
38
|
-
|
|
46
|
+
private state: u32;
|
|
47
|
+
|
|
48
|
+
constructor(seed: u64) {
|
|
49
|
+
this.reseed(seed);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
reseed(seed: u64): void {
|
|
53
|
+
const lo = <u32>seed;
|
|
54
|
+
const hi = <u32>(seed >> 32);
|
|
55
|
+
let mixed = lo ^ (hi * 0x9e3779b9) ^ 0xa341316c;
|
|
56
|
+
if (mixed == 0) mixed = 0x6d2b79f5;
|
|
57
|
+
this.state = mixed;
|
|
58
|
+
}
|
|
39
59
|
|
|
40
60
|
boolean(): bool {
|
|
41
61
|
return (this.nextU32() & 1) == 1;
|
|
@@ -46,59 +66,108 @@ export class FuzzSeed {
|
|
|
46
66
|
return unchecked(values[this.nextRange(0, values.length - 1)]);
|
|
47
67
|
}
|
|
48
68
|
|
|
49
|
-
i32(options: IntegerOptions<i32> =
|
|
50
|
-
|
|
69
|
+
i32(options: IntegerOptions<i32> | null = null): i32 {
|
|
70
|
+
const config = options != null ? options : DEFAULT_I32_OPTIONS;
|
|
71
|
+
return this.nextI32InRange(config.min, config.max, config.exclude);
|
|
51
72
|
}
|
|
52
73
|
|
|
53
|
-
u32(options: IntegerOptions<u32> =
|
|
54
|
-
|
|
74
|
+
u32(options: IntegerOptions<u32> | null = null): u32 {
|
|
75
|
+
const config = options != null ? options : DEFAULT_U32_OPTIONS;
|
|
76
|
+
return this.nextU32InRange(config.min, config.max, config.exclude);
|
|
55
77
|
}
|
|
56
78
|
|
|
57
|
-
f32(options: FloatOptions<f32> =
|
|
79
|
+
f32(options: FloatOptions<f32> | null = null): f32 {
|
|
80
|
+
const config = options != null ? options : DEFAULT_F32_OPTIONS;
|
|
58
81
|
return <f32>(
|
|
59
|
-
this.nextF64InRange<f32>(
|
|
82
|
+
this.nextF64InRange<f32>(config.min, config.max, config.exclude)
|
|
60
83
|
);
|
|
61
84
|
}
|
|
62
85
|
|
|
63
|
-
f64(options: FloatOptions<f64> =
|
|
64
|
-
|
|
86
|
+
f64(options: FloatOptions<f64> | null = null): f64 {
|
|
87
|
+
const config = options != null ? options : DEFAULT_F64_OPTIONS;
|
|
88
|
+
return this.nextF64InRange<f64>(config.min, config.max, config.exclude);
|
|
65
89
|
}
|
|
66
90
|
|
|
67
|
-
bytes(options: BytesOptions =
|
|
68
|
-
|
|
69
|
-
|
|
91
|
+
bytes(options: BytesOptions | null = null): Uint8Array {
|
|
92
|
+
const config = options != null ? options : DEFAULT_BYTES_OPTIONS;
|
|
93
|
+
validateLengthRange("seed.bytes()", config.min, config.max);
|
|
94
|
+
const length = this.nextRange(config.min, config.max);
|
|
70
95
|
const out = new Uint8Array(length);
|
|
96
|
+
const include = config.include;
|
|
97
|
+
const exclude = config.exclude;
|
|
98
|
+
if (include.length) {
|
|
99
|
+
if (!exclude.length) {
|
|
100
|
+
for (let i = 0; i < length; i++) {
|
|
101
|
+
unchecked(
|
|
102
|
+
(out[i] = <u8>unchecked(include[this.nextRange(0, include.length - 1)])),
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
return out;
|
|
106
|
+
}
|
|
107
|
+
for (let i = 0; i < length; i++) {
|
|
108
|
+
unchecked((out[i] = this.byteFromOptions(config)));
|
|
109
|
+
}
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
if (!exclude.length) {
|
|
113
|
+
for (let i = 0; i < length; i++) {
|
|
114
|
+
unchecked((out[i] = <u8>(this.nextU32() & 0xff)));
|
|
115
|
+
}
|
|
116
|
+
return out;
|
|
117
|
+
}
|
|
71
118
|
for (let i = 0; i < length; i++) {
|
|
72
|
-
out[i] = this.byteFromOptions(
|
|
119
|
+
unchecked((out[i] = this.byteFromOptions(config)));
|
|
73
120
|
}
|
|
74
121
|
return out;
|
|
75
122
|
}
|
|
76
123
|
|
|
77
|
-
buffer(options: BytesOptions =
|
|
124
|
+
buffer(options: BytesOptions | null = null): ArrayBuffer {
|
|
78
125
|
return this.bytes(options).buffer;
|
|
79
126
|
}
|
|
80
127
|
|
|
81
|
-
string(options: StringOptions =
|
|
82
|
-
|
|
83
|
-
|
|
128
|
+
string(options: StringOptions | null = null): string {
|
|
129
|
+
const config = options != null ? options : DEFAULT_STRING_OPTIONS;
|
|
130
|
+
validateLengthRange("seed.string()", config.min, config.max);
|
|
131
|
+
const alphabet = buildAlphabet(config);
|
|
84
132
|
if (!alphabet.length) {
|
|
85
133
|
panic();
|
|
86
134
|
}
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
135
|
+
const coreLength = this.nextRange(config.min, config.max);
|
|
136
|
+
const prefixLength = config.prefix.length;
|
|
137
|
+
const suffixLength = config.suffix.length;
|
|
138
|
+
const totalLength = prefixLength + coreLength + suffixLength;
|
|
139
|
+
if (!totalLength) return "";
|
|
140
|
+
|
|
141
|
+
// Allocate UTF-16 payload directly and fill code units in one pass.
|
|
142
|
+
const outPtr = __new(<usize>(totalLength << 1), idof<string>());
|
|
143
|
+
let cursor: usize = outPtr;
|
|
144
|
+
|
|
145
|
+
for (let i = 0; i < prefixLength; i++) {
|
|
146
|
+
store<u16>(cursor, <u16>config.prefix.charCodeAt(i));
|
|
147
|
+
cursor += 2;
|
|
91
148
|
}
|
|
92
|
-
|
|
93
|
-
|
|
149
|
+
|
|
150
|
+
const last = alphabet.length - 1;
|
|
151
|
+
for (let i = 0; i < coreLength; i++) {
|
|
152
|
+
store<u16>(cursor, <u16>unchecked(alphabet[this.nextRange(0, last)]));
|
|
153
|
+
cursor += 2;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
for (let i = 0; i < suffixLength; i++) {
|
|
157
|
+
store<u16>(cursor, <u16>config.suffix.charCodeAt(i));
|
|
158
|
+
cursor += 2;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return changetype<string>(outPtr);
|
|
94
162
|
}
|
|
95
163
|
|
|
96
164
|
array<T>(
|
|
97
165
|
item: (seed: FuzzSeed) => T,
|
|
98
|
-
options: ArrayOptions =
|
|
166
|
+
options: ArrayOptions | null = null,
|
|
99
167
|
): Array<T> {
|
|
100
|
-
|
|
101
|
-
|
|
168
|
+
const config = options != null ? options : DEFAULT_ARRAY_OPTIONS;
|
|
169
|
+
validateLengthRange("seed.array()", config.min, config.max);
|
|
170
|
+
const length = this.nextRange(config.min, config.max);
|
|
102
171
|
const out = new Array<T>(length);
|
|
103
172
|
for (let i = 0; i < length; i++) {
|
|
104
173
|
unchecked((out[i] = item(this)));
|
|
@@ -110,17 +179,23 @@ export class FuzzSeed {
|
|
|
110
179
|
const include = options.include;
|
|
111
180
|
const exclude = options.exclude;
|
|
112
181
|
if (include.length) {
|
|
182
|
+
if (!exclude.length) {
|
|
183
|
+
return <u8>unchecked(include[this.nextRange(0, include.length - 1)]);
|
|
184
|
+
}
|
|
113
185
|
for (let attempts = 0; attempts < 1024; attempts++) {
|
|
114
186
|
const picked = unchecked(
|
|
115
187
|
include[this.nextRange(0, include.length - 1)],
|
|
116
188
|
);
|
|
117
|
-
if (!exclude
|
|
189
|
+
if (!containsValue<u8>(exclude, picked)) return picked;
|
|
118
190
|
}
|
|
119
191
|
panic();
|
|
120
192
|
}
|
|
193
|
+
if (!exclude.length) {
|
|
194
|
+
return <u8>(this.nextU32() & 0xff);
|
|
195
|
+
}
|
|
121
196
|
for (let attempts = 0; attempts < 1024; attempts++) {
|
|
122
197
|
const value = <u8>(this.nextU32() & 0xff);
|
|
123
|
-
if (!exclude
|
|
198
|
+
if (!containsValue<u8>(exclude, value)) return value;
|
|
124
199
|
}
|
|
125
200
|
panic();
|
|
126
201
|
return 0;
|
|
@@ -128,6 +203,9 @@ export class FuzzSeed {
|
|
|
128
203
|
|
|
129
204
|
private nextI32InRange(min: i32, max: i32, exclude: i32[]): i32 {
|
|
130
205
|
if (max < min) panic();
|
|
206
|
+
if (!exclude.length) {
|
|
207
|
+
return max <= min ? min : min + <i32>(this.nextU32() % <u32>(max - min + 1));
|
|
208
|
+
}
|
|
131
209
|
for (let attempts = 0; attempts < 1024; attempts++) {
|
|
132
210
|
const value =
|
|
133
211
|
max <= min ? min : min + <i32>(this.nextU32() % <u32>(max - min + 1));
|
|
@@ -139,6 +217,9 @@ export class FuzzSeed {
|
|
|
139
217
|
|
|
140
218
|
private nextU32InRange(min: u32, max: u32, exclude: u32[]): u32 {
|
|
141
219
|
if (max < min) panic();
|
|
220
|
+
if (!exclude.length) {
|
|
221
|
+
return max <= min ? min : min + (this.nextU32() % (max - min + 1));
|
|
222
|
+
}
|
|
142
223
|
for (let attempts = 0; attempts < 1024; attempts++) {
|
|
143
224
|
const value = max <= min ? min : min + (this.nextU32() % (max - min + 1));
|
|
144
225
|
if (!containsValue<u32>(exclude, value)) return value;
|
|
@@ -151,6 +232,9 @@ export class FuzzSeed {
|
|
|
151
232
|
const left = <f64>min;
|
|
152
233
|
const right = <f64>max;
|
|
153
234
|
if (right < left) panic();
|
|
235
|
+
if (!exclude.length) {
|
|
236
|
+
return left + (right - left) * this.nextUnit();
|
|
237
|
+
}
|
|
154
238
|
for (let attempts = 0; attempts < 1024; attempts++) {
|
|
155
239
|
const value = left + (right - left) * this.nextUnit();
|
|
156
240
|
if (!containsFloatValue<T>(exclude, changetype<T>(value))) return value;
|
|
@@ -169,11 +253,14 @@ export class FuzzSeed {
|
|
|
169
253
|
}
|
|
170
254
|
|
|
171
255
|
private nextU32(): u32 {
|
|
172
|
-
|
|
173
|
-
let
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
256
|
+
// mulberry32: very fast integer-only PRNG suitable for fuzz input generation.
|
|
257
|
+
let x = this.state;
|
|
258
|
+
x += 0x6d2b79f5;
|
|
259
|
+
this.state = x;
|
|
260
|
+
let z = x;
|
|
261
|
+
z = <u32>Math.imul(z ^ (z >> 15), z | 1);
|
|
262
|
+
z ^= z + <u32>Math.imul(z ^ (z >> 7), z | 61);
|
|
263
|
+
return z ^ (z >> 14);
|
|
177
264
|
}
|
|
178
265
|
|
|
179
266
|
private nextU64(): u64 {
|
|
@@ -183,6 +270,21 @@ export class FuzzSeed {
|
|
|
183
270
|
}
|
|
184
271
|
}
|
|
185
272
|
|
|
273
|
+
const ASCII_ALPHABET: i32[] = rangeChars(32, 126);
|
|
274
|
+
const ALPHA_ALPHABET: i32[] = rangeChars(65, 90).concat(rangeChars(97, 122));
|
|
275
|
+
const DIGIT_ALPHABET: i32[] = rangeChars(48, 57);
|
|
276
|
+
const HEX_ALPHABET: i32[] = DIGIT_ALPHABET.concat(rangeChars(97, 102));
|
|
277
|
+
const ALNUM_ALPHABET: i32[] = ALPHA_ALPHABET.concat(DIGIT_ALPHABET);
|
|
278
|
+
const BASE64_ALPHABET: i32[] = ALPHA_ALPHABET.concat(DIGIT_ALPHABET).concat([
|
|
279
|
+
43,
|
|
280
|
+
47,
|
|
281
|
+
61,
|
|
282
|
+
]);
|
|
283
|
+
const IDENTIFIER_ALPHABET: i32[] = [95].concat(ALPHA_ALPHABET).concat(
|
|
284
|
+
DIGIT_ALPHABET,
|
|
285
|
+
);
|
|
286
|
+
const WHITESPACE_ALPHABET: i32[] = [9, 10, 13, 32];
|
|
287
|
+
|
|
186
288
|
export abstract class FuzzerBase {
|
|
187
289
|
public name: string;
|
|
188
290
|
public skipped: bool;
|
|
@@ -409,12 +511,13 @@ export class Fuzzer0<R> extends FuzzerBase {
|
|
|
409
511
|
run(seedBase: u64, runs: i32): FuzzerResult {
|
|
410
512
|
if (this.skipped) return createSkippedResult(this.name);
|
|
411
513
|
const result = createResult(this.name, runs);
|
|
514
|
+
const seed = new FuzzSeed(seedBase);
|
|
412
515
|
__fuzz_callback0 = changetype<() => usize>(this.callback);
|
|
413
516
|
__fuzz_returns_bool = this.returnsBool;
|
|
414
517
|
for (let i = 0; i < runs; i++) {
|
|
415
518
|
prepareFuzzIteration();
|
|
416
519
|
__fuzz_calls = 0;
|
|
417
|
-
|
|
520
|
+
seed.reseed(seedBase + <u64>i);
|
|
418
521
|
if (this.generator) {
|
|
419
522
|
this.generator(seed, changetype<() => R>(__fuzz_run0));
|
|
420
523
|
} else {
|
|
@@ -469,12 +572,13 @@ export class Fuzzer1<A, R> extends FuzzerBase {
|
|
|
469
572
|
run(seedBase: u64, runs: i32): FuzzerResult {
|
|
470
573
|
if (this.skipped) return createSkippedResult(this.name);
|
|
471
574
|
const result = createResult(this.name, runs);
|
|
575
|
+
const seed = new FuzzSeed(seedBase);
|
|
472
576
|
__fuzz_callback1 = changetype<(a: usize) => usize>(this.callback);
|
|
473
577
|
__fuzz_returns_bool = this.returnsBool;
|
|
474
578
|
for (let i = 0; i < runs; i++) {
|
|
475
579
|
prepareFuzzIteration();
|
|
476
580
|
__fuzz_calls = 0;
|
|
477
|
-
|
|
581
|
+
seed.reseed(seedBase + <u64>i);
|
|
478
582
|
if (!this.generator) {
|
|
479
583
|
failFuzzIteration(
|
|
480
584
|
"generate",
|
|
@@ -538,12 +642,13 @@ export class Fuzzer2<A, B, R> extends FuzzerBase {
|
|
|
538
642
|
run(seedBase: u64, runs: i32): FuzzerResult {
|
|
539
643
|
if (this.skipped) return createSkippedResult(this.name);
|
|
540
644
|
const result = createResult(this.name, runs);
|
|
645
|
+
const seed = new FuzzSeed(seedBase);
|
|
541
646
|
__fuzz_callback2 = changetype<(a: usize, b: usize) => usize>(this.callback);
|
|
542
647
|
__fuzz_returns_bool = this.returnsBool;
|
|
543
648
|
for (let i = 0; i < runs; i++) {
|
|
544
649
|
prepareFuzzIteration();
|
|
545
650
|
__fuzz_calls = 0;
|
|
546
|
-
|
|
651
|
+
seed.reseed(seedBase + <u64>i);
|
|
547
652
|
if (!this.generator) {
|
|
548
653
|
failFuzzIteration(
|
|
549
654
|
"generate",
|
|
@@ -606,6 +711,7 @@ export class Fuzzer3<A, B, C, R> extends FuzzerBase {
|
|
|
606
711
|
run(seedBase: u64, runs: i32): FuzzerResult {
|
|
607
712
|
if (this.skipped) return createSkippedResult(this.name);
|
|
608
713
|
const result = createResult(this.name, runs);
|
|
714
|
+
const seed = new FuzzSeed(seedBase);
|
|
609
715
|
__fuzz_callback3 = changetype<(a: usize, b: usize, c: usize) => usize>(
|
|
610
716
|
this.callback,
|
|
611
717
|
);
|
|
@@ -613,7 +719,7 @@ export class Fuzzer3<A, B, C, R> extends FuzzerBase {
|
|
|
613
719
|
for (let i = 0; i < runs; i++) {
|
|
614
720
|
prepareFuzzIteration();
|
|
615
721
|
__fuzz_calls = 0;
|
|
616
|
-
|
|
722
|
+
seed.reseed(seedBase + <u64>i);
|
|
617
723
|
if (!this.generator) {
|
|
618
724
|
failFuzzIteration(
|
|
619
725
|
"generate",
|
|
@@ -701,13 +807,13 @@ export function createFuzzer<T extends Function>(
|
|
|
701
807
|
}
|
|
702
808
|
|
|
703
809
|
function buildAlphabet(options: StringOptions): i32[] {
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
out.length = 0;
|
|
810
|
+
if (!options.include.length && !options.exclude.length) {
|
|
811
|
+
return baseAlphabet(options.charset);
|
|
707
812
|
}
|
|
813
|
+
const out = baseAlphabet(options.charset).slice(0);
|
|
708
814
|
for (let i = 0; i < options.include.length; i++) {
|
|
709
815
|
const value = unchecked(options.include[i]);
|
|
710
|
-
if (!out
|
|
816
|
+
if (!containsValue<i32>(out, value)) out.push(value);
|
|
711
817
|
}
|
|
712
818
|
for (let i = 0; i < options.exclude.length; i++) {
|
|
713
819
|
removeFirst(out, unchecked(options.exclude[i]));
|
|
@@ -716,21 +822,15 @@ function buildAlphabet(options: StringOptions): i32[] {
|
|
|
716
822
|
}
|
|
717
823
|
|
|
718
824
|
function baseAlphabet(charset: string): i32[] {
|
|
719
|
-
if (charset == "alpha") return
|
|
720
|
-
if (charset == "alnum")
|
|
721
|
-
|
|
722
|
-
if (charset == "
|
|
723
|
-
if (charset == "
|
|
724
|
-
if (charset == "
|
|
725
|
-
|
|
726
|
-
.concat(rangeChars(97, 122))
|
|
727
|
-
.concat(rangeChars(48, 57))
|
|
728
|
-
.concat([43, 47, 61]);
|
|
729
|
-
if (charset == "identifier")
|
|
730
|
-
return [95].concat(baseAlphabet("alpha")).concat(rangeChars(48, 57));
|
|
731
|
-
if (charset == "whitespace") return [9, 10, 13, 32];
|
|
825
|
+
if (charset == "alpha") return ALPHA_ALPHABET;
|
|
826
|
+
if (charset == "alnum") return ALNUM_ALPHABET;
|
|
827
|
+
if (charset == "digit") return DIGIT_ALPHABET;
|
|
828
|
+
if (charset == "hex") return HEX_ALPHABET;
|
|
829
|
+
if (charset == "base64") return BASE64_ALPHABET;
|
|
830
|
+
if (charset == "identifier") return IDENTIFIER_ALPHABET;
|
|
831
|
+
if (charset == "whitespace") return WHITESPACE_ALPHABET;
|
|
732
832
|
if (charset == "custom") return [];
|
|
733
|
-
return
|
|
833
|
+
return ASCII_ALPHABET;
|
|
734
834
|
}
|
|
735
835
|
|
|
736
836
|
function rangeChars(start: i32, end: i32): i32[] {
|
|
@@ -746,6 +846,7 @@ function removeFirst(values: i32[], needle: i32): void {
|
|
|
746
846
|
if (index >= 0) values.splice(index, 1);
|
|
747
847
|
}
|
|
748
848
|
|
|
849
|
+
|
|
749
850
|
function validateLengthRange(label: string, min: i32, max: i32): void {
|
|
750
851
|
if (min < 0 || max < 0) panic();
|
|
751
852
|
if (max < min) panic();
|
package/assembly/util/wipc.ts
CHANGED
|
@@ -33,6 +33,7 @@ const MAGIC_C: u8 = 0x43; // C
|
|
|
33
33
|
const HEADER_SIZE: i32 = 9;
|
|
34
34
|
const IOV_SIZE: usize = sizeof<usize>() * 2;
|
|
35
35
|
const U32_SIZE: usize = sizeof<u32>();
|
|
36
|
+
const REPORT_CHUNK_BYTES: i32 = 65536;
|
|
36
37
|
|
|
37
38
|
// @ts-ignore
|
|
38
39
|
const IS_BINDINGS: bool = isDefined(AS_TEST_BINDINGS);
|
|
@@ -162,7 +163,35 @@ export function requestFuzzConfig(): FuzzConfigReply {
|
|
|
162
163
|
}
|
|
163
164
|
|
|
164
165
|
export function sendReport(report: string): void {
|
|
165
|
-
|
|
166
|
+
const payload = String.UTF8.encode(report);
|
|
167
|
+
if (payload.byteLength <= REPORT_CHUNK_BYTES) {
|
|
168
|
+
sendFrame(MessageType.DATA, payload);
|
|
169
|
+
sendFrame(MessageType.CLOSE, new ArrayBuffer(0));
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const totalBytes = payload.byteLength;
|
|
174
|
+
const chunkCount = (totalBytes + REPORT_CHUNK_BYTES - 1) / REPORT_CHUNK_BYTES;
|
|
175
|
+
sendJson(
|
|
176
|
+
MessageType.CALL,
|
|
177
|
+
`{"kind":"report:start","encoding":"utf8-chunks","totalBytes":${totalBytes.toString()},"chunkCount":${chunkCount.toString()},"chunkBytes":${REPORT_CHUNK_BYTES.toString()}}`,
|
|
178
|
+
);
|
|
179
|
+
|
|
180
|
+
const ptr = changetype<usize>(payload);
|
|
181
|
+
let offset = 0;
|
|
182
|
+
while (offset < totalBytes) {
|
|
183
|
+
const size = min<i32>(REPORT_CHUNK_BYTES, totalBytes - offset);
|
|
184
|
+
const chunk = new ArrayBuffer(size);
|
|
185
|
+
memory.copy(changetype<usize>(chunk), ptr + offset, size);
|
|
186
|
+
sendFrame(MessageType.DATA, chunk);
|
|
187
|
+
offset += size;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
sendJson(
|
|
191
|
+
MessageType.CALL,
|
|
192
|
+
`{"kind":"report:end","totalBytes":${totalBytes.toString()},"chunkCount":${chunkCount.toString()}}`,
|
|
193
|
+
);
|
|
194
|
+
sendFrame(MessageType.CLOSE, new ArrayBuffer(0));
|
|
166
195
|
}
|
|
167
196
|
|
|
168
197
|
export function sendWarning(message: string): void {
|
|
@@ -8,6 +8,7 @@ import { persistCrashRecord } from "../crash-store.js";
|
|
|
8
8
|
const DEFAULT_CONFIG_PATH = path.join(process.cwd(), "./as-test.config.json");
|
|
9
9
|
const MAGIC = Buffer.from("WIPC");
|
|
10
10
|
const HEADER_SIZE = 9;
|
|
11
|
+
const MAX_DEFAULT_SEED = 0x7fffffff;
|
|
11
12
|
export async function fuzz(configPath = DEFAULT_CONFIG_PATH, selectors = [], modeName, overrides = {}) {
|
|
12
13
|
const loadedConfig = loadConfig(configPath, false);
|
|
13
14
|
const mode = applyMode(loadedConfig, modeName);
|
|
@@ -33,6 +34,9 @@ function resolveFuzzConfig(raw, overrides) {
|
|
|
33
34
|
if (typeof overrides.seed == "number") {
|
|
34
35
|
config.seed = overrides.seed;
|
|
35
36
|
}
|
|
37
|
+
else if (config.seed < 0) {
|
|
38
|
+
config.seed = generateRandomSeed();
|
|
39
|
+
}
|
|
36
40
|
if (typeof overrides.runs == "number") {
|
|
37
41
|
config.runs = overrides.runs;
|
|
38
42
|
}
|
|
@@ -50,6 +54,9 @@ function resolveFuzzConfig(raw, overrides) {
|
|
|
50
54
|
}
|
|
51
55
|
return config;
|
|
52
56
|
}
|
|
57
|
+
function generateRandomSeed() {
|
|
58
|
+
return Math.floor(Math.random() * (MAX_DEFAULT_SEED + 1));
|
|
59
|
+
}
|
|
53
60
|
function encodeRunsOverrideKind(kind) {
|
|
54
61
|
switch (kind) {
|
|
55
62
|
case "set":
|
|
@@ -71,20 +78,56 @@ async function runFuzzTarget(file, outDir, duplicateBasenames, config, buildStar
|
|
|
71
78
|
const binary = readFileSync(wasmPath);
|
|
72
79
|
const module = new WebAssembly.Module(binary);
|
|
73
80
|
let report = null;
|
|
81
|
+
let reportParseError = null;
|
|
82
|
+
const reportStream = {
|
|
83
|
+
sawChunkStart: false,
|
|
84
|
+
sawChunkEnd: false,
|
|
85
|
+
chunkCountExpected: 0,
|
|
86
|
+
chunkTotalBytesExpected: 0,
|
|
87
|
+
chunkFramesReceived: 0,
|
|
88
|
+
chunkBytesReceived: 0,
|
|
89
|
+
chunks: [],
|
|
90
|
+
};
|
|
74
91
|
const captured = captureFrames((type, payload, respond) => {
|
|
75
92
|
if (type == 0x02) {
|
|
76
93
|
const event = JSON.parse(payload.toString("utf8"));
|
|
77
|
-
|
|
94
|
+
const kind = String(event.kind ?? "");
|
|
95
|
+
if (kind == "fuzz:config") {
|
|
78
96
|
const resolved = config;
|
|
79
97
|
respond(`${config.runs}\n${config.seed}\n${resolved.runsOverrideKind ?? 0}\n${resolved.runsOverrideValue ?? 0}`);
|
|
80
98
|
}
|
|
99
|
+
else if (kind == "report:start") {
|
|
100
|
+
reportStream.sawChunkStart = true;
|
|
101
|
+
reportStream.sawChunkEnd = false;
|
|
102
|
+
reportStream.chunkCountExpected = Number(event.chunkCount ?? 0);
|
|
103
|
+
reportStream.chunkTotalBytesExpected = Number(event.totalBytes ?? 0);
|
|
104
|
+
reportStream.chunkFramesReceived = 0;
|
|
105
|
+
reportStream.chunkBytesReceived = 0;
|
|
106
|
+
reportStream.chunks = [];
|
|
107
|
+
}
|
|
108
|
+
else if (kind == "report:end") {
|
|
109
|
+
reportStream.sawChunkEnd = true;
|
|
110
|
+
}
|
|
81
111
|
else {
|
|
82
112
|
respond("");
|
|
83
113
|
}
|
|
84
114
|
return;
|
|
85
115
|
}
|
|
86
116
|
if (type == 0x03) {
|
|
87
|
-
|
|
117
|
+
if (reportStream.sawChunkStart && !reportStream.sawChunkEnd) {
|
|
118
|
+
reportStream.chunkFramesReceived++;
|
|
119
|
+
reportStream.chunkBytesReceived += payload.length;
|
|
120
|
+
reportStream.chunks.push(payload.toString("utf8"));
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
try {
|
|
124
|
+
report = JSON.parse(payload.toString("utf8"));
|
|
125
|
+
reportParseError = null;
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
reportParseError = String(error);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
88
131
|
}
|
|
89
132
|
});
|
|
90
133
|
try {
|
|
@@ -119,14 +162,35 @@ async function runFuzzTarget(file, outDir, duplicateBasenames, config, buildStar
|
|
|
119
162
|
};
|
|
120
163
|
}
|
|
121
164
|
const passthrough = captured.restore();
|
|
165
|
+
if (reportStream.sawChunkStart) {
|
|
166
|
+
if (reportStream.sawChunkEnd) {
|
|
167
|
+
const chunkedPayload = reportStream.chunks.join("");
|
|
168
|
+
try {
|
|
169
|
+
report = JSON.parse(chunkedPayload);
|
|
170
|
+
reportParseError = null;
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
reportParseError = String(error);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
122
177
|
if (!report?.fuzzers) {
|
|
178
|
+
const diagnostics = [
|
|
179
|
+
`chunked=${reportStream.sawChunkStart ? "yes" : "no"}`,
|
|
180
|
+
`chunkStart=${reportStream.sawChunkStart ? "yes" : "no"}`,
|
|
181
|
+
`chunkEnd=${reportStream.sawChunkEnd ? "yes" : "no"}`,
|
|
182
|
+
`chunkFrames=${reportStream.chunkFramesReceived}`,
|
|
183
|
+
`expectedChunkFrames=${reportStream.chunkCountExpected}`,
|
|
184
|
+
`chunkBytes=${reportStream.chunkBytesReceived}`,
|
|
185
|
+
`expectedChunkBytes=${reportStream.chunkTotalBytesExpected}`,
|
|
186
|
+
].join(", ");
|
|
123
187
|
const crash = persistCrashRecord(config.crashDir, {
|
|
124
188
|
kind: "fuzz",
|
|
125
189
|
file,
|
|
126
190
|
entryKey: buildFuzzCrashEntryKey(file, modeName ?? "default"),
|
|
127
191
|
mode: modeName ?? "default",
|
|
128
192
|
seed: config.seed,
|
|
129
|
-
error: `missing fuzz report payload from ${path.basename(file)}`,
|
|
193
|
+
error: `${reportParseError ? `invalid fuzz report payload: ${reportParseError}` : `missing fuzz report payload from ${path.basename(file)}`} (${diagnostics})`,
|
|
130
194
|
stdout: passthrough.stdout,
|
|
131
195
|
stderr: "",
|
|
132
196
|
});
|
package/bin/commands/run-core.js
CHANGED
|
@@ -1483,6 +1483,31 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1483
1483
|
let stderrPendingLine = "";
|
|
1484
1484
|
let stdoutBuffer = "";
|
|
1485
1485
|
let spawnError = null;
|
|
1486
|
+
let sawChannelClose = false;
|
|
1487
|
+
const runtimeEvents = {
|
|
1488
|
+
sawFileStart: false,
|
|
1489
|
+
sawFileEnd: false,
|
|
1490
|
+
fileName: path.basename(specFile),
|
|
1491
|
+
fileVerdict: "none",
|
|
1492
|
+
fileTime: "",
|
|
1493
|
+
suiteStarts: 0,
|
|
1494
|
+
suiteEnds: 0,
|
|
1495
|
+
assertionFails: 0,
|
|
1496
|
+
warnings: 0,
|
|
1497
|
+
logs: 0,
|
|
1498
|
+
};
|
|
1499
|
+
const reportStream = {
|
|
1500
|
+
dataFrames: 0,
|
|
1501
|
+
dataBytes: 0,
|
|
1502
|
+
sawChunkStart: false,
|
|
1503
|
+
sawChunkEnd: false,
|
|
1504
|
+
chunkCountExpected: 0,
|
|
1505
|
+
chunkBytesExpected: 0,
|
|
1506
|
+
chunkTotalBytesExpected: 0,
|
|
1507
|
+
chunkFramesReceived: 0,
|
|
1508
|
+
chunkBytesReceived: 0,
|
|
1509
|
+
chunks: [],
|
|
1510
|
+
};
|
|
1486
1511
|
child.on("error", (error) => {
|
|
1487
1512
|
spawnError = error;
|
|
1488
1513
|
});
|
|
@@ -1512,6 +1537,7 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1512
1537
|
const event = msg;
|
|
1513
1538
|
const kind = String(event.kind ?? "");
|
|
1514
1539
|
if (kind === "event:assert-fail") {
|
|
1540
|
+
runtimeEvents.assertionFails++;
|
|
1515
1541
|
reporter.onAssertionFail?.({
|
|
1516
1542
|
key: String(event.key ?? ""),
|
|
1517
1543
|
instr: String(event.instr ?? ""),
|
|
@@ -1522,6 +1548,8 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1522
1548
|
return;
|
|
1523
1549
|
}
|
|
1524
1550
|
if (kind === "event:file-start") {
|
|
1551
|
+
runtimeEvents.sawFileStart = true;
|
|
1552
|
+
runtimeEvents.fileName = String(event.file ?? runtimeEvents.fileName);
|
|
1525
1553
|
reporter.onFileStart?.({
|
|
1526
1554
|
file: String(event.file ?? "unknown"),
|
|
1527
1555
|
depth: 0,
|
|
@@ -1531,6 +1559,10 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1531
1559
|
return;
|
|
1532
1560
|
}
|
|
1533
1561
|
if (kind === "event:file-end") {
|
|
1562
|
+
runtimeEvents.sawFileEnd = true;
|
|
1563
|
+
runtimeEvents.fileName = String(event.file ?? runtimeEvents.fileName);
|
|
1564
|
+
runtimeEvents.fileVerdict = String(event.verdict ?? "none");
|
|
1565
|
+
runtimeEvents.fileTime = String(event.time ?? "");
|
|
1534
1566
|
reporter.onFileEnd?.({
|
|
1535
1567
|
file: String(event.file ?? "unknown"),
|
|
1536
1568
|
depth: 0,
|
|
@@ -1542,6 +1574,7 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1542
1574
|
return;
|
|
1543
1575
|
}
|
|
1544
1576
|
if (kind === "event:suite-start") {
|
|
1577
|
+
runtimeEvents.suiteStarts++;
|
|
1545
1578
|
reporter.onSuiteStart?.({
|
|
1546
1579
|
file: String(event.file ?? "unknown"),
|
|
1547
1580
|
depth: Number(event.depth ?? 0),
|
|
@@ -1551,6 +1584,7 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1551
1584
|
return;
|
|
1552
1585
|
}
|
|
1553
1586
|
if (kind === "event:suite-end") {
|
|
1587
|
+
runtimeEvents.suiteEnds++;
|
|
1554
1588
|
reporter.onSuiteEnd?.({
|
|
1555
1589
|
file: String(event.file ?? "unknown"),
|
|
1556
1590
|
depth: Number(event.depth ?? 0),
|
|
@@ -1561,12 +1595,14 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1561
1595
|
return;
|
|
1562
1596
|
}
|
|
1563
1597
|
if (kind === "event:warn") {
|
|
1598
|
+
runtimeEvents.warnings++;
|
|
1564
1599
|
reporter.onWarning?.({
|
|
1565
1600
|
message: String(event.message ?? ""),
|
|
1566
1601
|
});
|
|
1567
1602
|
return;
|
|
1568
1603
|
}
|
|
1569
1604
|
if (kind === "event:log") {
|
|
1605
|
+
runtimeEvents.logs++;
|
|
1570
1606
|
reporter.onLog?.({
|
|
1571
1607
|
file: String(event.file ?? "unknown"),
|
|
1572
1608
|
depth: Number(event.depth ?? 0),
|
|
@@ -1584,16 +1620,43 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1584
1620
|
this.send(MessageType.CALL, Buffer.from(`${result.ok ? "1" : "0"}\n${result.expected}`, "utf8"));
|
|
1585
1621
|
return;
|
|
1586
1622
|
}
|
|
1623
|
+
if (kind === "report:start") {
|
|
1624
|
+
reportStream.sawChunkStart = true;
|
|
1625
|
+
reportStream.sawChunkEnd = false;
|
|
1626
|
+
reportStream.chunkCountExpected = Number(event.chunkCount ?? 0);
|
|
1627
|
+
reportStream.chunkBytesExpected = Number(event.chunkBytes ?? 0);
|
|
1628
|
+
reportStream.chunkTotalBytesExpected = Number(event.totalBytes ?? 0);
|
|
1629
|
+
reportStream.chunkFramesReceived = 0;
|
|
1630
|
+
reportStream.chunkBytesReceived = 0;
|
|
1631
|
+
reportStream.chunks = [];
|
|
1632
|
+
return;
|
|
1633
|
+
}
|
|
1634
|
+
if (kind === "report:end") {
|
|
1635
|
+
reportStream.sawChunkEnd = true;
|
|
1636
|
+
return;
|
|
1637
|
+
}
|
|
1587
1638
|
this.sendJSON(MessageType.CALL, { ok: true, expected: "" });
|
|
1588
1639
|
}
|
|
1589
1640
|
onDataMessage(data) {
|
|
1641
|
+
reportStream.dataFrames++;
|
|
1642
|
+
reportStream.dataBytes += data.length;
|
|
1643
|
+
if (reportStream.sawChunkStart && !reportStream.sawChunkEnd) {
|
|
1644
|
+
reportStream.chunkFramesReceived++;
|
|
1645
|
+
reportStream.chunkBytesReceived += data.length;
|
|
1646
|
+
reportStream.chunks.push(data.toString("utf8"));
|
|
1647
|
+
return;
|
|
1648
|
+
}
|
|
1590
1649
|
try {
|
|
1591
1650
|
report = JSON.parse(data.toString("utf8"));
|
|
1651
|
+
parseError = null;
|
|
1592
1652
|
}
|
|
1593
1653
|
catch (error) {
|
|
1594
1654
|
parseError = String(error);
|
|
1595
1655
|
}
|
|
1596
1656
|
}
|
|
1657
|
+
onClose() {
|
|
1658
|
+
sawChannelClose = true;
|
|
1659
|
+
}
|
|
1597
1660
|
}
|
|
1598
1661
|
const _channel = new TestChannel(child.stdout, child.stdin);
|
|
1599
1662
|
const code = await new Promise((resolve) => {
|
|
@@ -1614,29 +1677,93 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1614
1677
|
});
|
|
1615
1678
|
return createRuntimeFailureReport(specFile, modeName, "failed to start test runtime", errorText, stdoutBuffer, stderrBuffer);
|
|
1616
1679
|
}
|
|
1680
|
+
if (reportStream.sawChunkStart) {
|
|
1681
|
+
if (!reportStream.sawChunkEnd) {
|
|
1682
|
+
parseError =
|
|
1683
|
+
parseError ??
|
|
1684
|
+
"missing report:end marker for chunked report payload";
|
|
1685
|
+
}
|
|
1686
|
+
else {
|
|
1687
|
+
const chunkedPayload = reportStream.chunks.join("");
|
|
1688
|
+
try {
|
|
1689
|
+
report = JSON.parse(chunkedPayload);
|
|
1690
|
+
parseError = null;
|
|
1691
|
+
}
|
|
1692
|
+
catch (error) {
|
|
1693
|
+
parseError = `could not parse chunked report payload: ${String(error)}`;
|
|
1694
|
+
}
|
|
1695
|
+
if (reportStream.chunkCountExpected > 0 &&
|
|
1696
|
+
reportStream.chunkFramesReceived !== reportStream.chunkCountExpected) {
|
|
1697
|
+
parseError =
|
|
1698
|
+
parseError ??
|
|
1699
|
+
`chunk count mismatch: expected ${reportStream.chunkCountExpected}, received ${reportStream.chunkFramesReceived}`;
|
|
1700
|
+
}
|
|
1701
|
+
if (reportStream.chunkTotalBytesExpected > 0 &&
|
|
1702
|
+
reportStream.chunkBytesReceived !== reportStream.chunkTotalBytesExpected) {
|
|
1703
|
+
parseError =
|
|
1704
|
+
parseError ??
|
|
1705
|
+
`chunk size mismatch: expected ${reportStream.chunkTotalBytesExpected} bytes, received ${reportStream.chunkBytesReceived}`;
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1617
1709
|
if (parseError) {
|
|
1618
1710
|
const errorText = `could not parse report payload: ${parseError}`;
|
|
1711
|
+
const diagnostics = buildRuntimeReportDiagnostics(code, sawChannelClose, reportStream, runtimeEvents);
|
|
1712
|
+
const fullError = `${errorText}\n${diagnostics}`;
|
|
1619
1713
|
persistCrashRecord(crashDir, {
|
|
1620
1714
|
kind: "test",
|
|
1621
1715
|
file: specFile,
|
|
1622
1716
|
mode: modeName ?? "default",
|
|
1623
|
-
error:
|
|
1717
|
+
error: fullError,
|
|
1624
1718
|
stdout: stdoutBuffer,
|
|
1625
1719
|
stderr: stderrBuffer,
|
|
1626
1720
|
});
|
|
1627
|
-
return createRuntimeFailureReport(specFile, modeName, "runtime returned an invalid report payload",
|
|
1721
|
+
return createRuntimeFailureReport(specFile, modeName, "runtime returned an invalid report payload", fullError, stdoutBuffer, stderrBuffer);
|
|
1628
1722
|
}
|
|
1629
1723
|
if (!report) {
|
|
1724
|
+
const synthesized = synthesizeReportFromRuntimeEvents(specFile, runtimeEvents);
|
|
1725
|
+
if (synthesized) {
|
|
1726
|
+
reporter.onWarning?.({
|
|
1727
|
+
message: "runtime report payload missing; reconstructed result from streamed lifecycle events",
|
|
1728
|
+
});
|
|
1729
|
+
if (code !== 0 || hasMeaningfulRuntimeOutput(stderrBuffer)) {
|
|
1730
|
+
const errorParts = [];
|
|
1731
|
+
if (code !== 0) {
|
|
1732
|
+
errorParts.push(`child process exited with code ${code}`);
|
|
1733
|
+
}
|
|
1734
|
+
const stderrText = normalizeRuntimeOutput(stderrBuffer);
|
|
1735
|
+
if (stderrText.length) {
|
|
1736
|
+
errorParts.push(stderrText);
|
|
1737
|
+
}
|
|
1738
|
+
const diagnostics = buildRuntimeReportDiagnostics(code, sawChannelClose, reportStream, runtimeEvents);
|
|
1739
|
+
errorParts.push(diagnostics);
|
|
1740
|
+
const errorText = errorParts.join("\n\n");
|
|
1741
|
+
persistCrashRecord(crashDir, {
|
|
1742
|
+
kind: "test",
|
|
1743
|
+
file: specFile,
|
|
1744
|
+
mode: modeName ?? "default",
|
|
1745
|
+
error: errorText || "runtime reported an unknown error",
|
|
1746
|
+
stdout: stdoutBuffer,
|
|
1747
|
+
stderr: stderrBuffer,
|
|
1748
|
+
});
|
|
1749
|
+
return appendRuntimeFailureReport(synthesized, specFile, modeName, code !== 0
|
|
1750
|
+
? `test runtime failed with exit code ${code}`
|
|
1751
|
+
: "test runtime wrote to stderr", errorText, stdoutBuffer, stderrBuffer);
|
|
1752
|
+
}
|
|
1753
|
+
return synthesized;
|
|
1754
|
+
}
|
|
1630
1755
|
const errorText = "missing report payload from test runtime";
|
|
1756
|
+
const diagnostics = buildRuntimeReportDiagnostics(code, sawChannelClose, reportStream, runtimeEvents);
|
|
1757
|
+
const fullError = `${errorText}\n${diagnostics}`;
|
|
1631
1758
|
persistCrashRecord(crashDir, {
|
|
1632
1759
|
kind: "test",
|
|
1633
1760
|
file: specFile,
|
|
1634
1761
|
mode: modeName ?? "default",
|
|
1635
|
-
error:
|
|
1762
|
+
error: fullError,
|
|
1636
1763
|
stdout: stdoutBuffer,
|
|
1637
1764
|
stderr: stderrBuffer,
|
|
1638
1765
|
});
|
|
1639
|
-
return createRuntimeFailureReport(specFile, modeName, "test runtime exited without sending a report",
|
|
1766
|
+
return createRuntimeFailureReport(specFile, modeName, "test runtime exited without sending a report", fullError, stdoutBuffer, stderrBuffer);
|
|
1640
1767
|
}
|
|
1641
1768
|
if (code !== 0 || hasMeaningfulRuntimeOutput(stderrBuffer)) {
|
|
1642
1769
|
const errorParts = [];
|
|
@@ -1662,6 +1789,52 @@ async function runProcess(invocation, specFile, crashDir, modeName, snapshots, s
|
|
|
1662
1789
|
}
|
|
1663
1790
|
return report;
|
|
1664
1791
|
}
|
|
1792
|
+
function synthesizeReportFromRuntimeEvents(specFile, runtimeEvents) {
|
|
1793
|
+
if (!runtimeEvents.sawFileEnd &&
|
|
1794
|
+
runtimeEvents.suiteStarts <= 0 &&
|
|
1795
|
+
runtimeEvents.suiteEnds <= 0) {
|
|
1796
|
+
return null;
|
|
1797
|
+
}
|
|
1798
|
+
let verdict = runtimeEvents.fileVerdict;
|
|
1799
|
+
if (verdict == "none" && runtimeEvents.assertionFails > 0) {
|
|
1800
|
+
verdict = "fail";
|
|
1801
|
+
}
|
|
1802
|
+
else if (verdict == "none" && runtimeEvents.sawFileEnd) {
|
|
1803
|
+
verdict = "ok";
|
|
1804
|
+
}
|
|
1805
|
+
return {
|
|
1806
|
+
suites: [
|
|
1807
|
+
{
|
|
1808
|
+
file: specFile,
|
|
1809
|
+
description: runtimeEvents.fileName || path.basename(specFile),
|
|
1810
|
+
depth: 0,
|
|
1811
|
+
kind: "file",
|
|
1812
|
+
verdict,
|
|
1813
|
+
time: {
|
|
1814
|
+
start: 0,
|
|
1815
|
+
end: 0,
|
|
1816
|
+
},
|
|
1817
|
+
suites: [],
|
|
1818
|
+
logs: [],
|
|
1819
|
+
tests: [],
|
|
1820
|
+
},
|
|
1821
|
+
],
|
|
1822
|
+
coverage: {
|
|
1823
|
+
total: 0,
|
|
1824
|
+
covered: 0,
|
|
1825
|
+
uncovered: 0,
|
|
1826
|
+
percent: 100,
|
|
1827
|
+
points: [],
|
|
1828
|
+
},
|
|
1829
|
+
};
|
|
1830
|
+
}
|
|
1831
|
+
function buildRuntimeReportDiagnostics(code, sawChannelClose, reportStream, runtimeEvents) {
|
|
1832
|
+
return [
|
|
1833
|
+
`runtime diagnostics: exitCode=${code}, channelClose=${sawChannelClose ? "yes" : "no"}`,
|
|
1834
|
+
`report stream: dataFrames=${reportStream.dataFrames}, dataBytes=${reportStream.dataBytes}, chunked=${reportStream.sawChunkStart ? "yes" : "no"}, chunkStart=${reportStream.sawChunkStart ? "yes" : "no"}, chunkEnd=${reportStream.sawChunkEnd ? "yes" : "no"}, chunkFrames=${reportStream.chunkFramesReceived}, expectedChunkFrames=${reportStream.chunkCountExpected}, chunkBytes=${reportStream.chunkBytesReceived}, expectedChunkBytes=${reportStream.chunkTotalBytesExpected}`,
|
|
1835
|
+
`runtime events: fileStart=${runtimeEvents.sawFileStart ? "yes" : "no"}, fileEnd=${runtimeEvents.sawFileEnd ? "yes" : "no"}, fileVerdict=${runtimeEvents.fileVerdict}, suiteStarts=${runtimeEvents.suiteStarts}, suiteEnds=${runtimeEvents.suiteEnds}, assertionFails=${runtimeEvents.assertionFails}, warnings=${runtimeEvents.warnings}, logs=${runtimeEvents.logs}`,
|
|
1836
|
+
].join("\n");
|
|
1837
|
+
}
|
|
1665
1838
|
function createRuntimeFailureReport(specFile, modeName, title, details, stdout, stderr) {
|
|
1666
1839
|
return appendRuntimeFailureReport({
|
|
1667
1840
|
suites: [],
|
package/bin/index.js
CHANGED
|
@@ -282,7 +282,7 @@ function printCommandHelp(command) {
|
|
|
282
282
|
process.stdout.write(" --disable <feature> Disable feature (coverage|try-as)\n");
|
|
283
283
|
process.stdout.write(" --fuzz Run fuzz targets after the normal test pass\n");
|
|
284
284
|
process.stdout.write(" --fuzz-runs <value> Override fuzz iteration count, e.g. 500, 1.5x, +10%, +100000\n");
|
|
285
|
-
process.stdout.write(" --fuzz-seed <n>
|
|
285
|
+
process.stdout.write(" --fuzz-seed <n> Pin fuzz seed for this run (default uses random seed)\n");
|
|
286
286
|
process.stdout.write(" --parallel Run files through an ordered worker pool using an automatic worker count\n");
|
|
287
287
|
process.stdout.write(" --jobs <n> Run files through an ordered worker pool\n");
|
|
288
288
|
process.stdout.write(" --build-jobs <n> Limit concurrent build tasks (defaults to --jobs)\n");
|
|
@@ -303,7 +303,7 @@ function printCommandHelp(command) {
|
|
|
303
303
|
process.stdout.write(" --config <path> Use a specific config file\n");
|
|
304
304
|
process.stdout.write(" --mode <name[,name...]> Run one or multiple named config modes\n");
|
|
305
305
|
process.stdout.write(" --runs <value> Override fuzz iteration count, e.g. 500, 1.5x, +10%, +100000\n");
|
|
306
|
-
process.stdout.write(" --seed <n>
|
|
306
|
+
process.stdout.write(" --seed <n> Pin fuzz seed (default uses random seed)\n");
|
|
307
307
|
process.stdout.write(" --jobs <n> Run files through an ordered worker pool\n");
|
|
308
308
|
process.stdout.write(" --build-jobs <n> Limit concurrent build tasks (defaults to --jobs)\n");
|
|
309
309
|
process.stdout.write(" --run-jobs <n> Limit concurrent run tasks (defaults to --jobs)\n");
|
package/bin/reporters/default.js
CHANGED
|
@@ -354,8 +354,8 @@ function renderFailedFuzzers(results) {
|
|
|
354
354
|
rendered = true;
|
|
355
355
|
}
|
|
356
356
|
console.log(`${chalk.bgRed(" FAIL ")} ${formatFuzzFailureTitle(modeResult.file, fuzzer.name)}`);
|
|
357
|
-
if (fuzzer.failure
|
|
358
|
-
|
|
357
|
+
if (fuzzer.failure) {
|
|
358
|
+
renderAssertionFailureDetails(fuzzer.failure.left, fuzzer.failure.right, fuzzer.failure.message);
|
|
359
359
|
}
|
|
360
360
|
console.log(chalk.dim(`Mode: ${modeResult.modeName}`));
|
|
361
361
|
console.log(chalk.dim(`Runs: ${fuzzer.passed + fuzzer.failed + fuzzer.crashed} completed (${fuzzer.passed} passed, ${fuzzer.failed} failed, ${fuzzer.crashed} crashed)`));
|
|
@@ -487,52 +487,11 @@ function collectSuiteFailures(suite, file, path, printed) {
|
|
|
487
487
|
if (printed.has(dedupeKey))
|
|
488
488
|
continue;
|
|
489
489
|
printed.add(dedupeKey);
|
|
490
|
-
const left = JSON.stringify(test.left);
|
|
491
|
-
const right = JSON.stringify(test.right);
|
|
492
|
-
if (left == "null" && right == "null") {
|
|
493
|
-
console.log(`${chalk.bgRed(" FAIL ")} ${chalk.dim(title)} ${chalk.dim("(" + where + ")")}`);
|
|
494
|
-
if (modeName.length) {
|
|
495
|
-
console.log(chalk.dim(`Mode: ${modeName}`));
|
|
496
|
-
}
|
|
497
|
-
const normalizedMessage = normalizeFailureMessage(message);
|
|
498
|
-
if (normalizedMessage.length) {
|
|
499
|
-
for (const line of normalizedMessage.split("\n")) {
|
|
500
|
-
console.log(chalk.dim(line));
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
else {
|
|
504
|
-
console.log(chalk.dim("runtime error"));
|
|
505
|
-
}
|
|
506
|
-
console.log("");
|
|
507
|
-
continue;
|
|
508
|
-
}
|
|
509
|
-
const diffResult = diff(left, right);
|
|
510
|
-
let expected = "";
|
|
511
|
-
for (const res of diffResult.diff) {
|
|
512
|
-
switch (res.type) {
|
|
513
|
-
case "correct":
|
|
514
|
-
expected += chalk.dim(res.value);
|
|
515
|
-
break;
|
|
516
|
-
case "extra":
|
|
517
|
-
expected += chalk.red.strikethrough(res.value);
|
|
518
|
-
break;
|
|
519
|
-
case "missing":
|
|
520
|
-
expected += chalk.bgBlack(res.value);
|
|
521
|
-
break;
|
|
522
|
-
case "wrong":
|
|
523
|
-
expected += chalk.bgRed(res.value);
|
|
524
|
-
break;
|
|
525
|
-
case "untouched":
|
|
526
|
-
case "spacer":
|
|
527
|
-
break;
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
490
|
console.log(`${chalk.bgRed(" FAIL ")} ${chalk.dim(title)} ${chalk.dim("(" + where + ")")}`);
|
|
531
491
|
if (modeName.length) {
|
|
532
492
|
console.log(chalk.dim(`Mode: ${modeName}`));
|
|
533
493
|
}
|
|
534
|
-
|
|
535
|
-
console.log(`${chalk.dim("(received) ->")} ${chalk.dim(left)}\n`);
|
|
494
|
+
renderAssertionFailureDetails(test.left, test.right, message);
|
|
536
495
|
}
|
|
537
496
|
const suites = Array.isArray(suiteAny.suites)
|
|
538
497
|
? suiteAny.suites
|
|
@@ -544,6 +503,46 @@ function collectSuiteFailures(suite, file, path, printed) {
|
|
|
544
503
|
function normalizeFailureMessage(message) {
|
|
545
504
|
return message.replace(/\r\n/g, "\n").trim();
|
|
546
505
|
}
|
|
506
|
+
function renderAssertionFailureDetails(leftRaw, rightRaw, messageRaw) {
|
|
507
|
+
const left = JSON.stringify(leftRaw);
|
|
508
|
+
const right = JSON.stringify(rightRaw);
|
|
509
|
+
const message = String(messageRaw ?? "");
|
|
510
|
+
if (left == "null" && right == "null") {
|
|
511
|
+
const normalizedMessage = normalizeFailureMessage(message);
|
|
512
|
+
if (normalizedMessage.length) {
|
|
513
|
+
for (const line of normalizedMessage.split("\n")) {
|
|
514
|
+
console.log(chalk.dim(line));
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
else {
|
|
518
|
+
console.log(chalk.dim("runtime error"));
|
|
519
|
+
}
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
const diffResult = diff(left, right);
|
|
523
|
+
let expected = "";
|
|
524
|
+
for (const res of diffResult.diff) {
|
|
525
|
+
switch (res.type) {
|
|
526
|
+
case "correct":
|
|
527
|
+
expected += chalk.dim(res.value);
|
|
528
|
+
break;
|
|
529
|
+
case "extra":
|
|
530
|
+
expected += chalk.red.strikethrough(res.value);
|
|
531
|
+
break;
|
|
532
|
+
case "missing":
|
|
533
|
+
expected += chalk.bgBlack(res.value);
|
|
534
|
+
break;
|
|
535
|
+
case "wrong":
|
|
536
|
+
expected += chalk.bgRed(res.value);
|
|
537
|
+
break;
|
|
538
|
+
case "untouched":
|
|
539
|
+
case "spacer":
|
|
540
|
+
break;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
console.log(`${chalk.dim("(expected) ->")} ${expected}`);
|
|
544
|
+
console.log(`${chalk.dim("(received) ->")} ${chalk.dim(left)}\n`);
|
|
545
|
+
}
|
|
547
546
|
function renderSnapshotSummary(snapshotSummary, leadingGap = true) {
|
|
548
547
|
if (leadingGap) {
|
|
549
548
|
console.log("");
|
package/bin/types.js
CHANGED
|
@@ -77,7 +77,7 @@ export class FuzzConfig {
|
|
|
77
77
|
constructor() {
|
|
78
78
|
this.input = ["./assembly/__fuzz__/*.fuzz.ts"];
|
|
79
79
|
this.runs = 1000;
|
|
80
|
-
this.seed =
|
|
80
|
+
this.seed = -1;
|
|
81
81
|
this.maxInputBytes = 4096;
|
|
82
82
|
this.target = "bindings";
|
|
83
83
|
this.corpusDir = "./.as-test/fuzz/corpus";
|
package/bin/util.js
CHANGED
|
@@ -115,7 +115,7 @@ export function loadConfig(CONFIG_PATH, warn = false) {
|
|
|
115
115
|
? [fuzzRaw.input]
|
|
116
116
|
: new FuzzConfig().input;
|
|
117
117
|
config.fuzz.runs = normalizePositiveNumber(config.fuzz.runs, 1000);
|
|
118
|
-
config.fuzz.seed = normalizeNonNegativeNumber(config.fuzz.seed,
|
|
118
|
+
config.fuzz.seed = normalizeNonNegativeNumber(config.fuzz.seed, -1);
|
|
119
119
|
config.fuzz.maxInputBytes = normalizePositiveNumber(config.fuzz.maxInputBytes, 4096);
|
|
120
120
|
config.fuzz.target =
|
|
121
121
|
typeof config.fuzz.target == "string" && config.fuzz.target.length
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "as-test",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.12",
|
|
4
4
|
"author": "Jairus Tanaka",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -68,6 +68,8 @@
|
|
|
68
68
|
"test": "node ./bin/index.js test --parallel",
|
|
69
69
|
"test:ci": "node ./bin/index.js test --parallel --tap --config ./as-test.ci.config.json",
|
|
70
70
|
"fuzz": "node ./bin/index.js fuzz",
|
|
71
|
+
"bench:seed": "node ./bin/index.js fuzz --config ./as-test.bench.config.json --clean",
|
|
72
|
+
"bench:seed:compare": "bash ./tools/bench-seed-compare.sh 7",
|
|
71
73
|
"test:examples": "npm --prefix ./examples run test",
|
|
72
74
|
"ci:act": "bash ./tools/act.sh push",
|
|
73
75
|
"ci:act:pr": "bash ./tools/act.sh pull_request",
|