modestbench 0.3.2 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +45 -2
- package/dist/adapters/ava-adapter.cjs +421 -0
- package/dist/adapters/ava-adapter.cjs.map +1 -0
- package/dist/adapters/ava-adapter.d.cts +39 -0
- package/dist/adapters/ava-adapter.d.cts.map +1 -0
- package/dist/adapters/ava-adapter.d.ts +39 -0
- package/dist/adapters/ava-adapter.d.ts.map +1 -0
- package/dist/adapters/ava-adapter.js +384 -0
- package/dist/adapters/ava-adapter.js.map +1 -0
- package/dist/adapters/ava-hooks.cjs +66 -0
- package/dist/adapters/ava-hooks.cjs.map +1 -0
- package/dist/adapters/ava-hooks.d.cts +24 -0
- package/dist/adapters/ava-hooks.d.cts.map +1 -0
- package/dist/adapters/ava-hooks.d.ts +24 -0
- package/dist/adapters/ava-hooks.d.ts.map +1 -0
- package/dist/adapters/ava-hooks.js +61 -0
- package/dist/adapters/ava-hooks.js.map +1 -0
- package/dist/adapters/ava-register.cjs +16 -0
- package/dist/adapters/ava-register.cjs.map +1 -0
- package/dist/adapters/ava-register.d.cts +11 -0
- package/dist/adapters/ava-register.d.cts.map +1 -0
- package/dist/adapters/ava-register.d.ts +11 -0
- package/dist/adapters/ava-register.d.ts.map +1 -0
- package/dist/adapters/ava-register.js +14 -0
- package/dist/adapters/ava-register.js.map +1 -0
- package/dist/adapters/mocha-adapter.cjs +254 -0
- package/dist/adapters/mocha-adapter.cjs.map +1 -0
- package/dist/adapters/mocha-adapter.d.cts +26 -0
- package/dist/adapters/mocha-adapter.d.cts.map +1 -0
- package/dist/adapters/mocha-adapter.d.ts +26 -0
- package/dist/adapters/mocha-adapter.d.ts.map +1 -0
- package/dist/adapters/mocha-adapter.js +217 -0
- package/dist/adapters/mocha-adapter.js.map +1 -0
- package/dist/adapters/node-test-adapter.cjs +335 -0
- package/dist/adapters/node-test-adapter.cjs.map +1 -0
- package/dist/adapters/node-test-adapter.d.cts +41 -0
- package/dist/adapters/node-test-adapter.d.cts.map +1 -0
- package/dist/adapters/node-test-adapter.d.ts +41 -0
- package/dist/adapters/node-test-adapter.d.ts.map +1 -0
- package/dist/adapters/node-test-adapter.js +298 -0
- package/dist/adapters/node-test-adapter.js.map +1 -0
- package/dist/adapters/node-test-hooks.cjs +72 -0
- package/dist/adapters/node-test-hooks.cjs.map +1 -0
- package/dist/adapters/node-test-hooks.d.cts +24 -0
- package/dist/adapters/node-test-hooks.d.cts.map +1 -0
- package/dist/adapters/node-test-hooks.d.ts +24 -0
- package/dist/adapters/node-test-hooks.d.ts.map +1 -0
- package/dist/adapters/node-test-hooks.js +67 -0
- package/dist/adapters/node-test-hooks.js.map +1 -0
- package/dist/adapters/node-test-register.cjs +7 -0
- package/dist/adapters/node-test-register.cjs.map +1 -0
- package/dist/adapters/node-test-register.d.cts +2 -0
- package/dist/adapters/node-test-register.d.cts.map +1 -0
- package/dist/adapters/node-test-register.d.ts +2 -0
- package/dist/adapters/node-test-register.d.ts.map +1 -0
- package/dist/adapters/node-test-register.js +5 -0
- package/dist/adapters/node-test-register.js.map +1 -0
- package/dist/adapters/types.cjs +152 -0
- package/dist/adapters/types.cjs.map +1 -0
- package/dist/adapters/types.d.cts +112 -0
- package/dist/adapters/types.d.cts.map +1 -0
- package/dist/adapters/types.d.ts +112 -0
- package/dist/adapters/types.d.ts.map +1 -0
- package/dist/adapters/types.js +148 -0
- package/dist/adapters/types.js.map +1 -0
- package/dist/cli/commands/init.cjs +21 -17
- package/dist/cli/commands/init.cjs.map +1 -1
- package/dist/cli/commands/init.d.cts.map +1 -1
- package/dist/cli/commands/init.d.ts.map +1 -1
- package/dist/cli/commands/init.js +21 -17
- package/dist/cli/commands/init.js.map +1 -1
- package/dist/cli/commands/run.cjs +6 -2
- package/dist/cli/commands/run.cjs.map +1 -1
- package/dist/cli/commands/run.js +6 -2
- package/dist/cli/commands/run.js.map +1 -1
- package/dist/cli/commands/test.cjs +392 -0
- package/dist/cli/commands/test.cjs.map +1 -0
- package/dist/cli/commands/test.d.cts +38 -0
- package/dist/cli/commands/test.d.cts.map +1 -0
- package/dist/cli/commands/test.d.ts +38 -0
- package/dist/cli/commands/test.d.ts.map +1 -0
- package/dist/cli/commands/test.js +388 -0
- package/dist/cli/commands/test.js.map +1 -0
- package/dist/cli/index.cjs +72 -1
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.d.cts.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +73 -2
- package/dist/cli/index.js.map +1 -1
- package/dist/constants.cjs +13 -1
- package/dist/constants.cjs.map +1 -1
- package/dist/constants.d.cts +12 -0
- package/dist/constants.d.cts.map +1 -1
- package/dist/constants.d.ts +12 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +12 -0
- package/dist/constants.js.map +1 -1
- package/dist/core/engine.cjs +4 -0
- package/dist/core/engine.cjs.map +1 -1
- package/dist/core/engine.d.cts.map +1 -1
- package/dist/core/engine.d.ts.map +1 -1
- package/dist/core/engine.js +4 -0
- package/dist/core/engine.js.map +1 -1
- package/dist/core/engines/tinybench-engine.cjs +163 -131
- package/dist/core/engines/tinybench-engine.cjs.map +1 -1
- package/dist/core/engines/tinybench-engine.d.cts +6 -0
- package/dist/core/engines/tinybench-engine.d.cts.map +1 -1
- package/dist/core/engines/tinybench-engine.d.ts +6 -0
- package/dist/core/engines/tinybench-engine.d.ts.map +1 -1
- package/dist/core/engines/tinybench-engine.js +163 -131
- package/dist/core/engines/tinybench-engine.js.map +1 -1
- package/dist/core/stats-utils.cjs +4 -2
- package/dist/core/stats-utils.cjs.map +1 -1
- package/dist/core/stats-utils.d.cts +1 -1
- package/dist/core/stats-utils.d.cts.map +1 -1
- package/dist/core/stats-utils.d.ts +1 -1
- package/dist/core/stats-utils.d.ts.map +1 -1
- package/dist/core/stats-utils.js +4 -2
- package/dist/core/stats-utils.js.map +1 -1
- package/dist/errors/base.cjs +2 -1
- package/dist/errors/base.cjs.map +1 -1
- package/dist/errors/base.d.cts.map +1 -1
- package/dist/errors/base.d.ts.map +1 -1
- package/dist/errors/base.js +2 -1
- package/dist/errors/base.js.map +1 -1
- package/dist/formatters/history/compare.cjs.map +1 -1
- package/dist/formatters/history/compare.d.cts.map +1 -1
- package/dist/formatters/history/compare.d.ts.map +1 -1
- package/dist/formatters/history/compare.js.map +1 -1
- package/dist/formatters/history/list.cjs.map +1 -1
- package/dist/formatters/history/list.d.cts.map +1 -1
- package/dist/formatters/history/list.d.ts.map +1 -1
- package/dist/formatters/history/list.js.map +1 -1
- package/dist/reporters/human.cjs +83 -27
- package/dist/reporters/human.cjs.map +1 -1
- package/dist/reporters/human.d.cts +1 -0
- package/dist/reporters/human.d.cts.map +1 -1
- package/dist/reporters/human.d.ts +1 -0
- package/dist/reporters/human.d.ts.map +1 -1
- package/dist/reporters/human.js +83 -27
- package/dist/reporters/human.js.map +1 -1
- package/dist/reporters/simple.cjs +68 -21
- package/dist/reporters/simple.cjs.map +1 -1
- package/dist/reporters/simple.d.cts +1 -0
- package/dist/reporters/simple.d.cts.map +1 -1
- package/dist/reporters/simple.d.ts +1 -0
- package/dist/reporters/simple.d.ts.map +1 -1
- package/dist/reporters/simple.js +68 -21
- package/dist/reporters/simple.js.map +1 -1
- package/dist/services/config-manager.cjs +1 -1
- package/dist/services/config-manager.cjs.map +1 -1
- package/dist/services/config-manager.d.cts.map +1 -1
- package/dist/services/config-manager.d.ts.map +1 -1
- package/dist/services/config-manager.js +2 -2
- package/dist/services/config-manager.js.map +1 -1
- package/dist/services/file-loader.cjs.map +1 -1
- package/dist/services/file-loader.d.cts.map +1 -1
- package/dist/services/file-loader.d.ts.map +1 -1
- package/dist/services/file-loader.js.map +1 -1
- package/package.json +63 -35
- package/src/adapters/ava-adapter.ts +553 -0
- package/src/adapters/ava-hooks.ts +65 -0
- package/src/adapters/ava-register.ts +15 -0
- package/src/adapters/mocha-adapter.ts +284 -0
- package/src/adapters/node-test-adapter.ts +391 -0
- package/src/adapters/node-test-hooks.ts +71 -0
- package/src/adapters/node-test-register.ts +5 -0
- package/src/adapters/types.ts +281 -0
- package/src/cli/commands/init.ts +25 -17
- package/src/cli/commands/run.ts +9 -2
- package/src/cli/commands/test.ts +546 -0
- package/src/cli/index.ts +81 -1
- package/src/constants.ts +15 -0
- package/src/core/engine.ts +5 -0
- package/src/core/engines/tinybench-engine.ts +213 -141
- package/src/core/stats-utils.ts +5 -3
- package/src/errors/base.ts +3 -2
- package/src/formatters/history/compare.ts +1 -3
- package/src/formatters/history/list.ts +1 -3
- package/src/reporters/human.ts +107 -36
- package/src/reporters/simple.ts +81 -22
- package/src/services/config-manager.ts +4 -5
- package/src/services/file-loader.ts +2 -3
package/src/reporters/human.ts
CHANGED
|
@@ -19,6 +19,11 @@ import type {
|
|
|
19
19
|
import { BaseReporter } from '../services/reporter-registry.js';
|
|
20
20
|
import { ansiChars, colors } from '../utils/ansi.js';
|
|
21
21
|
|
|
22
|
+
/**
|
|
23
|
+
* Minimum iterations required for reliable CV calculation
|
|
24
|
+
*/
|
|
25
|
+
const MIN_RELIABLE_ITERATIONS = 30;
|
|
26
|
+
|
|
22
27
|
/**
|
|
23
28
|
* Human-readable console reporter with colorized output
|
|
24
29
|
*/
|
|
@@ -38,6 +43,8 @@ export class HumanReporter extends BaseReporter {
|
|
|
38
43
|
|
|
39
44
|
private lastProgressLine = '';
|
|
40
45
|
|
|
46
|
+
private lowIterationCount = 0;
|
|
47
|
+
|
|
41
48
|
private maxTimePadWidth = 0; // Track maximum time padding width to prevent jitter
|
|
42
49
|
|
|
43
50
|
private progressWindowActive = false; // Track if progress window is rendered
|
|
@@ -165,14 +172,21 @@ export class HumanReporter extends BaseReporter {
|
|
|
165
172
|
const totalFiles = run.files.length;
|
|
166
173
|
|
|
167
174
|
// Calculate totals across all files
|
|
175
|
+
// Note: Suite-level failures (setup errors) are tracked in this.failures
|
|
176
|
+
// but not counted as task failures to keep statistics consistent
|
|
168
177
|
let totalSuites = 0;
|
|
169
178
|
let totalPassed = 0;
|
|
170
179
|
let totalFailed = 0;
|
|
171
180
|
let totalAborted = 0;
|
|
181
|
+
let suiteFailures = 0;
|
|
172
182
|
|
|
173
183
|
for (const file of run.files) {
|
|
174
184
|
totalSuites += file.suites.length;
|
|
175
185
|
for (const suite of file.suites) {
|
|
186
|
+
// Track suite-level errors separately (not as task failures)
|
|
187
|
+
if (suite.error) {
|
|
188
|
+
suiteFailures++;
|
|
189
|
+
}
|
|
176
190
|
totalPassed += suite.tasks.filter(
|
|
177
191
|
(t: TaskResult) => !t.error && !t.aborted,
|
|
178
192
|
).length;
|
|
@@ -195,10 +209,13 @@ export class HumanReporter extends BaseReporter {
|
|
|
195
209
|
this.printLine(
|
|
196
210
|
`${this.colorize('brightBlue', ' Tasks:')} ${this.colorize('brightWhite', String(totalPassed + totalFailed + totalAborted))}`,
|
|
197
211
|
);
|
|
198
|
-
|
|
199
|
-
|
|
212
|
+
// Check for any failures: task failures, suite failures (setup errors), or aborts
|
|
213
|
+
const hasFailures =
|
|
214
|
+
totalFailed > 0 || suiteFailures > 0 || totalAborted > 0;
|
|
215
|
+
if (hasFailures) {
|
|
216
|
+
if (totalFailed > 0 || suiteFailures > 0) {
|
|
200
217
|
this.printLine(
|
|
201
|
-
`${this.colorize('brightRed', ansiChars.cross + ' Failed:')} ${this.colorize('brightWhite', String(totalFailed))}`,
|
|
218
|
+
`${this.colorize('brightRed', ansiChars.cross + ' Failed:')} ${this.colorize('brightWhite', String(totalFailed + suiteFailures))}`,
|
|
202
219
|
);
|
|
203
220
|
}
|
|
204
221
|
if (totalPassed > 0) {
|
|
@@ -221,29 +238,35 @@ export class HumanReporter extends BaseReporter {
|
|
|
221
238
|
);
|
|
222
239
|
this.printLine();
|
|
223
240
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
241
|
+
// Display failed tasks/suites with details
|
|
242
|
+
if (this.failures.length > 0) {
|
|
243
|
+
this.printLine();
|
|
244
|
+
this.printLine(
|
|
245
|
+
this.colorize('brightRed', this.colorize('bold', 'Failed Tasks:')),
|
|
246
|
+
);
|
|
247
|
+
this.printLine();
|
|
248
|
+
|
|
249
|
+
for (const failure of this.failures) {
|
|
250
|
+
const displayPath = HumanReporter.formatPath(failure.file);
|
|
228
251
|
this.printLine(
|
|
229
|
-
this.colorize('
|
|
252
|
+
` ${this.colorize('dim', displayPath)} ${this.colorize('dim', '›')} ${this.colorize('white', failure.suite)} ${this.colorize('dim', '›')} ${this.colorize('brightWhite', failure.task)}`,
|
|
230
253
|
);
|
|
254
|
+
this.printLine(` ${this.colorize('brightRed', failure.error)}`);
|
|
231
255
|
this.printLine();
|
|
232
|
-
|
|
233
|
-
for (const failure of this.failures) {
|
|
234
|
-
const displayPath = HumanReporter.formatPath(failure.file);
|
|
235
|
-
this.printLine(
|
|
236
|
-
` ${this.colorize('dim', displayPath)} ${this.colorize('dim', '›')} ${this.colorize('white', failure.suite)} ${this.colorize('dim', '›')} ${this.colorize('brightWhite', failure.task)}`,
|
|
237
|
-
);
|
|
238
|
-
this.printLine(` ${this.colorize('brightRed', failure.error)}`);
|
|
239
|
-
this.printLine();
|
|
240
|
-
}
|
|
241
256
|
}
|
|
242
257
|
} else if (totalAborted === 0) {
|
|
243
258
|
// Only show "Rad" if no failures AND no aborts
|
|
244
259
|
const successMessage = `${this.colorize('brightMagenta', 'Rad. ☮')}`;
|
|
245
260
|
this.printLine(successMessage);
|
|
246
261
|
}
|
|
262
|
+
|
|
263
|
+
// Show warning for low iteration counts
|
|
264
|
+
if (this.lowIterationCount > 0) {
|
|
265
|
+
this.printLine();
|
|
266
|
+
this.printLine(
|
|
267
|
+
`${this.colorize('brightYellow', ansiChars.approx)} ${this.colorize('brightYellow', 'Warning:')} ${this.lowIterationCount} ${HumanReporter.pluralize('task', this.lowIterationCount)} had low iteration counts (<${MIN_RELIABLE_ITERATIONS}) which may affect statistical reliability`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
247
270
|
}
|
|
248
271
|
|
|
249
272
|
onError(error: Error): void {
|
|
@@ -373,6 +396,7 @@ export class HumanReporter extends BaseReporter {
|
|
|
373
396
|
this.failures = []; // Reset failures for new run
|
|
374
397
|
this.lastProgressLine = ''; // Reset for new run
|
|
375
398
|
this.maxTimePadWidth = 0; // Reset time padding width for new run
|
|
399
|
+
this.lowIterationCount = 0; // Reset low iteration count for new run
|
|
376
400
|
|
|
377
401
|
if (this.quiet) {
|
|
378
402
|
return;
|
|
@@ -428,6 +452,31 @@ export class HumanReporter extends BaseReporter {
|
|
|
428
452
|
return;
|
|
429
453
|
}
|
|
430
454
|
|
|
455
|
+
// Handle suite-level errors (e.g., setup failure)
|
|
456
|
+
if (result.error) {
|
|
457
|
+
// Track suite-level failure for end-of-run summary
|
|
458
|
+
this.failures.push({
|
|
459
|
+
error: result.error.message || String(result.error),
|
|
460
|
+
file: this.currentFile,
|
|
461
|
+
suite: result.name,
|
|
462
|
+
task: '(setup)',
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
const durationStr = BaseReporter.formatDuration(
|
|
466
|
+
result.duration * 1000000,
|
|
467
|
+
);
|
|
468
|
+
|
|
469
|
+
// Display suite setup failure
|
|
470
|
+
this.printLine(
|
|
471
|
+
` ${this.colorize('red', ansiChars.cross)} ${this.colorize('white', '(setup)')} ${this.colorize('red', 'FAILED')}`,
|
|
472
|
+
);
|
|
473
|
+
this.printLine(
|
|
474
|
+
` ${this.colorize('red', `${ansiChars.cross} Suite setup failed`)} ${this.colorize('gray', 'in')} ${this.colorize('cyan', durationStr)}`,
|
|
475
|
+
);
|
|
476
|
+
this.printLine();
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
|
|
431
480
|
// Tasks are printed immediately in onTaskResult, so just print suite summary
|
|
432
481
|
|
|
433
482
|
// Skip displaying summary for the implicit "default" suite
|
|
@@ -634,6 +683,9 @@ export class HumanReporter extends BaseReporter {
|
|
|
634
683
|
error: boolean;
|
|
635
684
|
errorMessage?: string;
|
|
636
685
|
iterations: number;
|
|
686
|
+
iterationsLen: number;
|
|
687
|
+
iterationsStr: string;
|
|
688
|
+
lowIterations: boolean;
|
|
637
689
|
name: string;
|
|
638
690
|
nameLength: number;
|
|
639
691
|
opsPerSecLen: number;
|
|
@@ -661,6 +713,9 @@ export class HumanReporter extends BaseReporter {
|
|
|
661
713
|
error: true,
|
|
662
714
|
errorMessage: result.error?.message || String(result.error),
|
|
663
715
|
iterations: 0,
|
|
716
|
+
iterationsLen: 0,
|
|
717
|
+
iterationsStr: '',
|
|
718
|
+
lowIterations: false,
|
|
664
719
|
name,
|
|
665
720
|
nameLength,
|
|
666
721
|
opsPerSecLen: 0,
|
|
@@ -674,12 +729,17 @@ export class HumanReporter extends BaseReporter {
|
|
|
674
729
|
const duration = BaseReporter.formatDuration(result.mean); // already in nanoseconds
|
|
675
730
|
const opsPerSec = BaseReporter.formatOpsPerSecond(result.opsPerSecond);
|
|
676
731
|
const rme = BaseReporter.formatPercentage(result.marginOfError); // already a percentage
|
|
732
|
+
const iterationsStr = `(${result.iterations} iter)`;
|
|
733
|
+
const lowIterations = result.iterations < MIN_RELIABLE_ITERATIONS;
|
|
677
734
|
|
|
678
735
|
return {
|
|
679
736
|
durationLen: this.getVisibleLength(duration),
|
|
680
737
|
durationStr: duration,
|
|
681
738
|
error: false,
|
|
682
739
|
iterations: result.iterations,
|
|
740
|
+
iterationsLen: iterationsStr.length,
|
|
741
|
+
iterationsStr,
|
|
742
|
+
lowIterations,
|
|
683
743
|
name,
|
|
684
744
|
nameLength,
|
|
685
745
|
opsPerSecLen: this.getVisibleLength(opsPerSec),
|
|
@@ -707,6 +767,10 @@ export class HumanReporter extends BaseReporter {
|
|
|
707
767
|
...formatted.filter((t) => !t.error).map((t) => t.rmeLen),
|
|
708
768
|
0,
|
|
709
769
|
);
|
|
770
|
+
const maxIterLen = Math.max(
|
|
771
|
+
...formatted.filter((t) => !t.error).map((t) => t.iterationsLen),
|
|
772
|
+
0,
|
|
773
|
+
);
|
|
710
774
|
const maxOpsLen = Math.max(
|
|
711
775
|
...formatted.filter((t) => !t.error).map((t) => t.opsPerSecLen),
|
|
712
776
|
0,
|
|
@@ -731,11 +795,13 @@ export class HumanReporter extends BaseReporter {
|
|
|
731
795
|
const wrappedLines = this.wrapText(task.name, MAX_NAME_WIDTH);
|
|
732
796
|
const continueIndent = BASE_INDENT + ' '; // 6 spaces for continuation lines
|
|
733
797
|
|
|
734
|
-
// Format stats string
|
|
798
|
+
// Format stats string with iterations
|
|
735
799
|
const durationPad = ' '.repeat(maxDurationLen - task.durationLen);
|
|
736
800
|
const rmePad = ' '.repeat(maxRmeLen - task.rmeLen);
|
|
801
|
+
const iterPad = ' '.repeat(maxIterLen - task.iterationsLen);
|
|
737
802
|
const opsPad = ' '.repeat(maxOpsLen - task.opsPerSecLen);
|
|
738
|
-
const
|
|
803
|
+
const iterColor = task.lowIterations ? 'brightRed' : 'cyan';
|
|
804
|
+
const statsStr = `${durationPad}${this.colorize('cyan', task.durationStr)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', task.rmeStr)} ${iterPad}${this.colorize(iterColor, task.iterationsStr)} ${bullet} ${opsPad}${this.colorize('magenta', task.opsPerSecStr)}`;
|
|
739
805
|
|
|
740
806
|
// Print first line with status
|
|
741
807
|
this.printLine(
|
|
@@ -766,26 +832,26 @@ export class HumanReporter extends BaseReporter {
|
|
|
766
832
|
);
|
|
767
833
|
}
|
|
768
834
|
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
);
|
|
835
|
+
// Track low iteration count
|
|
836
|
+
if (task.lowIterations) {
|
|
837
|
+
this.lowIterationCount++;
|
|
773
838
|
}
|
|
774
839
|
} else {
|
|
775
840
|
// Normal length - align on same line
|
|
776
841
|
const namePad = ' '.repeat(maxNameLen - task.nameLength);
|
|
777
842
|
const durationPad = ' '.repeat(maxDurationLen - task.durationLen);
|
|
778
843
|
const rmePad = ' '.repeat(maxRmeLen - task.rmeLen);
|
|
844
|
+
const iterPad = ' '.repeat(maxIterLen - task.iterationsLen);
|
|
779
845
|
const opsPad = ' '.repeat(maxOpsLen - task.opsPerSecLen);
|
|
846
|
+
const iterColor = task.lowIterations ? 'brightRed' : 'cyan';
|
|
780
847
|
|
|
781
848
|
this.printLine(
|
|
782
|
-
`${BASE_INDENT}${task.status} ${this.colorize('white', task.name)}${namePad}: ${durationPad}${this.colorize('cyan', task.durationStr)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', task.rmeStr)} ${bullet} ${opsPad}${this.colorize('magenta', task.opsPerSecStr)}`,
|
|
849
|
+
`${BASE_INDENT}${task.status} ${this.colorize('white', task.name)}${namePad}: ${durationPad}${this.colorize('cyan', task.durationStr)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', task.rmeStr)} ${iterPad}${this.colorize(iterColor, task.iterationsStr)} ${bullet} ${opsPad}${this.colorize('magenta', task.opsPerSecStr)}`,
|
|
783
850
|
);
|
|
784
851
|
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
);
|
|
852
|
+
// Track low iteration count
|
|
853
|
+
if (task.lowIterations) {
|
|
854
|
+
this.lowIterationCount++;
|
|
789
855
|
}
|
|
790
856
|
}
|
|
791
857
|
}
|
|
@@ -853,21 +919,27 @@ export class HumanReporter extends BaseReporter {
|
|
|
853
919
|
const duration = BaseReporter.formatDuration(result.mean);
|
|
854
920
|
const opsPerSec = BaseReporter.formatOpsPerSecond(result.opsPerSecond);
|
|
855
921
|
const rme = BaseReporter.formatPercentage(result.marginOfError);
|
|
922
|
+
const iterationsStr = `(${result.iterations} iter)`;
|
|
923
|
+
const lowIterations = result.iterations < MIN_RELIABLE_ITERATIONS;
|
|
856
924
|
|
|
857
925
|
// Use fixed widths for stats columns (reasonable maximums)
|
|
858
926
|
const DURATION_WIDTH = 10; // "999.99ms" max
|
|
859
927
|
const RME_WIDTH = 8; // "±999.99%" max
|
|
928
|
+
const ITER_WIDTH = 12; // "(99999 iter)" max
|
|
860
929
|
const OPS_WIDTH = 15; // "999.99K ops/sec" max
|
|
861
930
|
|
|
862
931
|
const durationLen = this.getVisibleLength(duration);
|
|
863
932
|
const rmeLen = this.getVisibleLength(rme);
|
|
933
|
+
const iterLen = iterationsStr.length;
|
|
864
934
|
const opsLen = this.getVisibleLength(opsPerSec);
|
|
865
935
|
|
|
866
936
|
// Stats formatting with fixed widths
|
|
867
|
-
const durationPad = ' '.repeat(DURATION_WIDTH - durationLen);
|
|
868
|
-
const rmePad = ' '.repeat(RME_WIDTH - rmeLen);
|
|
869
|
-
const
|
|
870
|
-
const
|
|
937
|
+
const durationPad = ' '.repeat(Math.max(0, DURATION_WIDTH - durationLen));
|
|
938
|
+
const rmePad = ' '.repeat(Math.max(0, RME_WIDTH - rmeLen));
|
|
939
|
+
const iterPad = ' '.repeat(Math.max(0, ITER_WIDTH - iterLen));
|
|
940
|
+
const opsPad = ' '.repeat(Math.max(0, OPS_WIDTH - opsLen));
|
|
941
|
+
const iterColor = lowIterations ? 'brightRed' : 'cyan';
|
|
942
|
+
const statsStr = `${durationPad}${this.colorize('cyan', duration)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', rme)} ${iterPad}${this.colorize(iterColor, iterationsStr)} ${bullet} ${opsPad}${this.colorize('magenta', opsPerSec)}`;
|
|
871
943
|
|
|
872
944
|
// Handle long names (wrap)
|
|
873
945
|
if (nameLength > MAX_NAME_WIDTH) {
|
|
@@ -917,10 +989,9 @@ export class HumanReporter extends BaseReporter {
|
|
|
917
989
|
);
|
|
918
990
|
}
|
|
919
991
|
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
);
|
|
992
|
+
// Track low iteration count
|
|
993
|
+
if (lowIterations) {
|
|
994
|
+
this.lowIterationCount++;
|
|
924
995
|
}
|
|
925
996
|
}
|
|
926
997
|
|
package/src/reporters/simple.ts
CHANGED
|
@@ -25,8 +25,14 @@ const symbols = {
|
|
|
25
25
|
checkmark: '√',
|
|
26
26
|
cross: '×',
|
|
27
27
|
plusMinus: '±',
|
|
28
|
+
warning: '⚠',
|
|
28
29
|
} as const;
|
|
29
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Minimum iterations required for reliable CV calculation
|
|
33
|
+
*/
|
|
34
|
+
const MIN_RELIABLE_ITERATIONS = 30;
|
|
35
|
+
|
|
30
36
|
/**
|
|
31
37
|
* Simple console reporter with plain text output (no colors or progress bars)
|
|
32
38
|
*/
|
|
@@ -42,6 +48,8 @@ export class SimpleReporter extends BaseReporter {
|
|
|
42
48
|
task: string;
|
|
43
49
|
}> = [];
|
|
44
50
|
|
|
51
|
+
private lowIterationCount = 0;
|
|
52
|
+
|
|
45
53
|
private readonly quiet: boolean;
|
|
46
54
|
|
|
47
55
|
private startTime = 0;
|
|
@@ -143,13 +151,20 @@ export class SimpleReporter extends BaseReporter {
|
|
|
143
151
|
const totalFiles = run.files.length;
|
|
144
152
|
|
|
145
153
|
// Calculate totals across all files
|
|
154
|
+
// Note: Suite-level failures (setup errors) are tracked in this.failures
|
|
155
|
+
// but not counted as task failures to keep statistics consistent
|
|
146
156
|
let totalSuites = 0;
|
|
147
157
|
let totalPassed = 0;
|
|
148
158
|
let totalFailed = 0;
|
|
159
|
+
let suiteFailures = 0;
|
|
149
160
|
|
|
150
161
|
for (const file of run.files) {
|
|
151
162
|
totalSuites += file.suites.length;
|
|
152
163
|
for (const suite of file.suites) {
|
|
164
|
+
// Track suite-level errors separately (not as task failures)
|
|
165
|
+
if (suite.error) {
|
|
166
|
+
suiteFailures++;
|
|
167
|
+
}
|
|
153
168
|
totalPassed += suite.tasks.filter((t: TaskResult) => !t.error).length;
|
|
154
169
|
totalFailed += suite.tasks.filter((t: TaskResult) => t.error).length;
|
|
155
170
|
}
|
|
@@ -159,8 +174,10 @@ export class SimpleReporter extends BaseReporter {
|
|
|
159
174
|
console.log('== Results');
|
|
160
175
|
console.log();
|
|
161
176
|
|
|
162
|
-
|
|
163
|
-
|
|
177
|
+
// Check for any failures: task failures or suite failures (setup errors)
|
|
178
|
+
const hasFailures = totalFailed > 0 || suiteFailures > 0;
|
|
179
|
+
if (hasFailures) {
|
|
180
|
+
console.log(`${symbols.cross} Failed: ${totalFailed + suiteFailures}`);
|
|
164
181
|
console.log(`${symbols.checkmark} Passed: ${totalPassed}`);
|
|
165
182
|
} else {
|
|
166
183
|
console.log(`${symbols.checkmark} All tasks passed: ${totalPassed}`);
|
|
@@ -173,25 +190,30 @@ export class SimpleReporter extends BaseReporter {
|
|
|
173
190
|
);
|
|
174
191
|
console.log();
|
|
175
192
|
|
|
176
|
-
|
|
193
|
+
// Display failed tasks/suites with details
|
|
194
|
+
if (this.failures.length > 0) {
|
|
177
195
|
console.log(`${symbols.cross.repeat(3)} Some benchmarks failed`);
|
|
196
|
+
console.log();
|
|
197
|
+
console.log('Failed Tasks:');
|
|
198
|
+
console.log();
|
|
178
199
|
|
|
179
|
-
|
|
180
|
-
|
|
200
|
+
for (const failure of this.failures) {
|
|
201
|
+
const displayPath = SimpleReporter.formatPath(failure.file);
|
|
202
|
+
console.log(` ${displayPath} > ${failure.suite} > ${failure.task}`);
|
|
203
|
+
console.log(` ${failure.error}`);
|
|
181
204
|
console.log();
|
|
182
|
-
console.log('Failed Tasks:');
|
|
183
|
-
console.log();
|
|
184
|
-
|
|
185
|
-
for (const failure of this.failures) {
|
|
186
|
-
const displayPath = SimpleReporter.formatPath(failure.file);
|
|
187
|
-
console.log(` ${displayPath} > ${failure.suite} > ${failure.task}`);
|
|
188
|
-
console.log(` ${failure.error}`);
|
|
189
|
-
console.log();
|
|
190
|
-
}
|
|
191
205
|
}
|
|
192
|
-
} else {
|
|
206
|
+
} else if (!hasFailures) {
|
|
193
207
|
console.log('All benchmarks completed successfully!');
|
|
194
208
|
}
|
|
209
|
+
|
|
210
|
+
// Show warning for low iteration counts
|
|
211
|
+
if (this.lowIterationCount > 0) {
|
|
212
|
+
console.log();
|
|
213
|
+
console.log(
|
|
214
|
+
`${symbols.warning} Warning: ${this.lowIterationCount} ${SimpleReporter.pluralize('task', this.lowIterationCount)} had low iteration counts (<${MIN_RELIABLE_ITERATIONS}) which may affect statistical reliability`,
|
|
215
|
+
);
|
|
216
|
+
}
|
|
195
217
|
}
|
|
196
218
|
|
|
197
219
|
onError(error: Error): void {
|
|
@@ -248,6 +270,7 @@ export class SimpleReporter extends BaseReporter {
|
|
|
248
270
|
onStart(run: BenchmarkRun): void {
|
|
249
271
|
this.startTime = Date.now();
|
|
250
272
|
this.failures = []; // Reset failures for new run
|
|
273
|
+
this.lowIterationCount = 0; // Reset low iteration count for new run
|
|
251
274
|
|
|
252
275
|
if (this.quiet) {
|
|
253
276
|
return;
|
|
@@ -285,6 +308,23 @@ export class SimpleReporter extends BaseReporter {
|
|
|
285
308
|
return;
|
|
286
309
|
}
|
|
287
310
|
|
|
311
|
+
// Handle suite-level errors (e.g., setup failure)
|
|
312
|
+
if (result.error) {
|
|
313
|
+
// Track suite-level failure for end-of-run summary
|
|
314
|
+
this.failures.push({
|
|
315
|
+
error: result.error.message || String(result.error),
|
|
316
|
+
file: this.currentFile,
|
|
317
|
+
suite: result.name,
|
|
318
|
+
task: '(setup)',
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
// Display suite setup failure
|
|
322
|
+
console.log(` ${symbols.cross} (setup) FAILED`);
|
|
323
|
+
console.log(` ${symbols.cross} Suite setup failed`);
|
|
324
|
+
console.log();
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
|
|
288
328
|
// Print all buffered task results with aligned columns
|
|
289
329
|
this.printAlignedSuiteResults();
|
|
290
330
|
|
|
@@ -366,6 +406,9 @@ export class SimpleReporter extends BaseReporter {
|
|
|
366
406
|
error: boolean;
|
|
367
407
|
errorMessage?: string;
|
|
368
408
|
iterations: number;
|
|
409
|
+
iterationsLen: number;
|
|
410
|
+
iterationsStr: string;
|
|
411
|
+
lowIterations: boolean;
|
|
369
412
|
name: string;
|
|
370
413
|
nameLength: number;
|
|
371
414
|
opsPerSecLen: number;
|
|
@@ -388,6 +431,9 @@ export class SimpleReporter extends BaseReporter {
|
|
|
388
431
|
error: true,
|
|
389
432
|
errorMessage: result.error?.message || String(result.error),
|
|
390
433
|
iterations: 0,
|
|
434
|
+
iterationsLen: 0,
|
|
435
|
+
iterationsStr: '',
|
|
436
|
+
lowIterations: false,
|
|
391
437
|
name,
|
|
392
438
|
nameLength,
|
|
393
439
|
opsPerSecLen: 0,
|
|
@@ -398,15 +444,20 @@ export class SimpleReporter extends BaseReporter {
|
|
|
398
444
|
};
|
|
399
445
|
}
|
|
400
446
|
|
|
401
|
-
const duration = BaseReporter.formatDuration(result.mean
|
|
447
|
+
const duration = BaseReporter.formatDuration(result.mean);
|
|
402
448
|
const opsPerSec = BaseReporter.formatOpsPerSecond(result.opsPerSecond);
|
|
403
449
|
const rme = BaseReporter.formatPercentage(result.marginOfError); // already a percentage
|
|
450
|
+
const iterationsStr = `(${result.iterations} iter)`;
|
|
451
|
+
const lowIterations = result.iterations < MIN_RELIABLE_ITERATIONS;
|
|
404
452
|
|
|
405
453
|
return {
|
|
406
454
|
durationLen: duration.length,
|
|
407
455
|
durationStr: duration,
|
|
408
456
|
error: false,
|
|
409
457
|
iterations: result.iterations,
|
|
458
|
+
iterationsLen: iterationsStr.length,
|
|
459
|
+
iterationsStr,
|
|
460
|
+
lowIterations,
|
|
410
461
|
name,
|
|
411
462
|
nameLength,
|
|
412
463
|
opsPerSecLen: opsPerSec.length,
|
|
@@ -434,6 +485,10 @@ export class SimpleReporter extends BaseReporter {
|
|
|
434
485
|
...formatted.filter((t) => !t.error).map((t) => t.rmeLen),
|
|
435
486
|
0,
|
|
436
487
|
);
|
|
488
|
+
const maxIterLen = Math.max(
|
|
489
|
+
...formatted.filter((t) => !t.error).map((t) => t.iterationsLen),
|
|
490
|
+
0,
|
|
491
|
+
);
|
|
437
492
|
const maxOpsLen = Math.max(
|
|
438
493
|
...formatted.filter((t) => !t.error).map((t) => t.opsPerSecLen),
|
|
439
494
|
0,
|
|
@@ -464,28 +519,32 @@ export class SimpleReporter extends BaseReporter {
|
|
|
464
519
|
const leadingPad = ' '.repeat(numbersStartPos);
|
|
465
520
|
const durationPad = ' '.repeat(maxDurationLen - task.durationLen);
|
|
466
521
|
const rmePad = ' '.repeat(maxRmeLen - task.rmeLen);
|
|
522
|
+
const iterPad = ' '.repeat(maxIterLen - task.iterationsLen);
|
|
467
523
|
const opsPad = ' '.repeat(maxOpsLen - task.opsPerSecLen);
|
|
468
524
|
|
|
469
525
|
console.log(
|
|
470
|
-
`${leadingPad}${durationPad}${task.durationStr} ${separator} ${symbols.plusMinus}${rmePad}${task.rmeStr} ${separator} ${opsPad}${task.opsPerSecStr}`,
|
|
526
|
+
`${leadingPad}${durationPad}${task.durationStr} ${separator} ${symbols.plusMinus}${rmePad}${task.rmeStr} ${iterPad}${task.iterationsStr} ${separator} ${opsPad}${task.opsPerSecStr}`,
|
|
471
527
|
);
|
|
472
528
|
|
|
473
|
-
|
|
474
|
-
|
|
529
|
+
// Track low iteration count
|
|
530
|
+
if (task.lowIterations) {
|
|
531
|
+
this.lowIterationCount++;
|
|
475
532
|
}
|
|
476
533
|
} else {
|
|
477
534
|
// Normal length - align on same line
|
|
478
535
|
const namePad = ' '.repeat(maxNameLen - task.nameLength);
|
|
479
536
|
const durationPad = ' '.repeat(maxDurationLen - task.durationLen);
|
|
480
537
|
const rmePad = ' '.repeat(maxRmeLen - task.rmeLen);
|
|
538
|
+
const iterPad = ' '.repeat(maxIterLen - task.iterationsLen);
|
|
481
539
|
const opsPad = ' '.repeat(maxOpsLen - task.opsPerSecLen);
|
|
482
540
|
|
|
483
541
|
console.log(
|
|
484
|
-
`${BASE_INDENT}${task.status} ${task.name}${namePad}: ${durationPad}${task.durationStr} ${separator} ${symbols.plusMinus}${rmePad}${task.rmeStr} ${separator} ${opsPad}${task.opsPerSecStr}`,
|
|
542
|
+
`${BASE_INDENT}${task.status} ${task.name}${namePad}: ${durationPad}${task.durationStr} ${separator} ${symbols.plusMinus}${rmePad}${task.rmeStr} ${iterPad}${task.iterationsStr} ${separator} ${opsPad}${task.opsPerSecStr}`,
|
|
485
543
|
);
|
|
486
544
|
|
|
487
|
-
|
|
488
|
-
|
|
545
|
+
// Track low iteration count
|
|
546
|
+
if (task.lowIterations) {
|
|
547
|
+
this.lowIterationCount++;
|
|
489
548
|
}
|
|
490
549
|
}
|
|
491
550
|
}
|
|
@@ -19,7 +19,7 @@ import type {
|
|
|
19
19
|
} from '../types/index.js';
|
|
20
20
|
|
|
21
21
|
import { safeParseConfig } from '../config/schema.js';
|
|
22
|
-
import { ErrorCodes } from '../constants.js';
|
|
22
|
+
import { DEFAULT_BENCHMARK_DIR, ErrorCodes } from '../constants.js';
|
|
23
23
|
import { ConfigLoadError, ConfigValidationError } from '../errors/index.js';
|
|
24
24
|
|
|
25
25
|
/**
|
|
@@ -56,7 +56,7 @@ const DEFAULT_CONFIG: ModestBenchConfig = {
|
|
|
56
56
|
limitBy: 'iterations', // Default to limiting by iteration count
|
|
57
57
|
metadata: {},
|
|
58
58
|
outputDir: undefined, // No default output directory - reporters will use stdout unless explicitly configured
|
|
59
|
-
pattern:
|
|
59
|
+
pattern: `${DEFAULT_BENCHMARK_DIR}/**/*.bench.{js,ts,mjs,cjs,mts,cts}`, // Search bench/ directory recursively
|
|
60
60
|
quiet: false,
|
|
61
61
|
reporterConfig: {},
|
|
62
62
|
reporters: [getDefaultReporter()],
|
|
@@ -320,9 +320,8 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
|
|
|
320
320
|
'.ts': async (filepath: string): Promise<unknown> => {
|
|
321
321
|
// Use cosmiconfig-typescript-loader to load TypeScript files
|
|
322
322
|
// This works without tsx in the import chain
|
|
323
|
-
const { TypeScriptLoader: createTypeScriptLoader } =
|
|
324
|
-
'cosmiconfig-typescript-loader'
|
|
325
|
-
);
|
|
323
|
+
const { TypeScriptLoader: createTypeScriptLoader } =
|
|
324
|
+
await import('cosmiconfig-typescript-loader');
|
|
326
325
|
const loader = createTypeScriptLoader();
|
|
327
326
|
const { readFile } = await import('node:fs/promises');
|
|
328
327
|
const content = await readFile(filepath, 'utf-8');
|
|
@@ -142,9 +142,8 @@ export class BenchmarkFileLoader implements FileLoader {
|
|
|
142
142
|
|
|
143
143
|
if (ext === '.ts' || ext === '.mts' || ext === '.cts') {
|
|
144
144
|
// For TypeScript files, use cosmiconfig-typescript-loader
|
|
145
|
-
const { TypeScriptLoader: createTypeScriptLoader } =
|
|
146
|
-
'cosmiconfig-typescript-loader'
|
|
147
|
-
);
|
|
145
|
+
const { TypeScriptLoader: createTypeScriptLoader } =
|
|
146
|
+
await import('cosmiconfig-typescript-loader');
|
|
148
147
|
const loader = createTypeScriptLoader();
|
|
149
148
|
module = (await loader(filePath, content)) as {
|
|
150
149
|
[key: string]: unknown;
|