fm-bench 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +1 -1
- package/src/table.js +163 -19
package/README.md
CHANGED
|
@@ -149,7 +149,7 @@ Token counts come from `fm token-count --quiet`. If `fm` cannot count a response
|
|
|
149
149
|
|
|
150
150
|
Measured runs stream by default so `fm-bench` can capture TTFT and streaming smoothness. Use `--no-stream` if you need buffered `fm respond` behavior; TTFT, TPOT, second-chunk, and chunk-gap fields that depend on streaming will be blank.
|
|
151
151
|
|
|
152
|
-
Terminal output is responsive. Wide terminals show full scoreboard and detail tables, medium terminals show a tighter operating-point table, and narrow terminals switch to compact model cards. Use `--width` to preview a layout and `--ascii` for log systems that do not render Unicode borders well.
|
|
152
|
+
Terminal output is responsive. Wide terminals show full scoreboard and detail tables, medium terminals show a tighter operating-point table, and narrow terminals switch to compact model cards. Long `NOTE`, `DESCRIPTION`, and model quota cells wrap so error context stays visible instead of being hidden behind ellipses. Use `--width` to preview a layout and `--ascii` for log systems that do not render Unicode borders well.
|
|
153
153
|
|
|
154
154
|
## Table Legend
|
|
155
155
|
|
|
@@ -161,6 +161,8 @@ fm-bench legend --json
|
|
|
161
161
|
fm-bench legend --csv
|
|
162
162
|
```
|
|
163
163
|
|
|
164
|
+
The legend wraps long definitions and rules to fit the terminal width instead of hiding text behind ellipses.
|
|
165
|
+
|
|
164
166
|
## Live Progress
|
|
165
167
|
|
|
166
168
|
Interactive terminal runs show a single-line status indicator on stderr while prompts are loaded, models are inspected, tokens are counted, warmups run, and benchmark jobs complete. The final report still prints to stdout, so `--json`, `--csv`, and `--out` remain automation-friendly.
|
package/package.json
CHANGED
package/src/table.js
CHANGED
|
@@ -3,24 +3,30 @@ import { stripAnsi } from './ansi.js';
|
|
|
3
3
|
export function renderTable(headers, rows, options = {}) {
|
|
4
4
|
const ascii = Boolean(options.ascii);
|
|
5
5
|
const maxCellWidth = options.maxCellWidth || 60;
|
|
6
|
-
const
|
|
6
|
+
const wrapColumns = normalizeWrapColumns(headers, options.wrapColumns);
|
|
7
|
+
const normalizedRows = rows.map((row) => row.map((cell, index) => {
|
|
7
8
|
const normalized = normalizeCell(cell);
|
|
8
9
|
return {
|
|
9
10
|
...normalized,
|
|
10
|
-
text: truncate(normalized.text, maxCellWidth)
|
|
11
|
+
text: wrapColumns.has(index) ? normalized.text : truncate(normalized.text, maxCellWidth)
|
|
11
12
|
};
|
|
12
13
|
}));
|
|
13
|
-
const widths = headers.map((header, index) => {
|
|
14
|
+
const widths = fitTableWidths(headers, headers.map((header, index) => {
|
|
14
15
|
const values = [truncate(header, maxCellWidth), ...normalizedRows.map((row) => row[index]?.text ?? '')];
|
|
16
|
+
if (wrapColumns.has(index)) {
|
|
17
|
+
return Math.max(...values.map((value) => Math.min(maxCellWidth, visibleLength(value))));
|
|
18
|
+
}
|
|
15
19
|
return Math.max(...values.map(visibleLength));
|
|
16
|
-
});
|
|
20
|
+
}), wrapColumns, terminalWidth(options));
|
|
17
21
|
const style = ascii ? ASCII_TABLE : UNICODE_TABLE;
|
|
18
22
|
|
|
19
23
|
const top = rule(style.topLeft, style.topJoin, style.topRight, style.horizontal, widths);
|
|
20
24
|
const middle = rule(style.midLeft, style.midJoin, style.midRight, style.horizontal, widths);
|
|
21
25
|
const bottom = rule(style.bottomLeft, style.bottomJoin, style.bottomRight, style.horizontal, widths);
|
|
22
26
|
const headerLine = rowLine(headers.map((header) => truncate(header, maxCellWidth)), widths, style, true, options);
|
|
23
|
-
const bodyLines =
|
|
27
|
+
const bodyLines = wrapColumns.size > 0
|
|
28
|
+
? normalizedRows.flatMap((row) => wrappedRowLines(row, widths, style, wrapColumns, options))
|
|
29
|
+
: normalizedRows.map((row) => rowLine(row, widths, style, false, options));
|
|
24
30
|
|
|
25
31
|
return [top, headerLine, middle, ...bodyLines, bottom].join('\n');
|
|
26
32
|
}
|
|
@@ -109,16 +115,10 @@ export function renderLegend(options = {}) {
|
|
|
109
115
|
]);
|
|
110
116
|
|
|
111
117
|
if (options.compact || width < 88) {
|
|
112
|
-
return
|
|
113
|
-
const suffix = item.rule ? ` Rule: ${item.rule}` : '';
|
|
114
|
-
return truncate(`${item.table.toUpperCase()} ${item.column}: ${item.definition}${suffix}`, width);
|
|
115
|
-
}).join('\n');
|
|
118
|
+
return renderCompactLegend(width);
|
|
116
119
|
}
|
|
117
120
|
|
|
118
|
-
return
|
|
119
|
-
...options,
|
|
120
|
-
maxCellWidth: Math.max(20, Math.floor((width - 48) / 2))
|
|
121
|
-
});
|
|
121
|
+
return renderWrappedTable(['table', 'column', 'definition', 'rule'], rows, legendColumnWidths(width), options);
|
|
122
122
|
}
|
|
123
123
|
|
|
124
124
|
export function formatMs(value) {
|
|
@@ -161,7 +161,7 @@ export function renderSummaryTable(summary, options = {}) {
|
|
|
161
161
|
cell(formatNumber(item.outputTokenThroughput), tones.systemTps),
|
|
162
162
|
cell(formatNumber(item.rps), tones.rps),
|
|
163
163
|
cell(formatPercent(item.latency.cv), cvTone(item.latency.cv)),
|
|
164
|
-
cell(item.available ? '' :
|
|
164
|
+
cell(item.available ? '' : cleanReason(item.skippedReason), item.available ? null : 'yellow')
|
|
165
165
|
];
|
|
166
166
|
|
|
167
167
|
if (mode === 'medium') {
|
|
@@ -200,7 +200,7 @@ export function renderSummaryTable(summary, options = {}) {
|
|
|
200
200
|
? (hasGoodput ? mediumHeaders : mediumHeaders.filter((header) => header !== 'good' && header !== 'good rps'))
|
|
201
201
|
: (hasGoodput ? wideHeaders : wideHeaders.filter((header) => header !== 'good' && header !== 'good rps'));
|
|
202
202
|
|
|
203
|
-
return renderTable(headers, rows, { ...options, maxCellWidth: mode === 'medium' ? 24 : 52 });
|
|
203
|
+
return renderTable(headers, rows, { ...options, maxCellWidth: mode === 'medium' ? 24 : 52, wrapColumns: ['note'] });
|
|
204
204
|
}
|
|
205
205
|
|
|
206
206
|
export function renderDetailTable(summary, options = {}) {
|
|
@@ -247,7 +247,7 @@ export function renderDetailTable(summary, options = {}) {
|
|
|
247
247
|
'description'
|
|
248
248
|
];
|
|
249
249
|
|
|
250
|
-
return renderTable(headers, rows, { ...options, maxCellWidth: mode === 'medium' ? 34 : 52 });
|
|
250
|
+
return renderTable(headers, rows, { ...options, maxCellWidth: mode === 'medium' ? 34 : 52, wrapColumns: ['description'] });
|
|
251
251
|
}
|
|
252
252
|
|
|
253
253
|
export function renderCompactSummary(summary, options = {}) {
|
|
@@ -292,8 +292,8 @@ export function renderModelsTable(models, options = {}) {
|
|
|
292
292
|
cell(model.name),
|
|
293
293
|
cell(model.available ? 'yes' : 'no', model.available ? 'green' : 'yellow'),
|
|
294
294
|
model.description || '-',
|
|
295
|
-
|
|
296
|
-
]), options);
|
|
295
|
+
cleanReason(model.quota || model.reason || '-')
|
|
296
|
+
]), { ...options, wrapColumns: ['description', 'quota'] });
|
|
297
297
|
}
|
|
298
298
|
|
|
299
299
|
function formatRangeMs(low, high) {
|
|
@@ -358,6 +358,89 @@ function rule(left, join, right, horizontal, widths) {
|
|
|
358
358
|
return `${left}${widths.map((width) => horizontal.repeat(width + 2)).join(join)}${right}`;
|
|
359
359
|
}
|
|
360
360
|
|
|
361
|
+
function renderWrappedTable(headers, rows, widths, options = {}) {
|
|
362
|
+
const ascii = Boolean(options.ascii);
|
|
363
|
+
const style = ascii ? ASCII_TABLE : UNICODE_TABLE;
|
|
364
|
+
const top = rule(style.topLeft, style.topJoin, style.topRight, style.horizontal, widths);
|
|
365
|
+
const middle = rule(style.midLeft, style.midJoin, style.midRight, style.horizontal, widths);
|
|
366
|
+
const bottom = rule(style.bottomLeft, style.bottomJoin, style.bottomRight, style.horizontal, widths);
|
|
367
|
+
const headerLine = rowLine(headers.map((header) => truncate(header, widths[headers.indexOf(header)])), widths, style, true, options);
|
|
368
|
+
const bodyLines = [];
|
|
369
|
+
|
|
370
|
+
for (const row of rows) {
|
|
371
|
+
const wrappedCells = row.map((value, index) => wrapText(formatCell(value), widths[index]));
|
|
372
|
+
const height = Math.max(...wrappedCells.map((lines) => lines.length));
|
|
373
|
+
for (let lineIndex = 0; lineIndex < height; lineIndex += 1) {
|
|
374
|
+
bodyLines.push(rowLine(wrappedCells.map((lines) => lines[lineIndex] ?? ''), widths, style, false, options));
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
return [top, headerLine, middle, ...bodyLines, bottom].join('\n');
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function normalizeWrapColumns(headers, wrapColumns = []) {
|
|
382
|
+
const normalized = new Set();
|
|
383
|
+
for (const column of wrapColumns || []) {
|
|
384
|
+
if (Number.isInteger(column)) {
|
|
385
|
+
if (column >= 0 && column < headers.length) normalized.add(column);
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
const index = headers.findIndex((header) => String(header).toLowerCase() === String(column).toLowerCase());
|
|
390
|
+
if (index >= 0) normalized.add(index);
|
|
391
|
+
}
|
|
392
|
+
return normalized;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function fitTableWidths(headers, widths, wrapColumns, targetWidth) {
|
|
396
|
+
if (!targetWidth || wrapColumns.size === 0 || renderedTableWidth(widths) <= targetWidth) return widths;
|
|
397
|
+
|
|
398
|
+
const fitted = [...widths];
|
|
399
|
+
const minimums = fitted.map((width, index) => (
|
|
400
|
+
wrapColumns.has(index)
|
|
401
|
+
? Math.max(visibleLength(String(headers[index]).toUpperCase()), Math.min(12, width))
|
|
402
|
+
: width
|
|
403
|
+
));
|
|
404
|
+
let excess = renderedTableWidth(fitted) - targetWidth;
|
|
405
|
+
|
|
406
|
+
while (excess > 0) {
|
|
407
|
+
const candidates = [...wrapColumns]
|
|
408
|
+
.filter((index) => fitted[index] > minimums[index])
|
|
409
|
+
.sort((a, b) => fitted[b] - fitted[a]);
|
|
410
|
+
if (candidates.length === 0) break;
|
|
411
|
+
|
|
412
|
+
for (const index of candidates) {
|
|
413
|
+
if (excess <= 0) break;
|
|
414
|
+
fitted[index] -= 1;
|
|
415
|
+
excess -= 1;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
return fitted;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function renderedTableWidth(widths) {
|
|
423
|
+
return widths.reduce((sum, width) => sum + width, 0) + (widths.length * 3) + 1;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function wrappedRowLines(row, widths, style, wrapColumns, options = {}) {
|
|
427
|
+
const wrappedCells = row.map((value, index) => {
|
|
428
|
+
const normalized = normalizeCell(value);
|
|
429
|
+
return {
|
|
430
|
+
tone: normalized.tone,
|
|
431
|
+
lines: wrapColumns.has(index) ? wrapText(normalized.text, widths[index]) : [normalized.text]
|
|
432
|
+
};
|
|
433
|
+
});
|
|
434
|
+
const height = Math.max(...wrappedCells.map((item) => item.lines.length));
|
|
435
|
+
const lines = [];
|
|
436
|
+
|
|
437
|
+
for (let lineIndex = 0; lineIndex < height; lineIndex += 1) {
|
|
438
|
+
lines.push(rowLine(wrappedCells.map((item) => cell(item.lines[lineIndex] ?? '', item.tone)), widths, style, false, options));
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
return lines;
|
|
442
|
+
}
|
|
443
|
+
|
|
361
444
|
function rowLine(row, widths, style, header = false, options = {}) {
|
|
362
445
|
return `${style.vertical}${row.map((cell, index) => {
|
|
363
446
|
const normalized = header ? { text: String(cell).toUpperCase(), tone: 'header' } : normalizeCell(cell);
|
|
@@ -403,12 +486,73 @@ function visibleLength(value) {
|
|
|
403
486
|
return stripAnsi(value).length;
|
|
404
487
|
}
|
|
405
488
|
|
|
489
|
+
function cleanReason(value) {
|
|
490
|
+
return String(value || '').replace(/\s+/g, ' ').trim();
|
|
491
|
+
}
|
|
492
|
+
|
|
406
493
|
function compactReason(value) {
|
|
407
|
-
const clean =
|
|
494
|
+
const clean = cleanReason(value);
|
|
408
495
|
if (clean.length <= 58) return clean;
|
|
409
496
|
return `${clean.slice(0, 55)}...`;
|
|
410
497
|
}
|
|
411
498
|
|
|
499
|
+
function renderCompactLegend(width) {
|
|
500
|
+
const lines = [];
|
|
501
|
+
for (const item of legendEntries()) {
|
|
502
|
+
lines.push(...wrapText(`${item.table.toUpperCase()} ${item.column}`, width));
|
|
503
|
+
lines.push(...wrapText(` Definition: ${item.definition}`, width));
|
|
504
|
+
if (item.rule) {
|
|
505
|
+
lines.push(...wrapText(` Rule: ${item.rule}`, width));
|
|
506
|
+
}
|
|
507
|
+
lines.push('');
|
|
508
|
+
}
|
|
509
|
+
if (lines.at(-1) === '') lines.pop();
|
|
510
|
+
return lines.join('\n');
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
function legendColumnWidths(width) {
|
|
514
|
+
const available = Math.max(40, width - 13);
|
|
515
|
+
const tableWidth = 7;
|
|
516
|
+
const columnWidth = Math.min(25, Math.max(18, Math.floor(available * 0.28)));
|
|
517
|
+
const remaining = Math.max(36, available - tableWidth - columnWidth);
|
|
518
|
+
const definitionWidth = Math.max(18, Math.floor(remaining * 0.52));
|
|
519
|
+
const ruleWidth = Math.max(18, remaining - definitionWidth);
|
|
520
|
+
return [tableWidth, columnWidth, definitionWidth, ruleWidth];
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
function wrapText(value, width) {
|
|
524
|
+
const text = String(value ?? '').replace(/\s+/g, ' ').trim();
|
|
525
|
+
if (!text) return [''];
|
|
526
|
+
if (width <= 1) return [text];
|
|
527
|
+
|
|
528
|
+
const lines = [];
|
|
529
|
+
let current = '';
|
|
530
|
+
for (const rawWord of text.split(' ')) {
|
|
531
|
+
const pieces = splitLongWord(rawWord, width);
|
|
532
|
+
for (const word of pieces) {
|
|
533
|
+
if (!current) {
|
|
534
|
+
current = word;
|
|
535
|
+
} else if (visibleLength(`${current} ${word}`) <= width) {
|
|
536
|
+
current = `${current} ${word}`;
|
|
537
|
+
} else {
|
|
538
|
+
lines.push(current);
|
|
539
|
+
current = word;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
if (current) lines.push(current);
|
|
544
|
+
return lines.length ? lines : [''];
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function splitLongWord(word, width) {
|
|
548
|
+
if (visibleLength(word) <= width) return [word];
|
|
549
|
+
const pieces = [];
|
|
550
|
+
for (let index = 0; index < word.length; index += width) {
|
|
551
|
+
pieces.push(word.slice(index, index + width));
|
|
552
|
+
}
|
|
553
|
+
return pieces;
|
|
554
|
+
}
|
|
555
|
+
|
|
412
556
|
function truncate(value, width) {
|
|
413
557
|
const text = String(value ?? '');
|
|
414
558
|
if (visibleLength(text) <= width) return text;
|