fm-bench 0.3.0 → 0.4.0

package/src/progress.js

@@ -0,0 +1,190 @@
+import { formatMs } from './table.js';
+
+const UNICODE_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+const ASCII_FRAMES = ['-', '\\', '|', '/'];
+
+const TONES = {
+  green: ['\x1b[32m', '\x1b[0m'],
+  yellow: ['\x1b[33m', '\x1b[0m'],
+  red: ['\x1b[31m', '\x1b[0m'],
+  dim: ['\x1b[2m', '\x1b[0m']
+};
+
+export function createProgress(options = {}) {
+  const stream = options.stream || process.stderr;
+  const enabled = options.enabled === true || (options.enabled === 'auto' && Boolean(stream.isTTY));
+  if (!enabled) return noopProgress();
+  return new StatusLine({ ...options, stream });
+}
+
+function noopProgress() {
+  return {
+    update() {},
+    stop() {}
+  };
+}
+
+class StatusLine {
+  constructor(options = {}) {
+    this.stream = options.stream || process.stderr;
+    this.color = Boolean(options.color);
+    this.ascii = Boolean(options.ascii);
+    this.frames = this.ascii ? ASCII_FRAMES : UNICODE_FRAMES;
+    this.frameIndex = 0;
+    this.startedAt = Date.now();
+    this.state = {
+      phase: 'starting',
+      message: 'starting',
+      completed: 0,
+      failed: 0,
+      total: null
+    };
+    this.timer = setInterval(() => this.render(), 90);
+    this.timer.unref?.();
+    this.render();
+  }
+
+  update(event = {}) {
+    if (event.type === 'phase') {
+      this.state.phase = event.phase || 'working';
+      this.state.message = event.message || this.state.message;
+    } else if (event.type === 'tokens:start') {
+      this.state.phase = 'tokens';
+      this.state.message = event.message || 'counting prompt tokens';
+      this.state.completed = 0;
+      this.state.total = event.total;
+    } else if (event.type === 'tokens:progress') {
+      this.state.phase = 'tokens';
+      this.state.message = `counted ${event.promptId || 'prompt'}`;
+      this.state.completed = event.completed;
+      this.state.total = event.total;
+    } else if (event.type === 'benchmark:start') {
+      this.state.phase = 'benchmark';
+      this.state.message = `running ${event.modelCount} model(s), ${event.promptCount} prompt(s)`;
+      this.state.completed = 0;
+      this.state.failed = 0;
+      this.state.total = event.total;
+    } else if (event.type === 'warmup:start') {
+      this.state.phase = 'warmup';
+      this.state.message = `warming c${event.concurrency} (${event.scenarioIndex}/${event.scenarioCount})`;
+      this.state.completed = 0;
+      this.state.total = event.total;
+    } else if (event.type === 'warmup:progress') {
+      this.state.phase = 'warmup';
+      this.state.message = `warmed ${event.model || 'model'} at c${event.concurrency}`;
+      this.state.completed = event.completed;
+      this.state.total = event.total;
+    } else if (event.type === 'scenario:start') {
+      this.state.phase = 'benchmark';
+      this.state.message = `measuring c${event.concurrency} (${event.scenarioIndex}/${event.scenarioCount})`;
+      this.state.total = event.total ?? this.state.total;
+    } else if (event.type === 'benchmark:progress') {
+      this.state.phase = 'benchmark';
+      this.state.message = `${event.model} ${event.promptId} run ${event.run} ${event.ok ? 'ok' : 'failed'} (${formatMs(event.durationMs)})`;
+      this.state.completed = event.completed;
+      this.state.failed = event.failed;
+      this.state.total = event.total;
+    } else if (event.type === 'benchmark:complete') {
+      this.state.phase = 'complete';
+      this.state.message = `complete ${event.completed}/${event.total}`;
+      this.state.completed = event.completed;
+      this.state.failed = event.failed;
+      this.state.total = event.total;
+    }
+    this.render();
+  }
+
+  stop(finalMessage = '') {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    if (this.stream.clearLine && this.stream.cursorTo) {
+      this.clearLine();
+    } else {
+      this.stream.write('\n');
+    }
+    if (finalMessage) this.stream.write(`${finalMessage}\n`);
+  }
+
+  render() {
+    const width = this.stream.columns || process.stderr.columns || 100;
+    const elapsed = Date.now() - this.startedAt;
+    const frame = this.frames[this.frameIndex % this.frames.length];
+    this.frameIndex += 1;
+    const progress = progressText(this.state);
+    const eta = etaText(this.state, elapsed);
+    const failures = this.state.failed > 0 ? this.tone(`fail ${this.state.failed}`, 'red') : '';
+    const parts = [
+      this.tone(frame, 'green'),
+      'fm-bench',
+      this.state.phase,
+      progress,
+      eta,
+      failures,
+      this.tone(this.state.message, 'dim')
+    ].filter(Boolean);
+    this.writeLine(truncateVisible(parts.join('  '), Math.max(20, width - 1)));
+  }
+
+  writeLine(line) {
+    if (this.stream.clearLine && this.stream.cursorTo) {
+      this.stream.clearLine(0);
+      this.stream.cursorTo(0);
+      this.stream.write(line);
+    } else {
+      this.stream.write(`\r${line}`);
+    }
+  }
+
+  clearLine() {
+    if (this.stream.clearLine && this.stream.cursorTo) {
+      this.stream.clearLine(0);
+      this.stream.cursorTo(0);
+    } else {
+      this.stream.write('\r');
+    }
+  }
+
+  tone(text, tone) {
+    if (!this.color || !TONES[tone]) return text;
+    const [open, close] = TONES[tone];
+    return `${open}${text}${close}`;
+  }
+}
+
+function progressText(state) {
+  if (!Number.isFinite(state.total) || state.total <= 0) return '';
+  const completed = Math.min(state.completed || 0, state.total);
+  const percent = Math.round((completed / state.total) * 100);
+  return `${completed}/${state.total} ${percent}%`;
+}
+
+function etaText(state, elapsedMs) {
+  if (!Number.isFinite(state.total) || state.total <= 0 || !Number.isFinite(state.completed) || state.completed <= 0) {
+    return '';
+  }
+  const remaining = Math.max(0, state.total - state.completed);
+  if (remaining === 0) return `elapsed ${formatMs(elapsedMs)}`;
+  const perItemMs = elapsedMs / state.completed;
+  return `eta ${formatMs(perItemMs * remaining)}`;
+}
+
+function truncateVisible(value, width) {
+  const text = String(value);
+  const clean = text.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, '');
+  if (clean.length <= width) return text;
+  let visible = 0;
+  let output = '';
+  for (let index = 0; index < text.length && visible < width - 1; index += 1) {
+    if (text[index] === '\x1b') {
+      const match = text.slice(index).match(/^\u001b\[[0-?]*[ -/]*[@-~]/);
+      if (match) {
+        output += match[0];
+        index += match[0].length - 1;
+        continue;
+      }
+    }
+    output += text[index];
+    visible += 1;
+  }
+  return `${output}…`;
+}

package/src/prompts.js

@@ -50,6 +50,28 @@ const PROFILES = {
       prompt: 'Create a compact test plan for benchmarking a local foundation model across short, medium, and long prompts.'
     }
   ],
+  client: [
+    {
+      id: 'short-chat',
+      prompt: 'In one sentence, explain why time to first token matters for an interactive assistant.'
+    },
+    {
+      id: 'content-generation',
+      prompt: 'Write a practical 180-word product update for developers explaining a new terminal benchmark feature. Keep it specific and avoid marketing fluff.'
+    },
+    {
+      id: 'structured-extraction',
+      prompt: 'Return compact valid JSON with keys "risk", "owner", "deadline", and "next_step" from this note: The benchmark release is blocked by flaky p95 latency on the PCC model. Maya owns the investigation and needs a fix before Friday.'
+    },
+    {
+      id: 'summarization-light',
+      prompt: 'Summarize this in three bullets: A serious local LLM benchmark should separate time to first token from total latency, report tail percentiles, include prompt and output token counts, measure throughput at multiple concurrency operating points, and preserve the raw prompt suite so future runs are comparable.'
+    },
+    {
+      id: 'code-analysis',
+      prompt: 'Review this JavaScript function for one correctness issue and one readability improvement: function p(v){let s=0;for(let i=0;i<=v.length;i++)s+=v[i];return s/v.length}'
+    }
+  ],
   stress: [
     {
       id: 'interactive-short',

package/src/report.js

@@ -27,9 +27,13 @@ export function flattenResults(results) {
     words: result.words,
     tokens_per_second: result.tokensPerSecond == null ? '' : round(result.tokensPerSecond),
     decode_tokens_per_second: result.decodeTokensPerSecond == null ? '' : round(result.decodeTokensPerSecond),
+    prefill_tokens_per_second: result.prefillTokensPerSecond == null ? '' : round(result.prefillTokensPerSecond),
     chars_per_second: round(result.charsPerSecond),
     streamed: result.streamed,
     stdout_chunks: result.stdoutChunks,
+    second_chunk_ms: round(result.secondChunkMs),
+    chunk_gap_avg_ms: round(result.chunkGapAvgMs),
+    chunk_gap_max_ms: round(result.chunkGapMaxMs),
     output_hash: result.outputHash || '',
     good: result.good == null ? '' : result.good,
     error: result.error || ''

package/src/stats.js

@@ -101,10 +101,15 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
     const charsPerSecond = summarizeNumbers(successes.map((result) => result.charsPerSecond));
     const tokensPerSecond = summarizeNumbers(successes.map((result) => result.tokensPerSecond).filter((value) => value != null));
     const decodeTokensPerSecond = summarizeNumbers(successes.map((result) => result.decodeTokensPerSecond).filter((value) => value != null));
+    const prefillTokensPerSecond = summarizeNumbers(successes.map((result) => result.prefillTokensPerSecond).filter((value) => value != null));
+    const secondChunk = summarizeNumbers(successes.map((result) => result.secondChunkMs).filter((value) => value != null));
+    const chunkGap = summarizeNumbers(successes.flatMap((result) => result.chunkGapsMs || []));
     const windowMs = modelWindowMs(successes);
     const rps = successes.length > 0 && windowMs > 0 ? successes.length / (windowMs / 1000) : null;
-    const goodputRps = goodResults.length > 0 && windowMs > 0 ? goodResults.length / (windowMs / 1000) : null;
+    const goodputRps = goodMeasured.length > 0 && windowMs > 0 ? goodResults.length / (windowMs / 1000) : null;
     const outputTokenThroughput = outputTokens.sum > 0 && windowMs > 0 ? outputTokens.sum / (windowMs / 1000) : null;
+    const totalTokens = promptTokens.sum + outputTokens.sum;
+    const totalTokenThroughput = totalTokens > 0 && windowMs > 0 ? totalTokens / (windowMs / 1000) : null;
 
     return {
       model: entry.model,
@@ -120,6 +125,7 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
       rps,
       goodputRps,
       outputTokenThroughput,
+      totalTokenThroughput,
       repeatability: summarizeRepeatability(successes),
       latency,
       ttft,
@@ -129,7 +135,10 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
       outputTokens,
       charsPerSecond,
       tokensPerSecond,
-      decodeTokensPerSecond
+      decodeTokensPerSecond,
+      prefillTokensPerSecond,
+      secondChunk,
+      chunkGap
     };
   });
 }