fm-bench 0.3.0 → 0.4.0

package/README.md

@@ -2,7 +2,7 @@
 
 `fm-bench` is a dynamic benchmark CLI for Apple's `fm` command on macOS 27 and newer.
 
-It discovers the models reported by `fm --help`, checks availability with `fm available`, runs repeatable prompt suites through `fm respond`, counts tokens with `fm token-count`, and prints a terminal table with latency and throughput stats.
+It discovers the models reported by `fm --help`, checks availability with `fm available`, runs repeatable prompt suites through `fm respond`, counts tokens with `fm token-count`, shows live progress while it works, and prints terminal tables with latency, throughput, stability, goodput, and streaming-quality stats.
 
 Apple introduced the preinstalled `fm` command for macOS 27 as part of the Foundation Models tooling. `fm-bench` intentionally shells out to the system `fm` binary instead of linking private APIs, so it can adapt as Apple adds models or changes availability.
 
@@ -35,20 +35,15 @@ fm-bench
 Example output:
 
 ```text
-fm-bench 0.2.0 | darwin/arm64 | fm
-prompts 3 | runs 1 | concurrency 1 | stream on | measured 3 | failed 0 | skipped models 0 | elapsed 11.36s
-
-┌────────┬────────┬──────┬────┬─────────┬──────────┬──────────┬─────────┬─────────┬──────────┬───────┬─────┬──────┐
-│ MODEL  │ STATUS │ RUNS │ OK │ SUCCESS │ TTFT P50 │ TTFT P95 │ E2E P50 │ E2E P95 │ TPOT P50 │ TOK/S │ RPS │ NOTE │
-├────────┼────────┼──────┼────┼─────────┼──────────┼──────────┼─────────┼─────────┼──────────┼───────┼─────┼──────┤
-│ system │ ok     │    3 │  3 │    100% │    409ms │    486ms │   2.29s │   5.97s │     14ms │  58.5 │ 0.3 │      │
-└────────┴────────┴──────┴────┴─────────┴──────────┴──────────┴─────────┴─────────┴──────────┴───────┴─────┴──────┘
-
-┌────────┬────────────┬─────────────┬─────────────┬──────────────┬─────────┬──────────┬────────┬──────────────────────────────────┐
-│ MODEL  │ IN TOK AVG │ OUT TOK AVG │ TOTAL TOK/S │ DECODE TOK/S │ E2E P99 │ TPOT P95 │ REPEAT │ DESCRIPTION                      │
-├────────┼────────────┼─────────────┼─────────────┼──────────────┼─────────┼──────────┼────────┼──────────────────────────────────┤
-│ system │         35 │         196 │        59.5 │         75.0 │   6.30s │     15ms │      - │ On-device Apple Foundation Model │
-└────────┴────────────┴─────────────┴─────────────┴──────────────┴─────────┴──────────┴────────┴──────────────────────────────────┘
+fm-bench 0.4.0 | darwin/arm64 | fm
+prompts 5 | runs 3 | concurrency 1,2 | stream on | measured 30 | failed 0 | skipped 0 | elapsed 42.10s | SLO TTFT<=750ms,E2E<=4.00s
+
+┌───┬────────┬────────┬─────────┬──────┬──────┬──────────┬──────┬──────────┬─────┬─────┐
+│ C │ MODEL  │ STATUS │ OK/RUNS │ SUCC │ GOOD │ GOOD RPS │ TTFT │ E2E P95  │ SYS │ CV  │
+├───┼────────┼────────┼─────────┼──────┼──────┼──────────┼──────┼──────────┼─────┼─────┤
+│ 1 │ system │ ok     │   15/15 │ 100% │  93% │      0.4 │ 318ms│    3.20s │  42 │ 12% │
+│ 2 │ system │ ok     │   15/15 │ 100% │  80% │      0.7 │ 501ms│    4.40s │  68 │ 21% │
+└───┴────────┴────────┴─────────┴──────┴──────┴──────────┴──────┴──────────┴─────┴─────┘
 ```
 
 ## Commands
@@ -72,6 +67,7 @@ fm-bench --models system,pcc --runs 3 --profile stress
 fm-bench --models system --runs 5 --profile interactive
 fm-bench --models system --runs 3 --profile throughput --warmup 1
 fm-bench --models system --profile interactive --sweep-concurrency 1,2,4
+fm-bench --profile client --sweep-concurrency 1,2 --request-rate 0.5 --ramp-up-ms 2000
 fm-bench --models system --runs 5 --slo-ttft-ms 750 --slo-e2e-ms 4000
 fm-bench --prompt "Reply with exactly: ok" --runs 5
 fm-bench --prompt-file prompts.json --format json --out reports/bench.json
@@ -85,9 +81,11 @@ Useful flags:
 - `--warmup <n>`: warmup runs per model before measurement.
 - `--concurrency <n>`: parallel `fm` processes.
 - `--sweep-concurrency <list>`: run separate measured operating points, such as `1,2,4`.
+- `--request-rate <rps>`: pace request starts at a target requests-per-second rate.
+- `--ramp-up-ms <n>`: gradually ramp request pacing over `n` milliseconds.
 - `--timeout-ms <n>`: timeout per `fm` call.
 - `--slo-ttft-ms <n>`, `--slo-e2e-ms <n>`, `--slo-tpot-ms <n>`: count goodput against latency budgets.
-- `--profile quick|standard|interactive|throughput|stress`: built-in prompt suite.
+- `--profile quick|standard|interactive|throughput|client|stress`: built-in prompt suite.
 - `--prompt <text>`: custom prompt, repeatable.
 - `--prompt-file <file>`: JSON, JSONL, or blank-line separated text prompts.
 - `--instructions <text>`: passed to `fm respond`.
@@ -95,6 +93,8 @@ Useful flags:
 - `--capture-output`: include raw model output in JSON reports.
 - `--json`, `--csv`, `--format table|json|csv`: choose output format.
 - `--ascii`: use plain ASCII table borders.
+- `--color`, `--no-color`: force or disable semantic ANSI colors. Colors are automatic on TTYs.
+- `--progress`, `--no-progress`: force or disable the live progress status line on stderr.
 - `--compact`: force the narrow terminal layout.
 - `--width <n>`: render as if the terminal has `n` columns.
 - `--out <file>`: save a report.
@@ -126,8 +126,11 @@ Plain text files are split on blank lines.
 - TTFT, or time to first streamed output.
 - E2E latency, or full response wall-clock latency.
 - TPOT, or decode time per output token after the first output token.
+- second-chunk delay and chunk-gap p95 as terminal-side streaming smoothness signals.
+- prefill tokens per second, or prompt tokens divided by TTFT.
 - output tokens per second per request.
 - total output token throughput across the measured window.
+- total token throughput, including prompt and output tokens.
 - requests per second across the measured window.
 - goodput percentage and goodput RPS when SLO flags are set.
 - coefficient of variation (CV) and confidence interval context for stability.
@@ -139,10 +142,28 @@ Plain text files are split on blank lines.
 
 Token counts come from `fm token-count --quiet`. If `fm` cannot count a response, token fields are left blank while character throughput is still reported.
 
-Measured runs stream by default so `fm-bench` can capture TTFT. Use `--no-stream` if you need buffered `fm respond` behavior; TTFT and TPOT fields that depend on streaming will be blank.
+Measured runs stream by default so `fm-bench` can capture TTFT and streaming smoothness. Use `--no-stream` if you need buffered `fm respond` behavior; TTFT, TPOT, second-chunk, and chunk-gap fields that depend on streaming will be blank.
 
 Terminal output is responsive. Wide terminals show full scoreboard and detail tables, medium terminals show a tighter operating-point table, and narrow terminals switch to compact model cards. Use `--width` to preview a layout and `--ascii` for log systems that do not render Unicode borders well.
 
+## Live Progress
+
+Interactive terminal runs show a single-line status indicator on stderr while prompts are loaded, models are inspected, tokens are counted, warmups run, and benchmark jobs complete. The final report still prints to stdout, so `--json`, `--csv`, and `--out` remain automation-friendly.
+
+Progress is automatic for table output on TTYs. Use `--progress` to force it or `--no-progress` to keep the terminal completely quiet until the report is ready.
+
+## Terminal Colors
+
+Table output uses semantic ANSI color on interactive terminals:
+
+- green: passing, steadier, or better than the current comparison set.
+- yellow: marginal, partial, or near a budget.
+- red: failing a budget, unstable, or slower/lower than peers.
+
+Success rate, goodput, repeatability, and CV use fixed benchmark thresholds. Throughput columns use relative ranking within the current run because “good” depends on the machine, model, prompt mix, and concurrency. TTFT, E2E, and TPOT use SLO thresholds when you pass `--slo-ttft-ms`, `--slo-e2e-ms`, or `--slo-tpot-ms`; otherwise they use lower-is-better relative ranking across the models and operating points in the report.
+
+Use `--color` to force ANSI colors in captured logs, or `--no-color` for plain output. `NO_COLOR=1` disables automatic color and `FORCE_COLOR=1` enables it.
+
 See [docs/methodology.md](docs/methodology.md) for the benchmark methodology and source references.
 
 ## Requirements

package/docs/methodology.md

@@ -9,6 +9,7 @@ The metric set follows common LLM inference benchmark practice:
 - Apple introduces the macOS 27 `fm` command as a preinstalled way to use Foundation Models from the terminal and scripts: <https://developer.apple.com/videos/play/wwdc2026/334/>
 - NVIDIA NIM benchmarking defines TTFT, end-to-end latency, inter-token latency / TPOT, tokens per second, and requests per second: <https://docs.nvidia.com/nim/benchmarking/llm/latest/metrics.html>
 - NVIDIA GenAI-Perf reports TTFT, inter-token latency, request latency, sequence lengths, output token throughput, and JSON/CSV artifacts: <https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/perf_analyzer/genai-perf/README.html>
+- NVIDIA AIPerf documents time to second token, inter-token latency, inter-chunk latency, per-user output throughput, and prefill throughput: <https://docs.nvidia.com/aiperf/reference/ai-perf-metrics-reference>
 - vLLM benchmark tooling reports TTFT, TPOT, ITL, E2E percentiles and SLO-oriented goodput: <https://docs.vllm.ai/en/stable/cli/bench/serve/>
 - MLCommons describes varying concurrency and reporting verified operating points for TTFT, throughput, interactivity, and response latency rather than interpolated performance: <https://mlcommons.org/2026/03/mlperf-endpoints-gen-ai-benchmarking/>
 - MLPerf Client emphasizes local client workloads with multiple task types and varying prompt/response lengths: <https://mlcommons.org/benchmarks/client/>
@@ -19,11 +20,16 @@ The metric set follows common LLM inference benchmark practice:
 - `E2E latency`: time from starting `fm respond` until the process exits and the full response is captured.
 - `generation_ms`: `E2E - TTFT`.
 - `TPOT`: `(E2E - TTFT) / (output_tokens - 1)`. The first output token is excluded so TPOT focuses on decode cadence.
+- `second_chunk_ms`: time between the first and second streamed stdout chunks. This is a terminal-side proxy for time-to-second-token style startup smoothness.
+- `chunk_gap`: the distribution of time between consecutive streamed stdout chunks. It is useful for spotting streaming jitter, but it is chunk-based rather than token-based because the `fm` CLI writes stdout chunks, not token timestamp events.
+- `prefill_tokens_per_second`: input prompt tokens divided by TTFT seconds. This estimates prompt-processing speed for streaming runs.
 - `tokens_per_second`: output tokens divided by E2E seconds for one request.
 - `decode_tokens_per_second`: output tokens after the first token divided by generation seconds.
 - `total output token throughput`: all successful output tokens for a model divided by that model's measured wall-clock window.
+- `total token throughput`: successful prompt and output tokens divided by that model's measured wall-clock window.
 - `RPS`: successful requests divided by that model's measured wall-clock window.
 - `goodput`: successful requests that also satisfy all provided SLO thresholds.
+- `goodput RPS`: SLO-passing requests divided by that model's measured wall-clock window. If SLOs are set and no requests pass, this is reported as zero.
 - `repeatability`: for repeated runs of the same prompt, the average share of runs that produced the most common normalized output hash.
 - `CV`: coefficient of variation, or sample standard deviation divided by the mean. Lower values indicate steadier latency for that metric.
 - `95% CI`: a t-distribution confidence interval around the sample mean. Treat it as useful context, not proof, especially with very small sample sizes.
@@ -32,12 +38,20 @@ The metric set follows common LLM inference benchmark practice:
 
 Use `--sweep-concurrency 1,2,4` to measure separate concurrency operating points. This follows the same idea as MLCommons endpoint reporting: a single peak number hides the tradeoff between system throughput and per-user responsiveness.
 
+Use `--request-rate <rps>` to pace request starts independently of concurrency. Concurrency limits how many `fm respond` processes can be active at once; request rate controls how quickly new work is admitted. Use `--ramp-up-ms` to avoid instantly shocking a model or quota path when you start a higher-rate run.
+
 `fm-bench` does not interpolate between operating points. It reports only what was actually measured.
 
+## Prompt Profiles
+
+The `client` profile is a pragmatic local-machine mix inspired by MLPerf Client's emphasis on multiple task categories and prompt/response lengths. It includes short chat, content generation, structured extraction, light summarization, and code analysis prompts. It is not a formal MLPerf submission suite; it is a convenient built-in workload for comparing your own Mac, OS build, and `fm` models over time.
+
 ## Caveats
 
 `fm-bench` uses `fm token-count --quiet` as the source of token counts, so token values follow Apple's local tokenizer behavior. It does not judge semantic quality unless you provide your own prompt suite and inspect captured outputs with `--capture-output`.
 
 Client-side measurements include process startup, local queueing, model prefill, streaming, detokenization, and terminal pipe overhead. That is intentional for a command-line benchmark, but it is not the same as an internal model-kernel benchmark.
 
-For serious comparisons, prefer at least three runs per prompt, include warmups, benchmark both interactive and throughput profiles, and compare models at the same concurrency operating points.
+Stream smoothness metrics use stdout chunk arrival times. A chunk can contain more than one token, and terminal or pipe buffering can affect chunk boundaries. Treat `second_chunk_ms` and `chunk_gap` as user-visible streaming diagnostics, not raw decoder telemetry.
+
+For serious comparisons, prefer at least three runs per prompt, include warmups, benchmark both interactive and throughput or client profiles, compare models at the same concurrency operating points, set SLOs that match your real UX budget, and save JSON reports for later analysis.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fm-bench",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Dynamic benchmark CLI for Apple's fm command on macOS 27+.",
   "type": "module",
   "bin": {

package/src/bench.js

@@ -36,7 +36,9 @@ export async function inspectModels(options = {}) {
 
 export async function runBenchmark(options = {}) {
   const startedAt = new Date().toISOString();
+  notify(options, { type: 'phase', phase: 'prompts', message: 'loading prompts' });
   const prompts = await loadPrompts(options);
+  notify(options, { type: 'phase', phase: 'models', message: 'discovering models' });
   const inspection = await inspectModels(options);
   const modelStatuses = options.availableOnly
     ? inspection.models.filter((model) => model.available)
@@ -45,15 +47,36 @@ export async function runBenchmark(options = {}) {
   const environment = await collectEnvironment(inspection.fmBin);
   const promptTokenCounts = new Map();
   const concurrencies = normalizeConcurrencySweep(options);
+  const totalRuns = concurrencies.length * runnableModels.length * prompts.length * options.runs;
 
+  notify(options, {
+    type: 'tokens:start',
+    total: prompts.length,
+    message: 'counting prompt tokens'
+  });
   for (const prompt of prompts) {
     const counted = await countTokens(inspection.fmBin, prompt.prompt, options);
     promptTokenCounts.set(prompt.id, counted.ok ? counted.count : null);
+    notify(options, {
+      type: 'tokens:progress',
+      completed: promptTokenCounts.size,
+      total: prompts.length,
+      promptId: prompt.id
+    });
   }
 
   const results = [];
   const scenarios = [];
-  for (const concurrency of concurrencies) {
+  let completedRuns = 0;
+  let failedRuns = 0;
+  notify(options, {
+    type: 'benchmark:start',
+    total: totalRuns,
+    modelCount: runnableModels.length,
+    promptCount: prompts.length,
+    scenarioCount: concurrencies.length
+  });
+  for (const [scenarioIndex, concurrency] of concurrencies.entries()) {
     const scenario = await runScenario({
       fmBin: inspection.fmBin,
       prompts,
@@ -61,7 +84,26 @@ export async function runBenchmark(options = {}) {
       modelStatuses,
       promptTokenCounts,
       options,
-      concurrency
+      concurrency,
+      scenarioIndex: scenarioIndex + 1,
+      scenarioCount: concurrencies.length,
+      onMeasuredResult: (result) => {
+        completedRuns += 1;
+        if (!result.ok) failedRuns += 1;
+        notify(options, {
+          type: 'benchmark:progress',
+          completed: completedRuns,
+          failed: failedRuns,
+          total: totalRuns,
+          concurrency,
+          model: result.model,
+          promptId: result.promptId,
+          run: result.run,
+          ok: result.ok,
+          durationMs: result.durationMs,
+          firstTokenMs: result.firstTokenMs
+        });
+      }
     });
     scenarios.push(scenario);
     results.push(...scenario.results);
@@ -73,7 +115,7 @@ export async function runBenchmark(options = {}) {
     || a.run - b.run);
 
   const summary = summarizeByModel(results, modelStatuses, { concurrencies });
-  return {
+  const payload = {
     tool: 'fm-bench',
     version: options.version,
     startedAt,
@@ -90,6 +132,13 @@ export async function runBenchmark(options = {}) {
     summary,
     results
   };
+  notify(options, {
+    type: 'benchmark:complete',
+    completed: completedRuns,
+    failed: failedRuns,
+    total: totalRuns
+  });
+  return payload;
 }
 
 async function runScenario(context) {
@@ -100,16 +149,40 @@ async function runScenario(context) {
     modelStatuses,
     promptTokenCounts,
     options,
-    concurrency
+    concurrency,
+    scenarioIndex,
+    scenarioCount,
+    onMeasuredResult
   } = context;
   const startedAt = new Date().toISOString();
 
+  const warmupTotal = options.warmup * runnableModels.length;
+  if (warmupTotal > 0) {
+    notify(options, {
+      type: 'warmup:start',
+      concurrency,
+      scenarioIndex,
+      scenarioCount,
+      total: warmupTotal
+    });
+  }
+  let warmupCompleted = 0;
   for (let warmupIndex = 0; warmupIndex < options.warmup; warmupIndex += 1) {
     for (const model of runnableModels) {
       await respond(fmBin, model.name, prompts[0].prompt, {
         ...options,
         stream: false
       });
+      warmupCompleted += 1;
+      notify(options, {
+        type: 'warmup:progress',
+        concurrency,
+        scenarioIndex,
+        scenarioCount,
+        completed: warmupCompleted,
+        total: warmupTotal,
+        model: model.name
+      });
     }
   }
 
@@ -124,15 +197,23 @@ async function runScenario(context) {
   }
 
   const results = [];
+  notify(options, {
+    type: 'scenario:start',
+    concurrency,
+    scenarioIndex,
+    scenarioCount,
+    total: jobs.length
+  });
   await runLimited(jobs, concurrency, async (job) => {
     const result = await runSingleBenchmark(fmBin, job, promptTokenCounts, options, benchmarkStartedAt);
     results.push(result);
+    if (onMeasuredResult) onMeasuredResult(result);
     if (!result.ok && options.failFast) {
       const error = new Error(result.error || `Benchmark failed for ${job.model.name}`);
       error.exitCode = 1;
       throw error;
     }
-  });
+  }, options);
 
   results.sort((a, b) => a.model.localeCompare(b.model)
     || a.promptId.localeCompare(b.promptId)
@@ -171,6 +252,12 @@ async function runSingleBenchmark(fmBin, job, promptTokenCounts, options, benchm
     : null;
   const chars = response.output.length;
   const words = response.output.trim() ? response.output.trim().split(/\s+/).length : 0;
+  const promptTokens = promptTokenCounts.get(job.prompt.id);
+  const prefillTokensPerSecond = promptTokens != null && firstTokenMs > 0
+    ? promptTokens / (firstTokenMs / 1000)
+    : null;
+  const chunkGapsMs = chunkGaps(response.stdoutChunkTimesMs);
+  const secondChunkMs = chunkGapsMs.length > 0 ? chunkGapsMs[0] : null;
 
   return {
     model: job.model.name,
@@ -182,17 +269,22 @@ async function runSingleBenchmark(fmBin, job, promptTokenCounts, options, benchm
     firstTokenMs,
     generationMs,
     tpotMs,
-    promptTokens: promptTokenCounts.get(job.prompt.id),
+    promptTokens,
     outputTokens: countedOutputTokens,
     chars,
     words,
     tokensPerSecond: countedOutputTokens != null && seconds > 0 ? countedOutputTokens / seconds : null,
     decodeTokensPerSecond,
+    prefillTokensPerSecond,
     charsPerSecond: seconds > 0 ? chars / seconds : 0,
     startOffsetMs,
     endOffsetMs,
     streamed: response.streamed,
     stdoutChunks: response.stdoutChunks,
+    secondChunkMs,
+    chunkGapsMs,
+    chunkGapAvgMs: average(chunkGapsMs),
+    chunkGapMaxMs: chunkGapsMs.length > 0 ? Math.max(...chunkGapsMs) : null,
     outputHash: response.ok ? hashOutput(response.output) : null,
     good: response.ok ? evaluateSlo({
       firstTokenMs,
@@ -204,12 +296,14 @@ async function runSingleBenchmark(fmBin, job, promptTokenCounts, options, benchm
   };
 }
 
-async function runLimited(items, concurrency, worker) {
+async function runLimited(items, concurrency, worker, options = {}) {
   let nextIndex = 0;
+  const waitForSlot = createPacer(options.requestRate, options.rampUpMs);
   const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
     while (nextIndex < items.length) {
       const index = nextIndex;
       nextIndex += 1;
+      await waitForSlot(index);
       await worker(items[index]);
     }
   });
@@ -233,6 +327,8 @@ function publicOptions(options) {
     concurrency: options.concurrency,
     sweepConcurrency: concurrencies.length > 1 ? concurrencies : [],
     timeoutMs: options.timeoutMs,
+    requestRate: options.requestRate || null,
+    rampUpMs: options.rampUpMs || null,
     profile: options.profile,
     promptCount: options.promptCount,
     greedy: options.greedy,
@@ -262,6 +358,53 @@ function hashOutput(output) {
     .slice(0, 16);
 }
 
+function chunkGaps(times = []) {
+  const gaps = [];
+  for (let index = 1; index < times.length; index += 1) {
+    gaps.push(Math.max(0, times[index] - times[index - 1]));
+  }
+  return gaps;
+}
+
+function average(values) {
+  const clean = values.filter((value) => Number.isFinite(value));
+  if (clean.length === 0) return null;
+  return clean.reduce((sum, value) => sum + value, 0) / clean.length;
+}
+
+function createPacer(requestRate, rampUpMs = 0) {
+  if (!Number.isFinite(requestRate) || requestRate <= 0) {
+    return async () => {};
+  }
+
+  const startedAt = process.hrtime.bigint();
+  const offsets = [];
+  const steadyIntervalMs = 1000 / requestRate;
+  const warmIntervalMs = steadyIntervalMs * 4;
+
+  return async (index) => {
+    while (offsets.length <= index) {
+      const previousOffset = offsets.length === 0 ? 0 : offsets[offsets.length - 1];
+      const fraction = rampUpMs > 0 ? Math.min(1, previousOffset / rampUpMs) : 1;
+      const interval = warmIntervalMs + ((steadyIntervalMs - warmIntervalMs) * fraction);
+      offsets.push(offsets.length === 0 ? 0 : previousOffset + interval);
+    }
+
+    const targetMs = offsets[index];
+    const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1e6;
+    const waitMs = targetMs - elapsedMs;
+    if (waitMs > 0) {
+      await new Promise((resolve) => setTimeout(resolve, waitMs));
+    }
+  };
+}
+
+function notify(options, event) {
+  if (typeof options.onProgress === 'function') {
+    options.onProgress(event);
+  }
+}
+
 function evaluateSlo(metrics, options) {
   const thresholds = [
     ['firstTokenMs', options.sloTtftMs],

package/src/cli.js

@@ -2,6 +2,7 @@ import fs from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import { inspectModels, runBenchmark } from './bench.js';
 import { runProcess } from './process.js';
+import { createProgress } from './progress.js';
 import { flattenResults, toCsv, writeReport } from './report.js';
 import { renderBenchmarkReport, renderModelsTable } from './table.js';
 
@@ -36,10 +37,21 @@ export async function runCli(argv = process.argv.slice(2)) {
     return;
   }
 
-  const payload = await runBenchmark({
-    ...parsed,
-    version: packageJson.version
+  const progress = createProgress({
+    ...renderOptions(parsed),
+    enabled: resolveProgress(parsed),
+    stream: process.stderr
   });
+  let payload;
+  try {
+    payload = await runBenchmark({
+      ...parsed,
+      version: packageJson.version,
+      onProgress: (event) => progress.update(event)
+    });
+  } finally {
+    progress.stop();
+  }
 
   if (parsed.format === 'json') {
     console.log(JSON.stringify(payload, null, 2));
@@ -71,6 +83,8 @@ export function parseArgs(argv) {
     warmup: 0,
     concurrency: 1,
     sweepConcurrency: [],
+    requestRate: null,
+    rampUpMs: 0,
     timeoutMs: 60_000,
     profile: 'standard',
     greedy: true,
@@ -84,6 +98,8 @@ export function parseArgs(argv) {
     failFast: false,
     verbose: false,
     ascii: false,
+    color: 'auto',
+    progress: 'auto',
     compact: false,
     width: null
   };
@@ -130,6 +146,12 @@ export function parseArgs(argv) {
           options.concurrency = options.sweepConcurrency[0];
         }
         break;
+      case '--request-rate':
+        options.requestRate = parsePositiveNumber(requireValue(arg, args), arg);
+        break;
+      case '--ramp-up-ms':
+        options.rampUpMs = parseNonNegativeInt(requireValue(arg, args), arg);
+        break;
       case '--timeout':
       case '--timeout-ms':
         options.timeoutMs = parsePositiveInt(requireValue(arg, args), arg);
@@ -193,6 +215,18 @@ export function parseArgs(argv) {
       case '--ascii':
         options.ascii = true;
         break;
+      case '--color':
+        options.color = 'always';
+        break;
+      case '--no-color':
+        options.color = 'never';
+        break;
+      case '--progress':
+        options.progress = 'always';
+        break;
+      case '--no-progress':
+        options.progress = 'never';
+        break;
       case '--compact':
         options.compact = true;
         break;
@@ -272,6 +306,12 @@ function parsePositiveInt(value, option) {
   return parsed;
 }
 
+function parsePositiveNumber(value, option) {
+  const parsed = Number.parseFloat(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) throw new Error(`${option} must be a positive number`);
+  return parsed;
+}
+
 function parseNonNegativeInt(value, option) {
   const parsed = Number.parseInt(value, 10);
   if (!Number.isInteger(parsed) || parsed < 0) throw new Error(`${option} must be a non-negative integer`);
@@ -291,11 +331,26 @@ function parsePositiveIntList(value, option) {
 function renderOptions(parsed) {
   return {
     ascii: parsed.ascii,
+    color: resolveColor(parsed.color),
     compact: parsed.compact,
     width: parsed.width
   };
 }
 
+function resolveColor(value) {
+  if (value === 'always') return true;
+  if (value === 'never') return false;
+  if (process.env.NO_COLOR) return false;
+  if (process.env.FORCE_COLOR && process.env.FORCE_COLOR !== '0') return true;
+  return Boolean(process.stdout.isTTY);
+}
+
+function resolveProgress(parsed) {
+  if (parsed.progress === 'always') return true;
+  if (parsed.progress === 'never') return false;
+  return parsed.format === 'table' ? 'auto' : false;
+}
+
 function helpText() {
   return `fm-bench ${packageJson.version}
 
@@ -313,13 +368,15 @@ Run options:
   -c, --concurrency <n>     Parallel fm processes (default: 1)
       --sweep-concurrency <list>
                             Run separate operating points, e.g. 1,2,4
+      --request-rate <rps>  Pace request starts at a target requests/sec
+      --ramp-up-ms <n>      Gradually ramp request pacing over n ms
       --timeout-ms <n>      Timeout per fm call in ms (default: 60000)
       --slo-ttft-ms <n>     Count request as good only if TTFT is <= n
       --slo-e2e-ms <n>      Count request as good only if E2E latency is <= n
       --slo-tpot-ms <n>     Count request as good only if TPOT is <= n
   -p, --prompt <text>       Prompt to benchmark; repeatable
       --prompt-file <file>  .json, .jsonl, or blank-line separated text prompts
-      --profile <name>      quick, standard, interactive, throughput, or stress
+      --profile <name>      quick, standard, interactive, throughput, client, or stress
   -i, --instructions <text> Instructions passed to fm respond
       --use-case <case>     Pass a system model use case through to fm
       --guardrails <level>  Pass a system model guardrail level through to fm
@@ -336,6 +393,10 @@ Output:
       --json                Alias for --format json
       --csv                 Alias for --format csv
       --ascii               Use plain ASCII tables instead of Unicode
+      --color               Force ANSI colors in table output
+      --no-color            Disable ANSI colors in table output
+      --progress            Force live progress on stderr
+      --no-progress         Disable live progress on stderr
       --compact             Force compact terminal layout
       --width <n>           Render for a specific terminal width
   -o, --out <file>          Save JSON or CSV report based on file extension
@@ -349,6 +410,7 @@ Environment:
 Examples:
   fm-bench
   fm-bench --models system,pcc --runs 3 --profile stress
+  fm-bench --profile client --sweep-concurrency 1,2 --request-rate 0.5
   fm-bench --prompt "Reply with exactly: ok" --json --out bench.json
   fm-bench models
   fm-bench doctor

package/src/fm.js

@@ -176,7 +176,8 @@ export async function respond(fmBin, model, prompt, options = {}) {
     durationMs: result.durationMs,
     firstOutputMs: streamed ? result.firstStdoutMs : null,
     streamed,
-    stdoutChunks: result.stdoutChunks
+    stdoutChunks: result.stdoutChunks,
+    stdoutChunkTimesMs: streamed ? result.stdoutChunkTimesMs : []
   };
 }
 

package/src/process.js

@@ -20,6 +20,7 @@ export function runProcess(command, args = [], options = {}) {
     let stderr = '';
     let stdoutChunks = 0;
     let stderrChunks = 0;
+    const stdoutChunkTimesMs = [];
     let firstStdoutMs = null;
     let firstStderrMs = null;
     let timedOut = false;
@@ -38,10 +39,12 @@ export function runProcess(command, args = [], options = {}) {
     child.stdout.setEncoding('utf8');
     child.stderr.setEncoding('utf8');
     child.stdout.on('data', (chunk) => {
+      const chunkAtMs = Number(process.hrtime.bigint() - startedAt) / 1e6;
       stdoutChunks += 1;
       if (firstStdoutMs == null && chunk.length > 0) {
-        firstStdoutMs = Number(process.hrtime.bigint() - startedAt) / 1e6;
+        firstStdoutMs = chunkAtMs;
       }
+      if (chunk.length > 0) stdoutChunkTimesMs.push(chunkAtMs);
       stdout += chunk;
     });
     child.stderr.on('data', (chunk) => {
@@ -65,6 +68,7 @@ export function runProcess(command, args = [], options = {}) {
         stderr: stderr || error.message,
         stdoutChunks,
         stderrChunks,
+        stdoutChunkTimesMs,
         firstStdoutMs,
         firstStderrMs,
         error,
@@ -86,6 +90,7 @@ export function runProcess(command, args = [], options = {}) {
         stderr,
         stdoutChunks,
         stderrChunks,
+        stdoutChunkTimesMs,
         firstStdoutMs,
         firstStderrMs,
         timedOut,