@localmode/bench 0.1.0 → 0.2.0

package/dist/index.d.cts

@@ -5,9 +5,9 @@
  * published statistics can be recomputed and audited server-side.
  */
 /** Protocol identifier embedded in every result. Bump only with a spec change. */
-declare const BENCH_PROTOCOL_VERSION = "localmode-bench/1";
+declare const BENCH_PROTOCOL_VERSION = "localmode-bench/2";
 /** Result JSON schema version (independent of the protocol semantics version). */
-declare const BENCH_SCHEMA_VERSION = 1;
+declare const BENCH_SCHEMA_VERSION = 2;
 /** Benchmark suite presets. `custom` = user-picked cells. */
 type BenchSuiteId = 'quick' | 'standard' | 'thorough' | 'custom';
 /**
@@ -69,6 +69,12 @@ interface BenchModelRef {
     /** Direct model URL when applicable (wllama GGUF, litert). */
     url?: string;
     requiresWebGPU?: boolean;
+    /**
+     * Appended to the quality-lane instruction line for every runtime of this
+     * pairing (e.g. " /no_think" to hold Qwen3 in non-thinking mode). Uniform
+     * across runtimes by construction, so fidelity comparisons stay valid.
+     */
+    qualityPromptSuffix?: string;
 }
 /** One recorded stream chunk: wall-clock timestamp + delta length in chars. */
 interface BenchChunk {
@@ -131,6 +137,8 @@ interface MemorySample {
     baseline?: number;
     postLoad?: number;
     postRun?: number;
+    /** Bytes at the moment a cell errored (load or workload failure). */
+    atError?: number;
     api: 'uaSpecific' | 'legacyHeap' | 'none';
 }
 /** Quality lane outcome attached to a cell. */
@@ -141,6 +149,13 @@ interface QualityResult {
     n: number;
     /** Per-item correctness or per-pair cosine, for auditability. */
     details?: number[];
+    /**
+     * Raw model outputs per item (MMLU lanes; capped at 400 chars each) so the
+     * score is recomputable server-side and parse failures are auditable.
+     */
+    outputs?: string[];
+    /** Fraction of items whose answer was parseable (MMLU lanes). */
+    parseRate?: number;
 }
 type BenchCellStatus = 'ok' | 'invalid' | 'error' | 'skipped';
 /** One benchmark cell: (runtime x model x workload) with its raw iterations. */
@@ -162,9 +177,11 @@ interface BenchCellResult {
     quality?: QualityResult;
     status: BenchCellStatus;
     invalidReasons?: string[];
+    /** Error that ended the cell; `cause` carries the wrapped provider error's message when present. */
     error?: {
         name: string;
         message: string;
+        cause?: string;
     };
 }
 /** Trace events global to the suite run (validity accounting). */
@@ -281,6 +298,16 @@ interface CellSummary {
     decodeChunksPerSec?: MetricSummary;
     prefillTokPerSecApprox?: MetricSummary;
     generatedChars?: MetricSummary;
+    /** Request wall time (startT → endT), all LLM lanes. */
+    totalMs?: MetricSummary;
+    /** End-to-end chars/s over the whole request (prefill + decode conflated). */
+    overallCharsPerSec?: MetricSummary;
+    /**
+     * False when the chunk trace is not genuinely incremental (single chunk, or
+     * the visible stream spans <20% of the request) — TTFT and decode metrics
+     * are then omitted because they would be timing artifacts, not measurements.
+     */
+    streamIncremental?: boolean;
     /** Embedding lanes. */
     singleLatencyMs?: MetricSummary;
     batchTextsPerSec?: MetricSummary;
@@ -288,6 +315,8 @@ interface CellSummary {
     loadMs?: number;
     loadCached?: boolean;
     qualityScore?: number;
+    /** Fraction of MMLU items whose answer parsed; a low value marks a format failure, not a fidelity one. */
+    qualityParseRate?: number;
     highVariance: boolean;
 }
 /** The unit of submission: one full suite run on one device. */
@@ -340,6 +369,14 @@ interface ValidationReport {
 
 /** Fixed generation budget for timed lanes (tg128, MLPerf-Client-sized). */
 declare const GENERATION_BUDGET = 128;
+/**
+ * Minimum generated characters for a timed iteration to count as a decode
+ * measurement (part of the versioned protocol). An instruct model that emits
+ * EOS after a handful of characters - typically an untemplated prompt or a
+ * tokenizer mismatch - produces no decode phase to time; such iterations are
+ * gated `degenerate-output` and the cell is marked invalid rather than scored.
+ */
+declare const MIN_GENERATED_CHARS = 16;
 /** LLM performance workloads. */
 declare const LLM_WORKLOADS: readonly LLMWorkloadSpec[];
 /** Embedding performance workloads. */
@@ -348,6 +385,25 @@ declare const EMBED_WORKLOADS: readonly EmbedWorkloadSpec[];
 declare const QUALITY_WORKLOADS: readonly QualityWorkloadSpec[];
 /** All workloads indexed by id. */
 declare const WORKLOADS_BY_ID: ReadonlyMap<string, LLMWorkloadSpec | EmbedWorkloadSpec | QualityWorkloadSpec>;
+/**
+ * Fixed runtime execution order (part of the versioned protocol). This makes
+ * the order deterministic so runtime interleaving is not a confounder across
+ * runs; it is a reproducibility measure, not a correctness fix. (The
+ * Transformers.js ORT-web lanes fail during session creation regardless of
+ * where they run in the order - a separate open runtime/adapter issue.) The
+ * order runs the WASM-arena runtimes first and the multi-GB-heap runtimes last.
+ * Within a runtime, catalog order is preserved.
+ */
+declare const RUNTIME_EXECUTION_ORDER: readonly BenchRuntimeId[];
+/**
+ * Sort planned cells into the protocol execution order (stable within a
+ * runtime). The runner applies this itself; exported for hosts and tests.
+ */
+declare function orderCells<T extends {
+    model: {
+        runtimeId: BenchRuntimeId;
+    };
+}>(cells: readonly T[]): T[];
 /** Run policy for a suite (part of the versioned protocol). */
 interface RunPolicy {
     /** Untimed warmup generations per cell (absorbs shader compile/JIT). */
@@ -704,19 +760,40 @@ declare function spearman(a: readonly number[], b: readonly number[]): number;
  */
 
 /** Version of the plausibility rule set (recorded alongside moderation). */
-declare const PLAUSIBILITY_RULES_VERSION = 1;
+declare const PLAUSIBILITY_RULES_VERSION = 2;
+/**
+ * Minimum fraction of the request that the visible chunk stream must span for
+ * the trace to count as incremental. LiteRT-LM's web surface delivers every
+ * chunk in a terminal burst (~0.8ms of a 30s request), so TTFT/decode derived
+ * from such a trace are timing artifacts; a real token stream spans most of
+ * the request by construction.
+ */
+declare const STREAM_COHERENCE_MIN_SPAN_RATIO = 0.2;
 /**
  * Structural validation of a submitted result. Returns human-readable errors
  * (empty array = shape ok). Deliberately hand-rolled: this package is
  * zero-dependency and the checks double as executable schema documentation.
  */
 declare function validateRunShape(value: unknown): string[];
+/**
+ * True when an iteration's chunk trace is genuinely incremental: at least two
+ * chunks, and the visible stream spans at least
+ * `STREAM_COHERENCE_MIN_SPAN_RATIO` of the request wall time. Anything else
+ * (a doGenerate fallback's single chunk, or a runtime that computes the whole
+ * generation and flushes it in a terminal burst) carries no usable TTFT or
+ * decode timing.
+ */
+declare function isIncrementalStream(it: LLMIteration): boolean;
 /**
  * Recompute a cell's summary purely from its raw iteration traces.
  * This function IS the metric definition:
  * - TTFT = first non-empty chunk timestamp − startT
  * - decode rate = (chars after first chunk) / (endT_lastChunk − t_firstChunk)
  * - prefill tok/s ≈ approxPromptTokens / TTFT (approximate by construction)
+ * - overall rate = total chars / (endT − startT), always derivable
+ * TTFT/decode/prefill are derived only when EVERY iteration passes the
+ * stream-coherence test (`streamIncremental`); overall rate and total wall
+ * time are reported for all LLM lanes.
  */
 declare function summarizeCell(cell: BenchCellResult, highVarianceCv?: number, approxPromptTokens?: number): CellSummary;
 /** Recompute all cell summaries for a run (workload prompt sizes looked up). */
@@ -784,11 +861,15 @@ interface LeaderboardRow {
     /** Median-of-medians metrics (only those applicable to the workload). */
     ttftMs?: number;
     decodeCharsPerSec?: number;
+    /** End-to-end chars/s (prefill + decode); the only rate for lanes whose stream is not incremental. */
+    overallCharsPerSec?: number;
     singleLatencyMs?: number;
     batchTextsPerSec?: number;
     loadColdMs?: number;
     loadWarmMs?: number;
     qualityScore?: number;
+    /** Median MMLU parse rate; below 1 the quality score is format-limited. */
+    qualityParseRate?: number;
     resolvedBackends: string[];
     browsers: string[];
     /** True when any contributing submission had a high-variance metric. */
@@ -843,26 +924,44 @@ declare const TINY_MMLU: readonly MMLUItem[];
  * model's training data. Always run at temperature 0, outside timed regions.
  */
 
-/** Fixed MCQ prompt template (part of the versioned protocol). */
-declare function formatMMLUPrompt(item: (typeof TINY_MMLU)[number]): string;
 /**
- * Parse the answer letter from a model response. Accepts "B", "B.", "(B)",
- * "Answer: B", or a response beginning with the exact choice text.
+ * Generation budget for MMLU items (part of the versioned protocol). Large
+ * enough to absorb an empty Qwen3-style `<think></think>` block plus a
+ * verbose "The answer is B." — the v1 budget of 8 truncated before any
+ * parseable letter on thinking-mode builds, scoring fidelity as 0.
+ */
+declare const MMLU_MAX_TOKENS = 48;
+/** Cap on each stored raw output (auditability without payload bloat). */
+declare const MMLU_OUTPUT_CAP = 400;
+/** Fixed MCQ prompt template (part of the versioned protocol). */
+declare function formatMMLUPrompt(item: (typeof TINY_MMLU)[number], promptSuffix?: string): string;
+/**
+ * Parse the answer letter from a model response. Reasoning blocks are
+ * stripped first; markdown emphasis around the letter is ignored. Accepts
+ * "B", "B.", "(B)", "**B**", "Answer: B", "Option C", "choice (B)", or a
+ * response beginning with the exact choice text. Letters are matched
+ * case-sensitively after a keyword so the article "a" ("the answer is a
+ * bit unclear") is never read as answer A; a lowercase letter counts only
+ * when it is the whole reply or leads it as "b." / "b)".
  *
  * @returns 0-3, or null when no unambiguous answer is present.
  */
 declare function parseMMLUAnswer(response: string, choices: readonly string[]): number | null;
 /**
  * Run the tinyMMLU fidelity task on a language model (temperature 0,
- * `maxTokens` 8, greedy answer parsing). Unparseable answers count as wrong.
+ * `MMLU_MAX_TOKENS` budget, greedy answer parsing). Unparseable answers count
+ * as wrong; raw outputs and the parse rate are recorded so the score is
+ * auditable and recomputable server-side.
  *
  * @param model - Any structurally-compatible LanguageModel.
- * @param items - Number of items from the 100-item set (25 or 100 in v1).
- * @returns Accuracy in [0,1] with per-item correctness details.
+ * @param items - Number of items from the 100-item set (25 or 100).
+ * @returns Accuracy in [0,1] with per-item correctness details and outputs.
  */
 declare function runMMLUFidelity(model: BenchLanguageModel, items: number, options?: {
     abortSignal?: AbortSignal;
     onProgress?: (done: number, total: number) => void;
+    /** Appended to the instruction line of every item (from the model catalog). */
+    promptSuffix?: string;
 }): Promise<QualityResult>;
 /**
  * Run the STS-B embedding-quality task: Spearman correlation of cosine
@@ -898,4 +997,4 @@ interface STSPair {
 /** 100-pair STS-B test subset (order preserved from the source dataset). */
 declare const STSB_SUBSET: readonly STSPair[];
 
-export { type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type MMLUItem, type MemorySample, type MetricSummary, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, mean, median, memoryApiAvailable, parseMMLUAnswer, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };
+export { type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseMMLUAnswer, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };

package/dist/index.d.ts

@@ -5,9 +5,9 @@
  * published statistics can be recomputed and audited server-side.
  */
 /** Protocol identifier embedded in every result. Bump only with a spec change. */
-declare const BENCH_PROTOCOL_VERSION = "localmode-bench/1";
+declare const BENCH_PROTOCOL_VERSION = "localmode-bench/2";
 /** Result JSON schema version (independent of the protocol semantics version). */
-declare const BENCH_SCHEMA_VERSION = 1;
+declare const BENCH_SCHEMA_VERSION = 2;
 /** Benchmark suite presets. `custom` = user-picked cells. */
 type BenchSuiteId = 'quick' | 'standard' | 'thorough' | 'custom';
 /**
@@ -69,6 +69,12 @@ interface BenchModelRef {
     /** Direct model URL when applicable (wllama GGUF, litert). */
     url?: string;
     requiresWebGPU?: boolean;
+    /**
+     * Appended to the quality-lane instruction line for every runtime of this
+     * pairing (e.g. " /no_think" to hold Qwen3 in non-thinking mode). Uniform
+     * across runtimes by construction, so fidelity comparisons stay valid.
+     */
+    qualityPromptSuffix?: string;
 }
 /** One recorded stream chunk: wall-clock timestamp + delta length in chars. */
 interface BenchChunk {
@@ -131,6 +137,8 @@ interface MemorySample {
     baseline?: number;
     postLoad?: number;
     postRun?: number;
+    /** Bytes at the moment a cell errored (load or workload failure). */
+    atError?: number;
     api: 'uaSpecific' | 'legacyHeap' | 'none';
 }
 /** Quality lane outcome attached to a cell. */
@@ -141,6 +149,13 @@ interface QualityResult {
     n: number;
     /** Per-item correctness or per-pair cosine, for auditability. */
     details?: number[];
+    /**
+     * Raw model outputs per item (MMLU lanes; capped at 400 chars each) so the
+     * score is recomputable server-side and parse failures are auditable.
+     */
+    outputs?: string[];
+    /** Fraction of items whose answer was parseable (MMLU lanes). */
+    parseRate?: number;
 }
 type BenchCellStatus = 'ok' | 'invalid' | 'error' | 'skipped';
 /** One benchmark cell: (runtime x model x workload) with its raw iterations. */
@@ -162,9 +177,11 @@ interface BenchCellResult {
     quality?: QualityResult;
     status: BenchCellStatus;
     invalidReasons?: string[];
+    /** Error that ended the cell; `cause` carries the wrapped provider error's message when present. */
     error?: {
         name: string;
         message: string;
+        cause?: string;
     };
 }
 /** Trace events global to the suite run (validity accounting). */
@@ -281,6 +298,16 @@ interface CellSummary {
     decodeChunksPerSec?: MetricSummary;
     prefillTokPerSecApprox?: MetricSummary;
     generatedChars?: MetricSummary;
+    /** Request wall time (startT → endT), all LLM lanes. */
+    totalMs?: MetricSummary;
+    /** End-to-end chars/s over the whole request (prefill + decode conflated). */
+    overallCharsPerSec?: MetricSummary;
+    /**
+     * False when the chunk trace is not genuinely incremental (single chunk, or
+     * the visible stream spans <20% of the request) — TTFT and decode metrics
+     * are then omitted because they would be timing artifacts, not measurements.
+     */
+    streamIncremental?: boolean;
     /** Embedding lanes. */
     singleLatencyMs?: MetricSummary;
     batchTextsPerSec?: MetricSummary;
@@ -288,6 +315,8 @@ interface CellSummary {
     loadMs?: number;
     loadCached?: boolean;
     qualityScore?: number;
+    /** Fraction of MMLU items whose answer parsed; a low value marks a format failure, not a fidelity one. */
+    qualityParseRate?: number;
     highVariance: boolean;
 }
 /** The unit of submission: one full suite run on one device. */
@@ -340,6 +369,14 @@ interface ValidationReport {
 
 /** Fixed generation budget for timed lanes (tg128, MLPerf-Client-sized). */
 declare const GENERATION_BUDGET = 128;
+/**
+ * Minimum generated characters for a timed iteration to count as a decode
+ * measurement (part of the versioned protocol). An instruct model that emits
+ * EOS after a handful of characters - typically an untemplated prompt or a
+ * tokenizer mismatch - produces no decode phase to time; such iterations are
+ * gated `degenerate-output` and the cell is marked invalid rather than scored.
+ */
+declare const MIN_GENERATED_CHARS = 16;
 /** LLM performance workloads. */
 declare const LLM_WORKLOADS: readonly LLMWorkloadSpec[];
 /** Embedding performance workloads. */
@@ -348,6 +385,25 @@ declare const EMBED_WORKLOADS: readonly EmbedWorkloadSpec[];
 declare const QUALITY_WORKLOADS: readonly QualityWorkloadSpec[];
 /** All workloads indexed by id. */
 declare const WORKLOADS_BY_ID: ReadonlyMap<string, LLMWorkloadSpec | EmbedWorkloadSpec | QualityWorkloadSpec>;
+/**
+ * Fixed runtime execution order (part of the versioned protocol). This makes
+ * the order deterministic so runtime interleaving is not a confounder across
+ * runs; it is a reproducibility measure, not a correctness fix. (The
+ * Transformers.js ORT-web lanes fail during session creation regardless of
+ * where they run in the order - a separate open runtime/adapter issue.) The
+ * order runs the WASM-arena runtimes first and the multi-GB-heap runtimes last.
+ * Within a runtime, catalog order is preserved.
+ */
+declare const RUNTIME_EXECUTION_ORDER: readonly BenchRuntimeId[];
+/**
+ * Sort planned cells into the protocol execution order (stable within a
+ * runtime). The runner applies this itself; exported for hosts and tests.
+ */
+declare function orderCells<T extends {
+    model: {
+        runtimeId: BenchRuntimeId;
+    };
+}>(cells: readonly T[]): T[];
 /** Run policy for a suite (part of the versioned protocol). */
 interface RunPolicy {
     /** Untimed warmup generations per cell (absorbs shader compile/JIT). */
@@ -704,19 +760,40 @@ declare function spearman(a: readonly number[], b: readonly number[]): number;
  */
 
 /** Version of the plausibility rule set (recorded alongside moderation). */
-declare const PLAUSIBILITY_RULES_VERSION = 1;
+declare const PLAUSIBILITY_RULES_VERSION = 2;
+/**
+ * Minimum fraction of the request that the visible chunk stream must span for
+ * the trace to count as incremental. LiteRT-LM's web surface delivers every
+ * chunk in a terminal burst (~0.8ms of a 30s request), so TTFT/decode derived
+ * from such a trace are timing artifacts; a real token stream spans most of
+ * the request by construction.
+ */
+declare const STREAM_COHERENCE_MIN_SPAN_RATIO = 0.2;
 /**
  * Structural validation of a submitted result. Returns human-readable errors
  * (empty array = shape ok). Deliberately hand-rolled: this package is
  * zero-dependency and the checks double as executable schema documentation.
  */
 declare function validateRunShape(value: unknown): string[];
+/**
+ * True when an iteration's chunk trace is genuinely incremental: at least two
+ * chunks, and the visible stream spans at least
+ * `STREAM_COHERENCE_MIN_SPAN_RATIO` of the request wall time. Anything else
+ * (a doGenerate fallback's single chunk, or a runtime that computes the whole
+ * generation and flushes it in a terminal burst) carries no usable TTFT or
+ * decode timing.
+ */
+declare function isIncrementalStream(it: LLMIteration): boolean;
 /**
  * Recompute a cell's summary purely from its raw iteration traces.
  * This function IS the metric definition:
  * - TTFT = first non-empty chunk timestamp − startT
  * - decode rate = (chars after first chunk) / (endT_lastChunk − t_firstChunk)
  * - prefill tok/s ≈ approxPromptTokens / TTFT (approximate by construction)
+ * - overall rate = total chars / (endT − startT), always derivable
+ * TTFT/decode/prefill are derived only when EVERY iteration passes the
+ * stream-coherence test (`streamIncremental`); overall rate and total wall
+ * time are reported for all LLM lanes.
  */
 declare function summarizeCell(cell: BenchCellResult, highVarianceCv?: number, approxPromptTokens?: number): CellSummary;
 /** Recompute all cell summaries for a run (workload prompt sizes looked up). */
@@ -784,11 +861,15 @@ interface LeaderboardRow {
     /** Median-of-medians metrics (only those applicable to the workload). */
     ttftMs?: number;
     decodeCharsPerSec?: number;
+    /** End-to-end chars/s (prefill + decode); the only rate for lanes whose stream is not incremental. */
+    overallCharsPerSec?: number;
     singleLatencyMs?: number;
     batchTextsPerSec?: number;
     loadColdMs?: number;
     loadWarmMs?: number;
     qualityScore?: number;
+    /** Median MMLU parse rate; below 1 the quality score is format-limited. */
+    qualityParseRate?: number;
     resolvedBackends: string[];
     browsers: string[];
     /** True when any contributing submission had a high-variance metric. */
@@ -843,26 +924,44 @@ declare const TINY_MMLU: readonly MMLUItem[];
  * model's training data. Always run at temperature 0, outside timed regions.
  */
 
-/** Fixed MCQ prompt template (part of the versioned protocol). */
-declare function formatMMLUPrompt(item: (typeof TINY_MMLU)[number]): string;
 /**
- * Parse the answer letter from a model response. Accepts "B", "B.", "(B)",
- * "Answer: B", or a response beginning with the exact choice text.
+ * Generation budget for MMLU items (part of the versioned protocol). Large
+ * enough to absorb an empty Qwen3-style `<think></think>` block plus a
+ * verbose "The answer is B." — the v1 budget of 8 truncated before any
+ * parseable letter on thinking-mode builds, scoring fidelity as 0.
+ */
+declare const MMLU_MAX_TOKENS = 48;
+/** Cap on each stored raw output (auditability without payload bloat). */
+declare const MMLU_OUTPUT_CAP = 400;
+/** Fixed MCQ prompt template (part of the versioned protocol). */
+declare function formatMMLUPrompt(item: (typeof TINY_MMLU)[number], promptSuffix?: string): string;
+/**
+ * Parse the answer letter from a model response. Reasoning blocks are
+ * stripped first; markdown emphasis around the letter is ignored. Accepts
+ * "B", "B.", "(B)", "**B**", "Answer: B", "Option C", "choice (B)", or a
+ * response beginning with the exact choice text. Letters are matched
+ * case-sensitively after a keyword so the article "a" ("the answer is a
+ * bit unclear") is never read as answer A; a lowercase letter counts only
+ * when it is the whole reply or leads it as "b." / "b)".
  *
  * @returns 0-3, or null when no unambiguous answer is present.
  */
 declare function parseMMLUAnswer(response: string, choices: readonly string[]): number | null;
 /**
  * Run the tinyMMLU fidelity task on a language model (temperature 0,
- * `maxTokens` 8, greedy answer parsing). Unparseable answers count as wrong.
+ * `MMLU_MAX_TOKENS` budget, greedy answer parsing). Unparseable answers count
+ * as wrong; raw outputs and the parse rate are recorded so the score is
+ * auditable and recomputable server-side.
  *
  * @param model - Any structurally-compatible LanguageModel.
- * @param items - Number of items from the 100-item set (25 or 100 in v1).
- * @returns Accuracy in [0,1] with per-item correctness details.
+ * @param items - Number of items from the 100-item set (25 or 100).
+ * @returns Accuracy in [0,1] with per-item correctness details and outputs.
  */
 declare function runMMLUFidelity(model: BenchLanguageModel, items: number, options?: {
     abortSignal?: AbortSignal;
     onProgress?: (done: number, total: number) => void;
+    /** Appended to the instruction line of every item (from the model catalog). */
+    promptSuffix?: string;
 }): Promise<QualityResult>;
 /**
  * Run the STS-B embedding-quality task: Spearman correlation of cosine
@@ -898,4 +997,4 @@ interface STSPair {
 /** 100-pair STS-B test subset (order preserved from the source dataset). */
 declare const STSB_SUBSET: readonly STSPair[];
 
-export { type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type MMLUItem, type MemorySample, type MetricSummary, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, mean, median, memoryApiAvailable, parseMMLUAnswer, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };
+export { type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseMMLUAnswer, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };