@localmode/bench 0.1.0 → 0.3.0

package/dist/index.d.ts

@@ -5,9 +5,9 @@
  * published statistics can be recomputed and audited server-side.
  */
 /** Protocol identifier embedded in every result. Bump only with a spec change. */
-declare const BENCH_PROTOCOL_VERSION = "localmode-bench/1";
+declare const BENCH_PROTOCOL_VERSION = "localmode-bench/2";
 /** Result JSON schema version (independent of the protocol semantics version). */
-declare const BENCH_SCHEMA_VERSION = 1;
+declare const BENCH_SCHEMA_VERSION = 2;
 /** Benchmark suite presets. `custom` = user-picked cells. */
 type BenchSuiteId = 'quick' | 'standard' | 'thorough' | 'custom';
 /**
@@ -69,6 +69,12 @@ interface BenchModelRef {
     /** Direct model URL when applicable (wllama GGUF, litert). */
     url?: string;
     requiresWebGPU?: boolean;
+    /**
+     * Appended to the quality-lane instruction line for every runtime of this
+     * pairing (e.g. " /no_think" to hold Qwen3 in non-thinking mode). Uniform
+     * across runtimes by construction, so fidelity comparisons stay valid.
+     */
+    qualityPromptSuffix?: string;
 }
 /** One recorded stream chunk: wall-clock timestamp + delta length in chars. */
 interface BenchChunk {
@@ -131,6 +137,8 @@ interface MemorySample {
     baseline?: number;
     postLoad?: number;
     postRun?: number;
+    /** Bytes at the moment a cell errored (load or workload failure). */
+    atError?: number;
     api: 'uaSpecific' | 'legacyHeap' | 'none';
 }
 /** Quality lane outcome attached to a cell. */
@@ -141,6 +149,13 @@ interface QualityResult {
     n: number;
     /** Per-item correctness or per-pair cosine, for auditability. */
     details?: number[];
+    /**
+     * Raw model outputs per item (MMLU lanes; capped at 400 chars each) so the
+     * score is recomputable server-side and parse failures are auditable.
+     */
+    outputs?: string[];
+    /** Fraction of items whose answer was parseable (MMLU lanes). */
+    parseRate?: number;
 }
 type BenchCellStatus = 'ok' | 'invalid' | 'error' | 'skipped';
 /** One benchmark cell: (runtime x model x workload) with its raw iterations. */
@@ -162,9 +177,11 @@ interface BenchCellResult {
     quality?: QualityResult;
     status: BenchCellStatus;
     invalidReasons?: string[];
+    /** Error that ended the cell; `cause` carries the wrapped provider error's message when present. */
     error?: {
         name: string;
         message: string;
+        cause?: string;
     };
 }
 /** Trace events global to the suite run (validity accounting). */
@@ -182,6 +199,14 @@ interface BrowserInfo {
         brand: string;
         version: string;
     }>;
+    /** Rendering engine, derived from the UA (Blink / Gecko / WebKit). */
+    engine?: 'Blink' | 'Gecko' | 'WebKit' | 'unknown';
+    /** `navigator.vendor` (e.g. "Google Inc.", "Apple Computer, Inc."). */
+    vendor?: string;
+    /** `navigator.webdriver` — true under browser automation. */
+    webdriver?: boolean;
+    /** `navigator.pdfViewerEnabled`, a cheap headless/kiosk signal. */
+    pdfViewerEnabled?: boolean;
 }
 /** OS identification. Version is 'unknown-frozen' on engines with frozen UAs. */
 interface OSInfo {
@@ -189,7 +214,30 @@ interface OSInfo {
     version: string;
     architecture?: string;
     bitness?: string;
+    /** Device model from UA-CH (Android only; empty elsewhere by spec). */
     model?: string;
+    /** Whether the OS is running in a WoW64-style emulation layer (UA-CH `wow64`). */
+    wow64?: boolean;
+    /** `navigator.platform` (legacy, frozen but still informative: "MacIntel", "Win32", "iPhone"). */
+    navigatorPlatform?: string;
+}
+/** Form-factor classification, derived from UA-CH form factors, the UA, and touch. */
+type DeviceType = 'phone' | 'tablet' | 'desktop' | 'xr' | 'tv' | 'unknown';
+/** Device / form-factor signals. */
+interface DeviceInfo {
+    type: DeviceType;
+    /** UA-CH `mobile` bit (Chromium) or a UA-derived guess elsewhere. */
+    mobile: boolean;
+    /** UA-CH `formFactors` (Chromium 125+): Desktop / Mobile / Tablet / XR / EInk / Watch / Automotive. */
+    formFactors?: string[];
+    /** `navigator.maxTouchPoints`. */
+    maxTouchPoints: number;
+    /** `(pointer: coarse)` media query — primary input is a touch surface. */
+    pointerCoarse?: boolean;
+    /** `(hover: none)` media query — no hover-capable primary input. */
+    hoverNone?: boolean;
+    /** Standalone / fullscreen display mode (installed PWA or kiosk). */
+    displayMode?: string;
 }
 /** WebGPU adapter identity + selected limits (all fields may be empty strings). */
 interface GPUInfo {
@@ -198,11 +246,155 @@ interface GPUInfo {
     architecture?: string;
     device?: string;
     description?: string;
+    /** Adapter-reported subgroup sizes (Chromium 130+ `info.subgroupMinSize/MaxSize`). */
+    subgroupMinSize?: number;
+    subgroupMaxSize?: number;
     isFallbackAdapter?: boolean;
     features?: string[];
     limits?: Record<string, number>;
+    /** `navigator.gpu.getPreferredCanvasFormat()`. */
+    preferredCanvasFormat?: string;
+    /** WGSL language features the implementation reports (`navigator.gpu.wgslLanguageFeatures`). */
+    wgslLanguageFeatures?: string[];
+}
+/** WebGL identity + capacity signals (a second, older GPU identity channel). */
+interface WebGLInfo {
+    /** Best context available: 'webgl2', 'webgl', or null when neither creates. */
+    contextKind: 'webgl2' | 'webgl' | null;
+    /** UNMASKED_VENDOR_WEBGL when the debug extension exists, else VENDOR. */
+    vendor?: string;
+    /** UNMASKED_RENDERER_WEBGL when the debug extension exists, else RENDERER. */
+    renderer?: string;
+    version?: string;
+    shadingLanguageVersion?: string;
+    maxTextureSize?: number;
+    maxRenderbufferSize?: number;
+    maxVertexUniformVectors?: number;
+    maxFragmentUniformVectors?: number;
+    /** Number of extensions exposed (identity signal without shipping the whole list). */
+    extensionCount?: number;
+    /** Whether the renderer string names a software rasterizer (SwiftShader, llvmpipe, ...). */
+    softwareRenderer?: boolean;
+}
+/**
+ * WebAssembly proposal support, each probed by validating a canonical module
+ * (the same byte sequences `wasm-feature-detect` uses). Missing keys mean the
+ * probe itself failed, not that the feature is absent.
+ */
+interface WasmFeatureSupport {
+    simd: boolean;
+    relaxedSimd: boolean;
+    threads: boolean;
+    bulkMemory: boolean;
+    exceptions: boolean;
+    /** Exception handling with `exnref` (the newer, standardized form). */
+    exceptionsFinal: boolean;
+    extendedConst: boolean;
+    gc: boolean;
+    memory64: boolean;
+    multiMemory: boolean;
+    multiValue: boolean;
+    mutableGlobals: boolean;
+    referenceTypes: boolean;
+    saturatedFloatToInt: boolean;
+    signExtensions: boolean;
+    tailCall: boolean;
+    typedFunctionReferences: boolean;
+    /** 128-bit wide arithmetic (`i64.add128` family). */
+    wideArithmetic: boolean;
+    /** JavaScript Promise Integration (`WebAssembly.Suspending`). */
+    jspi: boolean;
+    /** Type reflection (`WebAssembly.Function`). */
+    typeReflection: boolean;
+    /** `WebAssembly.compileStreaming` exists. */
+    streamingCompilation: boolean;
+    /** JS String Builtins (`js-string` import module), probed via the imports/builtins option. */
+    jsStringBuiltins: boolean;
+    /** Type-level `WebAssembly.Memory` growth limit reachable in this engine (pages of 64 KiB). */
+    maxMemoryPages?: number;
+}
+/**
+ * Availability of the browser APIs the runtimes and the paper care about.
+ * Each entry is a plain presence check (the feature exists on this page),
+ * not a functional test. Chrome Built-in AI is reported by its
+ * `availability()` string where the API exists.
+ */
+interface APIAvailability {
+    webgpu: boolean;
+    webgl2: boolean;
+    webnn: boolean;
+    /** OPFS: `navigator.storage.getDirectory` exists AND resolved. */
+    opfs: boolean;
+    /** Result of `navigator.storage.persisted()` where supported. */
+    persistedStorage?: boolean;
+    indexedDB: boolean;
+    cacheApi: boolean;
+    serviceWorker: boolean;
+    webWorkers: boolean;
+    offscreenCanvas: boolean;
+    webLocks: boolean;
+    broadcastChannel: boolean;
+    wakeLock: boolean;
+    computePressure: boolean;
+    performanceMemory: boolean;
+    measureUserAgentSpecificMemory: boolean;
+    schedulerYield: boolean;
+    webCodecs: boolean;
+    audioWorklet: boolean;
+    mediaDevices: boolean;
+    webTransport: boolean;
+    /** Chrome Built-in AI (Gemini Nano) surfaces, by `availability()` verdict when reachable. */
+    promptApi?: string;
+    summarizerApi?: string;
+    translatorApi?: string;
+    languageDetectorApi?: string;
 }
-/** Full environment capture for a run. Clamped fields are labeled as such. */
+/** Network Information API (Chromium + Android) snapshot at capture time. */
+interface NetworkInfo {
+    supported: boolean;
+    effectiveType?: string;
+    /** Connection type (wifi / cellular / ethernet / ...) where the UA exposes it. */
+    type?: string;
+    downlinkMbps?: number;
+    rttMs?: number;
+    saveData?: boolean;
+    online?: boolean;
+}
+/** Display / viewport snapshot. */
+interface DisplayInfo {
+    width: number;
+    height: number;
+    availWidth?: number;
+    availHeight?: number;
+    dpr: number;
+    colorDepth?: number;
+    orientation?: string;
+    viewportWidth?: number;
+    viewportHeight?: number;
+    /** `(dynamic-range: high)` media query. */
+    hdr?: boolean;
+    /** `(color-gamut: p3)` media query. */
+    wideGamut?: boolean;
+    /** `screen.isExtended` (Window Management API) — more than one display attached. */
+    isExtended?: boolean;
+    /** Reduced-motion / forced-colors preferences, cheap OS-level signals. */
+    prefersReducedMotion?: boolean;
+    prefersColorScheme?: 'light' | 'dark' | 'no-preference';
+}
+/** Locale / clock signals. */
+interface LocaleInfo {
+    timeZone?: string;
+    /** Minutes offset from UTC at capture time (`Date#getTimezoneOffset`). */
+    timeZoneOffsetMinutes?: number;
+    locale?: string;
+    /** `Intl.DateTimeFormat().resolvedOptions().calendar`. */
+    calendar?: string;
+}
+/**
+ * Full environment capture for a run. Every field beyond the first block is
+ * additive and best-effort: a probe that fails records nothing for its key and
+ * never affects the others. Clamped or capped fields are labeled as such.
+ */
 interface EnvironmentCapture {
     capturedAt: string;
     browser: BrowserInfo;
@@ -214,23 +406,46 @@ interface EnvironmentCapture {
         /** navigator.deviceMemory (GB) — Chromium-only, capped at 8. */
         deviceMemoryGB: number | null;
         deviceMemoryCapped: boolean;
+        /** `performance.memory.jsHeapSizeLimit` (Chromium) — the V8 heap ceiling for this tab. */
+        jsHeapSizeLimitBytes?: number;
+        /** `performance.memory.usedJSHeapSize` at capture (the idle baseline). */
+        jsHeapUsedBytes?: number;
     };
     gpu: GPUInfo;
     /** WebGL renderer string, a secondary GPU identity signal. */
     webglRenderer: string | null;
+    /** Detailed WebGL identity + capacity (superset of `webglRenderer`). */
+    webgl?: WebGLInfo;
+    /**
+     * GPU model parsed from the WebGL renderer string (ANGLE unwrapped),
+     * e.g. "Apple M4", "NVIDIA GeForce RTX 4070", "Mali-G78 MP20". Absent when
+     * no WebGL context could be created.
+     */
+    gpuModel?: string;
     flags: {
         crossOriginIsolated: boolean;
         sharedArrayBuffer: boolean;
         wasmSimd: boolean;
+        /** `window.isSecureContext`. */
+        secureContext?: boolean;
+        /** Full WebAssembly proposal matrix (superset of `wasmSimd`). */
+        wasm?: WasmFeatureSupport;
     };
+    /** Presence checks for the APIs the runtimes depend on. */
+    apis?: APIAvailability;
+    device?: DeviceInfo;
     storage: {
         quotaBytes?: number;
         usageBytes?: number;
+        usageDetails?: Record<string, number>;
     } | null;
     power: {
         batterySupported: boolean;
         charging?: boolean;
         level?: number;
+        /** Seconds until full / empty (Infinity serialized as absent). */
+        chargingTimeSec?: number;
+        dischargingTimeSec?: number;
     };
     pressure: {
         supported: boolean;
@@ -243,7 +458,17 @@ interface EnvironmentCapture {
         height: number;
         dpr: number;
     } | null;
+    /** Detailed display snapshot (superset of `screen`). */
+    display?: DisplayInfo;
+    network?: NetworkInfo;
+    locale?: LocaleInfo;
     languages?: string[];
+    /** Raw `navigator.userAgent` — kept verbatim so future parsers can re-derive fields. */
+    userAgent?: string;
+    /** Page origin the run executed on (distinguishes production from local/staging). */
+    pageOrigin?: string;
+    /** `document.visibilityState` at capture; a hidden tab is throttled. */
+    visibilityState?: string;
     /** Free-text device self-report — displayed as "user-reported", never trusted. */
     userReportedDevice?: string;
 }
@@ -281,6 +506,16 @@ interface CellSummary {
     decodeChunksPerSec?: MetricSummary;
     prefillTokPerSecApprox?: MetricSummary;
     generatedChars?: MetricSummary;
+    /** Request wall time (startT → endT), all LLM lanes. */
+    totalMs?: MetricSummary;
+    /** End-to-end chars/s over the whole request (prefill + decode conflated). */
+    overallCharsPerSec?: MetricSummary;
+    /**
+     * False when the chunk trace is not genuinely incremental (single chunk, or
+     * the visible stream spans <20% of the request) — TTFT and decode metrics
+     * are then omitted because they would be timing artifacts, not measurements.
+     */
+    streamIncremental?: boolean;
     /** Embedding lanes. */
     singleLatencyMs?: MetricSummary;
     batchTextsPerSec?: MetricSummary;
@@ -288,19 +523,32 @@ interface CellSummary {
     loadMs?: number;
     loadCached?: boolean;
     qualityScore?: number;
+    /** Fraction of MMLU items whose answer parsed; a low value marks a format failure, not a fidelity one. */
+    qualityParseRate?: number;
     highVariance: boolean;
 }
+/** Identity of the software that produced a run. */
+interface HarnessInfo {
+    name: string;
+    version: string;
+    appVersion?: string;
+    /**
+     * Versions of the runtime packages bundled into the harness at build time
+     * (e.g. `{ "@huggingface/transformers": "4.2.0", "@wllama/wllama": "3.5.1" }`),
+     * keyed by npm package name. Per-cell `runtimeVersion` names the same value
+     * for the runtime that produced that cell.
+     */
+    runtimeVersions?: Record<string, string>;
+    /** Git commit of the harness build where the host exposes it. */
+    commit?: string;
+}
 /** The unit of submission: one full suite run on one device. */
 interface BenchRunResult {
     protocol: typeof BENCH_PROTOCOL_VERSION;
     schemaVersion: typeof BENCH_SCHEMA_VERSION;
     runId: string;
     createdAt: string;
-    harness: {
-        name: string;
-        version: string;
-        appVersion?: string;
-    };
+    harness: HarnessInfo;
     suite: BenchSuiteId;
     environment: EnvironmentCapture;
     fingerprint: FingerprintResult | null;
@@ -340,6 +588,14 @@ interface ValidationReport {
 
 /** Fixed generation budget for timed lanes (tg128, MLPerf-Client-sized). */
 declare const GENERATION_BUDGET = 128;
+/**
+ * Minimum generated characters for a timed iteration to count as a decode
+ * measurement (part of the versioned protocol). An instruct model that emits
+ * EOS after a handful of characters - typically an untemplated prompt or a
+ * tokenizer mismatch - produces no decode phase to time; such iterations are
+ * gated `degenerate-output` and the cell is marked invalid rather than scored.
+ */
+declare const MIN_GENERATED_CHARS = 16;
 /** LLM performance workloads. */
 declare const LLM_WORKLOADS: readonly LLMWorkloadSpec[];
 /** Embedding performance workloads. */
@@ -348,6 +604,25 @@ declare const EMBED_WORKLOADS: readonly EmbedWorkloadSpec[];
 declare const QUALITY_WORKLOADS: readonly QualityWorkloadSpec[];
 /** All workloads indexed by id. */
 declare const WORKLOADS_BY_ID: ReadonlyMap<string, LLMWorkloadSpec | EmbedWorkloadSpec | QualityWorkloadSpec>;
+/**
+ * Fixed runtime execution order (part of the versioned protocol). This makes
+ * the order deterministic so runtime interleaving is not a confounder across
+ * runs; it is a reproducibility measure, not a correctness fix. (The
+ * Transformers.js ORT-web lanes fail during session creation regardless of
+ * where they run in the order - a separate open runtime/adapter issue.) The
+ * order runs the WASM-arena runtimes first and the multi-GB-heap runtimes last.
+ * Within a runtime, catalog order is preserved.
+ */
+declare const RUNTIME_EXECUTION_ORDER: readonly BenchRuntimeId[];
+/**
+ * Sort planned cells into the protocol execution order (stable within a
+ * runtime). The runner applies this itself; exported for hosts and tests.
+ */
+declare function orderCells<T extends {
+    model: {
+        runtimeId: BenchRuntimeId;
+    };
+}>(cells: readonly T[]): T[];
 /** Run policy for a suite (part of the versioned protocol). */
 interface RunPolicy {
     /** Untimed warmup generations per cell (absorbs shader compile/JIT). */
@@ -508,11 +783,7 @@ interface RunSuiteOptions {
     policy: RunPolicy;
     llmAdapters: ReadonlyMap<string, LLMRuntimeAdapter>;
     embedAdapters: ReadonlyMap<string, EmbeddingRuntimeAdapter>;
-    harness: {
-        name: string;
-        version: string;
-        appVersion?: string;
-    };
+    harness: HarnessInfo;
     hooks?: RunnerHooks;
     abortSignal?: AbortSignal;
     /** Skip the fingerprint microbenchmark (tests only; submissions require it). */
@@ -539,6 +810,11 @@ declare function runBenchmarkSuite(options: RunSuiteOptions): Promise<BenchRunRe
  * provenance: UA Client Hints on Chromium; UA parsing elsewhere with the OS
  * version marked 'unknown-frozen' (UA strings are frozen by design on Gecko
  * and WebKit). Clamped fields (cores, deviceMemory) are labeled clamped.
+ *
+ * Everything a browser will disclose is recorded, whether or not the current
+ * analysis uses it: WebGPU + WebGL identity, the WebAssembly proposal matrix,
+ * API availability, form factor, display, network, locale, power. Every probe
+ * is individually guarded; a failing probe records nothing for its key.
  */
 
 /**
@@ -553,6 +829,64 @@ declare function runBenchmarkSuite(options: RunSuiteOptions): Promise<BenchRunRe
 declare function captureEnvironment(options?: {
     userReportedDevice?: string;
 }): Promise<EnvironmentCapture>;
+/**
+ * UA-string fallback for browsers without UA Client Hints (every WebKit
+ * browser). Conservative: browser name + version token, platform, and the OS
+ * version only where the UA genuinely carries one.
+ *
+ * @example
+ * parseUserAgent('... CriOS/145.0.7632.72 Mobile/15E148 Safari/604.1').browser.name; // 'Chrome iOS'
+ */
+declare function parseUserAgent(ua: string): {
+    browser: BrowserInfo;
+    os: OSInfo;
+};
+/**
+ * Rendering engine from the UA. Every iOS browser is WebKit regardless of its
+ * brand (App Store policy), so the iOS check comes first.
+ *
+ * @example
+ * detectEngine('... CriOS/145.0 ...'); // 'WebKit'
+ */
+declare function detectEngine(ua: string): BrowserInfo['engine'];
+/** Inputs to the form-factor derivation, each optional. */
+interface DeviceTypeSignals {
+    ua: string;
+    maxTouchPoints: number;
+    /** UA-CH `formFactors` high-entropy hint (Chromium 125+). */
+    formFactors?: string[];
+    /** UA-CH `mobile` bit. */
+    mobile?: boolean;
+}
+/**
+ * Derive a form factor. UA-CH form factors win when present; otherwise the UA
+ * decides, with `maxTouchPoints` unmasking an iPad that reports itself as a
+ * Mac (iPadOS 13+ default) and separating Android tablets (no `Mobile` token)
+ * from phones.
+ *
+ * @example
+ * deriveDeviceType({ ua: navigator.userAgent, maxTouchPoints: navigator.maxTouchPoints });
+ */
+declare function deriveDeviceType(signals: DeviceTypeSignals): DeviceType;
+/**
+ * GPU model from a WebGL renderer string. ANGLE wraps the model as
+ * `ANGLE (<vendor>, <model>[ (0x...)] <backend...>, <api>)`; Metal adds a
+ * `ANGLE Metal Renderer: ` prefix. Native strings are returned as-is.
+ *
+ * @example
+ * parseGpuModel('ANGLE (Apple, ANGLE Metal Renderer: Apple M4, Unspecified Version)'); // 'Apple M4'
+ */
+declare function parseGpuModel(renderer: string | null | undefined): string | undefined;
+/**
+ * WebAssembly proposal support, probed the way `wasm-feature-detect` does:
+ * validate or compile the smallest module that uses each feature. The byte
+ * sequences are those of wasm-feature-detect 1.9.0, inlined so the harness
+ * keeps its zero-dependency contract.
+ *
+ * @example
+ * const wasm = await detectWasmFeatures(); // { simd: true, threads: true, ... }
+ */
+declare function detectWasmFeatures(): Promise<WasmFeatureSupport>;
 
 /**
  * Memory sampling at protocol points (baseline / post-load / post-run).
@@ -704,19 +1038,40 @@ declare function spearman(a: readonly number[], b: readonly number[]): number;
  */
 
 /** Version of the plausibility rule set (recorded alongside moderation). */
-declare const PLAUSIBILITY_RULES_VERSION = 1;
+declare const PLAUSIBILITY_RULES_VERSION = 2;
+/**
+ * Minimum fraction of the request that the visible chunk stream must span for
+ * the trace to count as incremental. LiteRT-LM's web surface delivers every
+ * chunk in a terminal burst (~0.8ms of a 30s request), so TTFT/decode derived
+ * from such a trace are timing artifacts; a real token stream spans most of
+ * the request by construction.
+ */
+declare const STREAM_COHERENCE_MIN_SPAN_RATIO = 0.2;
 /**
  * Structural validation of a submitted result. Returns human-readable errors
  * (empty array = shape ok). Deliberately hand-rolled: this package is
  * zero-dependency and the checks double as executable schema documentation.
  */
 declare function validateRunShape(value: unknown): string[];
+/**
+ * True when an iteration's chunk trace is genuinely incremental: at least two
+ * chunks, and the visible stream spans at least
+ * `STREAM_COHERENCE_MIN_SPAN_RATIO` of the request wall time. Anything else
+ * (a doGenerate fallback's single chunk, or a runtime that computes the whole
+ * generation and flushes it in a terminal burst) carries no usable TTFT or
+ * decode timing.
+ */
+declare function isIncrementalStream(it: LLMIteration): boolean;
 /**
  * Recompute a cell's summary purely from its raw iteration traces.
  * This function IS the metric definition:
  * - TTFT = first non-empty chunk timestamp − startT
  * - decode rate = (chars after first chunk) / (endT_lastChunk − t_firstChunk)
  * - prefill tok/s ≈ approxPromptTokens / TTFT (approximate by construction)
+ * - overall rate = total chars / (endT − startT), always derivable
+ * TTFT/decode/prefill are derived only when EVERY iteration passes the
+ * stream-coherence test (`streamIncremental`); overall rate and total wall
+ * time are reported for all LLM lanes.
  */
 declare function summarizeCell(cell: BenchCellResult, highVarianceCv?: number, approxPromptTokens?: number): CellSummary;
 /** Recompute all cell summaries for a run (workload prompt sizes looked up). */
@@ -784,11 +1139,15 @@ interface LeaderboardRow {
     /** Median-of-medians metrics (only those applicable to the workload). */
     ttftMs?: number;
     decodeCharsPerSec?: number;
+    /** End-to-end chars/s (prefill + decode); the only rate for lanes whose stream is not incremental. */
+    overallCharsPerSec?: number;
     singleLatencyMs?: number;
     batchTextsPerSec?: number;
     loadColdMs?: number;
     loadWarmMs?: number;
     qualityScore?: number;
+    /** Median MMLU parse rate; below 1 the quality score is format-limited. */
+    qualityParseRate?: number;
     resolvedBackends: string[];
     browsers: string[];
     /** True when any contributing submission had a high-variance metric. */
@@ -843,26 +1202,44 @@ declare const TINY_MMLU: readonly MMLUItem[];
  * model's training data. Always run at temperature 0, outside timed regions.
  */
 
+/**
+ * Generation budget for MMLU items (part of the versioned protocol). Large
+ * enough to absorb an empty Qwen3-style `<think></think>` block plus a
+ * verbose "The answer is B." — the v1 budget of 8 truncated before any
+ * parseable letter on thinking-mode builds, scoring fidelity as 0.
+ */
+declare const MMLU_MAX_TOKENS = 48;
+/** Cap on each stored raw output (auditability without payload bloat). */
+declare const MMLU_OUTPUT_CAP = 400;
 /** Fixed MCQ prompt template (part of the versioned protocol). */
-declare function formatMMLUPrompt(item: (typeof TINY_MMLU)[number]): string;
+declare function formatMMLUPrompt(item: (typeof TINY_MMLU)[number], promptSuffix?: string): string;
 /**
- * Parse the answer letter from a model response. Accepts "B", "B.", "(B)",
- * "Answer: B", or a response beginning with the exact choice text.
+ * Parse the answer letter from a model response. Reasoning blocks are
+ * stripped first; markdown emphasis around the letter is ignored. Accepts
+ * "B", "B.", "(B)", "**B**", "Answer: B", "Option C", "choice (B)", or a
+ * response beginning with the exact choice text. Letters are matched
+ * case-sensitively after a keyword so the article "a" ("the answer is a
+ * bit unclear") is never read as answer A; a lowercase letter counts only
+ * when it is the whole reply or leads it as "b." / "b)".
  *
  * @returns 0-3, or null when no unambiguous answer is present.
  */
 declare function parseMMLUAnswer(response: string, choices: readonly string[]): number | null;
 /**
  * Run the tinyMMLU fidelity task on a language model (temperature 0,
- * `maxTokens` 8, greedy answer parsing). Unparseable answers count as wrong.
+ * `MMLU_MAX_TOKENS` budget, greedy answer parsing). Unparseable answers count
+ * as wrong; raw outputs and the parse rate are recorded so the score is
+ * auditable and recomputable server-side.
  *
  * @param model - Any structurally-compatible LanguageModel.
- * @param items - Number of items from the 100-item set (25 or 100 in v1).
- * @returns Accuracy in [0,1] with per-item correctness details.
+ * @param items - Number of items from the 100-item set (25 or 100).
+ * @returns Accuracy in [0,1] with per-item correctness details and outputs.
  */
 declare function runMMLUFidelity(model: BenchLanguageModel, items: number, options?: {
     abortSignal?: AbortSignal;
     onProgress?: (done: number, total: number) => void;
+    /** Appended to the instruction line of every item (from the model catalog). */
+    promptSuffix?: string;
 }): Promise<QualityResult>;
 /**
  * Run the STS-B embedding-quality task: Spearman correlation of cosine
@@ -898,4 +1275,4 @@ interface STSPair {
 /** 100-pair STS-B test subset (order preserved from the source dataset). */
 declare const STSB_SUBSET: readonly STSPair[];
 
-export { type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type MMLUItem, type MemorySample, type MetricSummary, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, mean, median, memoryApiAvailable, parseMMLUAnswer, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };
+export { type APIAvailability, type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, type DeviceInfo, type DeviceType, type DeviceTypeSignals, type DisplayInfo, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type HarnessInfo, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type LocaleInfo, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type NetworkInfo, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, type WasmFeatureSupport, type WebGLInfo, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deriveDeviceType, detectEngine, detectWasmFeatures, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseGpuModel, parseMMLUAnswer, parseUserAgent, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };