@localmode/bench 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,901 @@
+/**
+ * Protocol types for the LocalMode LocalMode Bench — the cross-runtime in-browser
+ * AI benchmark. A `BenchRunResult` is the unit of submission: it carries raw
+ * per-chunk timing traces (never just client-computed aggregates) so that all
+ * published statistics can be recomputed and audited server-side.
+ */
+/** Protocol identifier embedded in every result. Bump only with a spec change. */
+declare const BENCH_PROTOCOL_VERSION = "localmode-bench/1";
+/** Result JSON schema version (independent of the protocol semantics version). */
+declare const BENCH_SCHEMA_VERSION = 1;
+/** Benchmark suite presets. `custom` = user-picked cells. */
+type BenchSuiteId = 'quick' | 'standard' | 'thorough' | 'custom';
+/**
+ * A benchmark runtime lane. Transformers.js is split into two lanes because
+ * WebGPU vs WASM is the paper's core comparison axis.
+ */
+type BenchRuntimeId = 'transformers-webgpu' | 'transformers-wasm' | 'webllm' | 'wllama' | 'litert' | 'chrome-ai' | 'mediapipe';
+/** Workload kinds. Quality lanes are separate from timed performance lanes. */
+type BenchWorkloadKind = 'llm-generate' | 'embed-single' | 'embed-batch' | 'quality-mmlu' | 'quality-sts';
+/** A fixed LLM generation workload (public prompt, deterministic settings). */
+interface LLMWorkloadSpec {
+    id: string;
+    kind: 'llm-generate';
+    /** Human-readable label, e.g. "Chat pp128/tg128". */
+    label: string;
+    prompt: string;
+    systemPrompt?: string;
+    /**
+     * Approximate prompt token count (reference estimate; exact counts are
+     * tokenizer-specific and computed post-hoc from the stored prompt).
+     */
+    approxPromptTokens: number;
+    /** Generation budget. Decode metrics use the actual generated length. */
+    maxTokens: number;
+    /** Always 0 for performance runs (deterministic where the runtime allows). */
+    temperature: number;
+}
+/** A fixed embedding workload over deterministic public texts. */
+interface EmbedWorkloadSpec {
+    id: string;
+    kind: 'embed-single' | 'embed-batch';
+    label: string;
+    /** Fixed input texts. Length 1 for single-latency, N for batch throughput. */
+    texts: string[];
+}
+/** Quality-fidelity workloads (temp 0, scored, untimed-region rules relaxed). */
+interface QualityWorkloadSpec {
+    id: string;
+    kind: 'quality-mmlu' | 'quality-sts';
+    label: string;
+    /** Number of items to evaluate (subset of the bundled dataset). */
+    items: number;
+}
+type BenchWorkloadSpec = LLMWorkloadSpec | EmbedWorkloadSpec | QualityWorkloadSpec;
+/** Static reference to a model as benchmarked in one runtime lane. */
+interface BenchModelRef {
+    /** Cross-runtime pairing id, e.g. "qwen3-0.6b" — same weights family. */
+    benchModelId: string;
+    runtimeId: BenchRuntimeId;
+    /** The provider-native model id passed to the runtime. */
+    providerModelId: string;
+    displayName: string;
+    /** "llm" or "embedding". */
+    task: 'llm' | 'embedding';
+    parameterCount?: string;
+    quantization?: string;
+    sizeBytes?: number;
+    contextLength?: number;
+    /** Direct model URL when applicable (wllama GGUF, litert). */
+    url?: string;
+    requiresWebGPU?: boolean;
+}
+/** One recorded stream chunk: wall-clock timestamp + delta length in chars. */
+interface BenchChunk {
+    /** `performance.now()` at chunk receipt (ms, page-relative). */
+    t: number;
+    /** Character length of this chunk's text delta. */
+    c: number;
+}
+/** Provider-reported usage, recorded as auxiliary data (often estimated). */
+interface ProviderUsage {
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    durationMs: number;
+    /** How trustworthy the provider counts are for this runtime. */
+    fidelity: 'measured' | 'estimated' | 'chunk-count';
+}
+/** One timed LLM generation iteration with its raw trace. */
+interface LLMIteration {
+    /** `performance.now()` just before the stream is requested. */
+    startT: number;
+    /** Per-chunk receipt trace. TTFT derives from the first entry with c > 0. */
+    chunks: BenchChunk[];
+    /** `performance.now()` after the terminal chunk. */
+    endT: number;
+    /** Full generated text (enables post-hoc exact tokenization + auditing). */
+    text: string;
+    providerUsage?: ProviderUsage;
+    finishReason?: string;
+    /** Validity-gate events that fired during this iteration (empty = valid). */
+    gates: string[];
+}
+/** One timed embedding iteration. */
+interface EmbedIteration {
+    startT: number;
+    endT: number;
+    /** Number of texts embedded in this iteration. */
+    count: number;
+    /** Embedding dimensions reported by the model. */
+    dimensions: number;
+    gates: string[];
+}
+/** Model-load phase record for a cell (null when the cell reused a live model). */
+interface LoadRecord {
+    /** Provider cache probe before load: true=warm, false=cold, undefined=unknown. */
+    cached: boolean | undefined;
+    startT: number;
+    endT: number;
+    /** Progress milestones (coarse, at most ~50 samples). */
+    progress?: Array<{
+        t: number;
+        pct: number;
+    }>;
+    /** Catalog-declared download size, when known. */
+    declaredBytes?: number;
+}
+/** Memory sampling at protocol points (Chromium-only APIs; absent elsewhere). */
+interface MemorySample {
+    /** Bytes reported at suite baseline, after load, after timed runs. */
+    baseline?: number;
+    postLoad?: number;
+    postRun?: number;
+    api: 'uaSpecific' | 'legacyHeap' | 'none';
+}
+/** Quality lane outcome attached to a cell. */
+interface QualityResult {
+    taskId: string;
+    /** Primary score: accuracy (MMLU) or Spearman rho (STS). */
+    score: number;
+    n: number;
+    /** Per-item correctness or per-pair cosine, for auditability. */
+    details?: number[];
+}
+type BenchCellStatus = 'ok' | 'invalid' | 'error' | 'skipped';
+/** One benchmark cell: (runtime x model x workload) with its raw iterations. */
+interface BenchCellResult {
+    /** `${runtimeId}/${benchModelId}/${workloadId}` */
+    cellId: string;
+    runtimeId: BenchRuntimeId;
+    runtimeVersion?: string;
+    model: BenchModelRef;
+    workloadId: string;
+    workloadKind: BenchWorkloadKind;
+    /** Backend actually used (probed, never the requested one). */
+    resolvedBackend: string;
+    load: LoadRecord | null;
+    /** Untimed warmup duration (ms), when a warmup ran. */
+    warmupMs?: number;
+    iterations: LLMIteration[] | EmbedIteration[];
+    memory?: MemorySample;
+    quality?: QualityResult;
+    status: BenchCellStatus;
+    invalidReasons?: string[];
+    error?: {
+        name: string;
+        message: string;
+    };
+}
+/** Trace events global to the suite run (validity accounting). */
+interface TraceEvent {
+    t: number;
+    type: 'suite-start' | 'suite-end' | 'visibility-hidden' | 'visibility-visible' | 'wakelock-acquired' | 'wakelock-released' | 'pressure-change' | 'gpu-device-lost' | 'cooldown-start' | 'cooldown-end' | 'abort';
+    detail?: string;
+}
+/** Browser identification with its provenance. */
+interface BrowserInfo {
+    name: string;
+    version: string;
+    source: 'ua-ch' | 'ua-parse';
+    brands?: Array<{
+        brand: string;
+        version: string;
+    }>;
+}
+/** OS identification. Version is 'unknown-frozen' on engines with frozen UAs. */
+interface OSInfo {
+    platform: string;
+    version: string;
+    architecture?: string;
+    bitness?: string;
+    model?: string;
+}
+/** WebGPU adapter identity + selected limits (all fields may be empty strings). */
+interface GPUInfo {
+    available: boolean;
+    vendor?: string;
+    architecture?: string;
+    device?: string;
+    description?: string;
+    isFallbackAdapter?: boolean;
+    features?: string[];
+    limits?: Record<string, number>;
+}
+/** Full environment capture for a run. Clamped fields are labeled as such. */
+interface EnvironmentCapture {
+    capturedAt: string;
+    browser: BrowserInfo;
+    os: OSInfo;
+    hardware: {
+        /** navigator.hardwareConcurrency — clamped/randomized on Gecko/WebKit. */
+        cores: number | null;
+        coresClamped: boolean;
+        /** navigator.deviceMemory (GB) — Chromium-only, capped at 8. */
+        deviceMemoryGB: number | null;
+        deviceMemoryCapped: boolean;
+    };
+    gpu: GPUInfo;
+    /** WebGL renderer string, a secondary GPU identity signal. */
+    webglRenderer: string | null;
+    flags: {
+        crossOriginIsolated: boolean;
+        sharedArrayBuffer: boolean;
+        wasmSimd: boolean;
+    };
+    storage: {
+        quotaBytes?: number;
+        usageBytes?: number;
+    } | null;
+    power: {
+        batterySupported: boolean;
+        charging?: boolean;
+        level?: number;
+    };
+    pressure: {
+        supported: boolean;
+        lastState?: string;
+    };
+    /** Inferred performance.now() quantum in microseconds (grid inference). */
+    timerResolutionUs: number | null;
+    screen: {
+        width: number;
+        height: number;
+        dpr: number;
+    } | null;
+    languages?: string[];
+    /** Free-text device self-report — displayed as "user-reported", never trusted. */
+    userReportedDevice?: string;
+}
+/** Deterministic JS matmul microbenchmark result (hardware fingerprint). */
+interface FingerprintResult {
+    /** Millions of fused multiply-adds per second. */
+    mflops: number;
+    n: number;
+    iterations: number;
+    durationMs: number;
+    /** Checksum of the final matrix — proves the work actually ran. */
+    checksum: number;
+}
+/** Statistical summary of one metric across iterations. */
+interface MetricSummary {
+    n: number;
+    median: number;
+    mean: number;
+    sd: number;
+    iqr: number;
+    min: number;
+    max: number;
+    /** 95% confidence interval half-width (Student-t). */
+    ci95: number;
+    /** Coefficient of variation (sd/mean), 0 when mean is 0. */
+    cv: number;
+}
+/** Derived per-cell summary (recomputable from the raw trace by anyone). */
+interface CellSummary {
+    cellId: string;
+    status: BenchCellStatus;
+    /** LLM lanes. */
+    ttftMs?: MetricSummary;
+    decodeCharsPerSec?: MetricSummary;
+    decodeChunksPerSec?: MetricSummary;
+    prefillTokPerSecApprox?: MetricSummary;
+    generatedChars?: MetricSummary;
+    /** Embedding lanes. */
+    singleLatencyMs?: MetricSummary;
+    batchTextsPerSec?: MetricSummary;
+    /** Load phase. */
+    loadMs?: number;
+    loadCached?: boolean;
+    qualityScore?: number;
+    highVariance: boolean;
+}
+/** The unit of submission: one full suite run on one device. */
+interface BenchRunResult {
+    protocol: typeof BENCH_PROTOCOL_VERSION;
+    schemaVersion: typeof BENCH_SCHEMA_VERSION;
+    runId: string;
+    createdAt: string;
+    harness: {
+        name: string;
+        version: string;
+        appVersion?: string;
+    };
+    suite: BenchSuiteId;
+    environment: EnvironmentCapture;
+    fingerprint: FingerprintResult | null;
+    cells: BenchCellResult[];
+    events: TraceEvent[];
+    /** Client-computed summaries (advisory; server recomputes from the trace). */
+    clientSummaries?: CellSummary[];
+    /** Server-issued anti-forgery nonce (verified tier only). */
+    nonce?: string;
+    /** SHA-256 of the canonical JSON of this object without `digest`. */
+    digest?: string;
+}
+/** A plausibility/integrity finding attached by validation. */
+interface PlausibilityFlag {
+    code: string;
+    cellId?: string;
+    message: string;
+    severity: 'warn' | 'reject';
+}
+/** Outcome of full validation (shape + recompute + plausibility). */
+interface ValidationReport {
+    ok: boolean;
+    shapeErrors: string[];
+    flags: PlausibilityFlag[];
+    summaries: CellSummary[];
+}
+
+/**
+ * Protocol constants: fixed public workloads and the run policy. These are
+ * part of the versioned benchmark spec — any change to prompts, token budgets,
+ * or policy numbers requires a protocol version bump (see BENCH_PROTOCOL_VERSION).
+ *
+ * Metric naming follows llama-bench conventions (ppN = prefill on an
+ * N-token prompt, tgN = token generation over N tokens) and MLPerf Client
+ * definitions (TTFT; decode rate excludes the first token).
+ */
+
+/** Fixed generation budget for timed lanes (tg128, MLPerf-Client-sized). */
+declare const GENERATION_BUDGET = 128;
+/** LLM performance workloads. */
+declare const LLM_WORKLOADS: readonly LLMWorkloadSpec[];
+/** Embedding performance workloads. */
+declare const EMBED_WORKLOADS: readonly EmbedWorkloadSpec[];
+/** Quality-fidelity workloads (separate lane, temperature 0). */
+declare const QUALITY_WORKLOADS: readonly QualityWorkloadSpec[];
+/** All workloads indexed by id. */
+declare const WORKLOADS_BY_ID: ReadonlyMap<string, LLMWorkloadSpec | EmbedWorkloadSpec | QualityWorkloadSpec>;
+/** Run policy for a suite (part of the versioned protocol). */
+interface RunPolicy {
+    /** Untimed warmup generations per cell (absorbs shader compile/JIT). */
+    warmupRuns: number;
+    /** Timed iterations per cell. */
+    timedRuns: number;
+    /** Idle cool-down between cells (thermal recovery), ms. */
+    cooldownMs: number;
+    /** Gate the next cell on Compute Pressure <= 'fair' when available. */
+    pressureGate: boolean;
+    /** Max wait for the pressure gate before proceeding anyway, ms. */
+    pressureGateTimeoutMs: number;
+    /** After a cold load, dispose + reload to also measure the warm load. */
+    measureWarmReload: boolean;
+    /** CV above this fraction marks a summary metric as high-variance. */
+    highVarianceCv: number;
+}
+/** Policies per suite preset. */
+declare const RUN_POLICIES: Record<'quick' | 'standard' | 'thorough', RunPolicy>;
+
+/**
+ * Runtime adapter contract. Adapters wire a browser ML runtime (Transformers.js,
+ * WebLLM, wllama, LiteRT, Chrome Built-in AI, MediaPipe) to the runner. They are
+ * injected by the harness host (e.g. the localmode.ai bench page) so this
+ * package stays free of provider dependencies — the same pattern the LocalMode
+ * blocks use for provider wiring.
+ *
+ * The runner consumes the `@localmode/core` model interfaces structurally
+ * (type-only peer dependency): any object with a compatible `doStream`/`doEmbed`
+ * works, including models from other libraries.
+ */
+
+/** Structural subset of `@localmode/core`'s StreamChunk. */
+interface BenchStreamChunk {
+    text: string;
+    done: boolean;
+    finishReason?: string;
+    usage?: {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+        durationMs: number;
+    };
+}
+/** Structural subset of `@localmode/core`'s LanguageModel. */
+interface BenchLanguageModel {
+    readonly modelId: string;
+    readonly provider: string;
+    doStream?(options: {
+        prompt: string;
+        systemPrompt?: string;
+        maxTokens?: number;
+        temperature?: number;
+        abortSignal?: AbortSignal;
+    }): AsyncIterable<BenchStreamChunk>;
+    doGenerate(options: {
+        prompt: string;
+        systemPrompt?: string;
+        maxTokens?: number;
+        temperature?: number;
+        abortSignal?: AbortSignal;
+    }): Promise<{
+        text: string;
+        finishReason: string;
+        usage: ProviderUsage | Omit<ProviderUsage, 'fidelity'>;
+    }>;
+}
+/** Structural subset of `@localmode/core`'s EmbeddingModel. */
+interface BenchEmbeddingModel {
+    readonly modelId: string;
+    readonly provider: string;
+    readonly dimensions: number;
+    doEmbed(options: {
+        values: string[];
+        abortSignal?: AbortSignal;
+    }): Promise<{
+        embeddings: Float32Array[];
+    }>;
+}
+/** Availability probe result for a runtime lane on this device. */
+interface AdapterAvailability {
+    ok: boolean;
+    /** Human-readable reason when not available ('no WebGPU adapter', ...). */
+    reason?: string;
+}
+/** Load-progress callback payload (normalized percent when known). */
+interface AdapterLoadProgress {
+    pct?: number;
+}
+/** A loaded LLM handle: the model plus its resolved execution backend. */
+interface LoadedLLM {
+    model: BenchLanguageModel;
+    /** Backend actually in use — 'webgpu' | 'wasm' | 'gpu' | 'cpu' | 'chrome-builtin'. */
+    resolvedBackend: string;
+    dispose(): Promise<void>;
+}
+/** A loaded embedding-model handle. */
+interface LoadedEmbedder {
+    model: BenchEmbeddingModel;
+    resolvedBackend: string;
+    dispose(): Promise<void>;
+}
+/** Adapter for an LLM runtime lane. */
+interface LLMRuntimeAdapter {
+    readonly runtimeId: BenchRuntimeId;
+    readonly displayName: string;
+    /** Underlying runtime library version, when known (recorded per cell). */
+    readonly runtimeVersion?: string;
+    /** Is this lane usable on the current device/browser? */
+    isAvailable(): Promise<AdapterAvailability>;
+    /** Provider cache probe — true=warm, false=cold, undefined=unknown. */
+    isModelCached(model: BenchModelRef): Promise<boolean | undefined>;
+    /** Load (download + compile) the model. Must NOT run any inference. */
+    load(model: BenchModelRef, options: {
+        onProgress?: (p: AdapterLoadProgress) => void;
+        abortSignal?: AbortSignal;
+    }): Promise<LoadedLLM>;
+}
+/** Adapter for an embedding runtime lane. */
+interface EmbeddingRuntimeAdapter {
+    readonly runtimeId: BenchRuntimeId;
+    readonly displayName: string;
+    readonly runtimeVersion?: string;
+    isAvailable(): Promise<AdapterAvailability>;
+    isModelCached(model: BenchModelRef): Promise<boolean | undefined>;
+    load(model: BenchModelRef, options: {
+        onProgress?: (p: AdapterLoadProgress) => void;
+        abortSignal?: AbortSignal;
+    }): Promise<LoadedEmbedder>;
+}
+/** Usage fidelity per runtime — how trustworthy provider token counts are. */
+declare const USAGE_FIDELITY: Record<BenchRuntimeId, ProviderUsage['fidelity']>;
+
+/**
+ * The suite runner: executes (runtime x model x workload) cells under the
+ * versioned run policy — cache probe, load, untimed warmup, N timed runs with
+ * raw per-chunk traces, validity gates, memory protocol points, cool-downs —
+ * and assembles the submittable BenchRunResult.
+ */
+
+/** One planned cell. Cells sharing (runtimeId, model) reuse the loaded model. */
+interface PlannedCell {
+    model: BenchModelRef;
+    workload: LLMWorkloadSpec | EmbedWorkloadSpec | QualityWorkloadSpec;
+}
+/** Progress callbacks for a host UI. */
+interface RunnerHooks {
+    onCellStart?(cellId: string, index: number, total: number): void;
+    onCellFinish?(cell: BenchCellResult): void;
+    onLoadProgress?(cellId: string, pct: number | undefined): void;
+    onIteration?(cellId: string, iteration: number, total: number): void;
+    onPhase?(phase: string): void;
+}
+/** Inputs to a suite run. */
+interface RunSuiteOptions {
+    suite: BenchSuiteId;
+    cells: PlannedCell[];
+    policy: RunPolicy;
+    llmAdapters: ReadonlyMap<string, LLMRuntimeAdapter>;
+    embedAdapters: ReadonlyMap<string, EmbeddingRuntimeAdapter>;
+    harness: {
+        name: string;
+        version: string;
+        appVersion?: string;
+    };
+    hooks?: RunnerHooks;
+    abortSignal?: AbortSignal;
+    /** Skip the fingerprint microbenchmark (tests only; submissions require it). */
+    skipFingerprint?: boolean;
+    userReportedDevice?: string;
+}
+/**
+ * Run a benchmark suite and return the full, submittable result (raw traces
+ * included; `clientSummaries` computed with the same code the server uses).
+ *
+ * Cells are grouped by (runtime, model): the model loads once, its workloads
+ * run back-to-back, then it is disposed before the next model (peak-memory
+ * hygiene). Validity gates never retry silently — affected iterations carry
+ * their gate events and the cell is marked invalid.
+ *
+ * @throws {DOMException} AbortError when `abortSignal` aborts.
+ * @example
+ * const result = await runBenchmarkSuite({ suite: 'quick', cells, policy, ... });
+ */
+declare function runBenchmarkSuite(options: RunSuiteOptions): Promise<BenchRunResult>;
+
+/**
+ * Environment capture. Records identity + capability signals with honest
+ * provenance: UA Client Hints on Chromium; UA parsing elsewhere with the OS
+ * version marked 'unknown-frozen' (UA strings are frozen by design on Gecko
+ * and WebKit). Clamped fields (cores, deviceMemory) are labeled clamped.
+ */
+
+/**
+ * Capture the full benchmark environment. Never throws — every probe is
+ * individually guarded and absent signals are recorded as absent.
+ *
+ * @param options.userReportedDevice - Optional free-text device self-report
+ *   (displayed as "user-reported", never treated as ground truth).
+ * @example
+ * const env = await captureEnvironment();
+ */
+declare function captureEnvironment(options?: {
+    userReportedDevice?: string;
+}): Promise<EnvironmentCapture>;
+
+/**
+ * Memory sampling at protocol points (baseline / post-load / post-run).
+ * Uses `performance.measureUserAgentSpecificMemory()` (Chromium, requires
+ * cross-origin isolation, resolves at the next GC — up to ~20 s), with the
+ * legacy Chromium JS-heap as fallback. Never call inside a timed region.
+ */
+/** Which memory API is usable in this context. */
+declare function memoryApiAvailable(): 'uaSpecific' | 'legacyHeap' | 'none';
+/**
+ * Sample total page memory in bytes, or null when unavailable / timed out.
+ * The UA-specific measurement is raced against `timeoutMs` because it only
+ * resolves at the next garbage collection.
+ *
+ * @param timeoutMs - Max wait for the GC-gated measurement (default 25000).
+ * @example
+ * const bytes = await sampleMemoryBytes(); // 734003200 | null
+ */
+declare function sampleMemoryBytes(timeoutMs?: number): Promise<number | null>;
+
+/**
+ * Wall-clock timing helpers. All benchmark timestamps use `performance.now()`
+ * (5 µs resolution on cross-origin-isolated Chromium, coarser elsewhere);
+ * decode rates are derived from endpoint timestamps, never from averaged
+ * per-token deltas, because per-token gaps sit at or below timer quantization.
+ */
+/** Monotonic high-resolution timestamp in ms (performance.now, Date fallback). */
+declare function hrNow(): number;
+/**
+ * Infer the effective `performance.now()` quantum by sampling timer deltas.
+ * Genuine traces sit on this grid; the value is recorded in the environment
+ * capture and used by the timer-grid integrity check.
+ *
+ * @param samples - Number of distinct increments to observe (default 64).
+ * @returns Inferred quantum in microseconds, or null outside a browser.
+ * @example
+ * const q = inferTimerResolutionUs(); // 5 on isolated Chromium
+ */
+declare function inferTimerResolutionUs(samples?: number): number | null;
+/** Await a real delay (setTimeout) — used for cool-downs between cells. */
+declare function sleep(ms: number, abortSignal?: AbortSignal): Promise<void>;
+
+/**
+ * Deterministic single-thread JS matmul microbenchmark. Every submission runs
+ * it; the resulting MFLOPS is a hardware fingerprint that claimed inference
+ * rates must plausibly correlate with, and the checksum proves the work ran.
+ */
+
+/**
+ * Run the fingerprint microbenchmark: repeated n x n f32 matmuls for at least
+ * `minDurationMs`, single-threaded, deterministic inputs.
+ *
+ * @param options.n - Matrix dimension (default 160; ~8.2 MFLOP per iteration).
+ * @param options.minDurationMs - Minimum wall time to run (default 600).
+ * @returns MFLOPS, iteration count, and a checksum of the final product.
+ * @example
+ * const fp = await runFingerprint(); // { mflops: 1800.4, ... }
+ */
+declare function runFingerprint(options?: {
+    n?: number;
+    minDurationMs?: number;
+}): Promise<FingerprintResult>;
+
+/**
+ * Suite-level trace recorder: validity-relevant events (tab visibility, wake
+ * lock, compute pressure, GPU device loss, aborts) with wall-clock timestamps.
+ * Iterations that overlap a hidden period are invalidated, never retried
+ * silently — the trace is part of the submitted, auditable record.
+ */
+
+/** Records trace events and tracks the current validity-gate state. */
+declare class TraceRecorder {
+    private readonly events;
+    private disposers;
+    private hidden;
+    private lastPressureState;
+    private wakeLockSentinel;
+    /** Append an event at the current timestamp. */
+    record(type: TraceEvent['type'], detail?: string): void;
+    /** True while the page is hidden (timed regions overlapping this are invalid). */
+    get isHidden(): boolean;
+    /** Last observed compute-pressure state ('nominal'|'fair'|'serious'|'critical'). */
+    get pressureState(): string | undefined;
+    /** All recorded events (live reference; snapshot with `[...events]`). */
+    get all(): readonly TraceEvent[];
+    /**
+     * Attach browser listeners (visibility, compute pressure) and request a
+     * screen wake lock. Safe to call outside a browser (records nothing).
+     */
+    attach(): Promise<void>;
+    /** Release listeners and the wake lock. */
+    dispose(): Promise<void>;
+}
+
+/**
+ * Statistics for benchmark reporting: median headline, mean ± SD, IQR, 95% CI
+ * (Student-t for the small n this protocol uses), CV, and geometric mean for
+ * cross-workload aggregation (never across devices).
+ */
+
+/**
+ * Linear-interpolated quantile (type-7, the R/NumPy default).
+ *
+ * @param values - Sample values (need not be sorted).
+ * @param q - Quantile in [0, 1].
+ * @returns The interpolated quantile.
+ * @throws {RangeError} When `values` is empty or `q` is out of range.
+ * @example
+ * quantile([1, 2, 3, 4], 0.5); // 2.5
+ */
+declare function quantile(values: readonly number[], q: number): number;
+/** Median of a sample. @see quantile */
+declare function median(values: readonly number[]): number;
+/** Arithmetic mean. @throws {RangeError} on an empty sample. */
+declare function mean(values: readonly number[]): number;
+/** Sample standard deviation (n-1 denominator). Returns 0 for n < 2. */
+declare function stddev(values: readonly number[]): number;
+/**
+ * Geometric mean. Used to aggregate across workloads within one device run
+ * (Speedometer/JetStream convention). All values must be positive.
+ *
+ * @throws {RangeError} on an empty sample or non-positive values.
+ */
+declare function geomean(values: readonly number[]): number;
+/**
+ * Full summary of one metric across timed iterations.
+ *
+ * @param values - One value per timed iteration (ms, tok/s, ...).
+ * @returns Median headline plus dispersion measures; `ci95` is the Student-t
+ *   95% half-width (0 when n < 2).
+ * @example
+ * summarize([102, 99, 104]).median; // 102
+ */
+declare function summarize(values: readonly number[]): MetricSummary;
+/**
+ * Spearman rank correlation between two equal-length samples, with average
+ * ranks for ties. Used by the STS embedding-quality lane.
+ *
+ * @returns rho in [-1, 1].
+ * @throws {RangeError} when lengths differ or n < 2.
+ */
+declare function spearman(a: readonly number[], b: readonly number[]): number;
+
+/**
+ * Validation: structural schema checks, deterministic summary recomputation
+ * from raw traces (the same code runs client- and server-side), and versioned
+ * plausibility/integrity checks for community submissions. Flags are attached,
+ * never silently dropped — flagged runs are quarantined publicly.
+ */
+
+/** Version of the plausibility rule set (recorded alongside moderation). */
+declare const PLAUSIBILITY_RULES_VERSION = 1;
+/**
+ * Structural validation of a submitted result. Returns human-readable errors
+ * (empty array = shape ok). Deliberately hand-rolled: this package is
+ * zero-dependency and the checks double as executable schema documentation.
+ */
+declare function validateRunShape(value: unknown): string[];
+/**
+ * Recompute a cell's summary purely from its raw iteration traces.
+ * This function IS the metric definition:
+ * - TTFT = first non-empty chunk timestamp − startT
+ * - decode rate = (chars after first chunk) / (endT_lastChunk − t_firstChunk)
+ * - prefill tok/s ≈ approxPromptTokens / TTFT (approximate by construction)
+ */
+declare function summarizeCell(cell: BenchCellResult, highVarianceCv?: number, approxPromptTokens?: number): CellSummary;
+/** Recompute all cell summaries for a run (workload prompt sizes looked up). */
+declare function summarizeRun(run: BenchRunResult, highVarianceCv?: number, promptTokensByWorkload?: ReadonlyMap<string, number>): CellSummary[];
+/**
+ * Versioned plausibility/integrity checks over a submitted run. Returns flags;
+ * `severity: 'reject'` flags route the run to quarantine, `warn` flags are
+ * displayed. Rules (v1): timestamp monotonicity, decode-rate envelopes,
+ * duration consistency, timer-grid conformance, fingerprint sanity,
+ * environment cross-field consistency.
+ */
+declare function checkPlausibility(run: BenchRunResult): PlausibilityFlag[];
+/**
+ * Full validation pipeline for a submission: shape, digest-independent
+ * summary recompute, client-summary agreement (>1% relative disagreement on
+ * medians is flagged), and plausibility rules.
+ */
+declare function validateSubmission(run: BenchRunResult): ValidationReport;
+
+/**
+ * Canonical JSON serialization + SHA-256 digest for run results. The digest
+ * covers the canonical form of the result WITHOUT its `digest` field, so any
+ * party can recompute and verify it (isomorphic: browser and Node).
+ */
+
+/**
+ * Deterministic JSON: object keys sorted lexicographically at every level,
+ * arrays in order, no whitespace. Non-finite numbers serialize as null
+ * (JSON.stringify semantics) so digests stay portable.
+ *
+ * @example
+ * canonicalJson({ b: 1, a: [2, { d: 3, c: 4 }] }); // '{"a":[2,{"c":4,"d":3}],"b":1}'
+ */
+declare function canonicalJson(value: unknown): string;
+/** SHA-256 hex of a UTF-8 string via Web Crypto (browser + Node >= 19). */
+declare function sha256Hex(text: string): Promise<string>;
+/**
+ * Compute the run digest: SHA-256 of the canonical JSON of the result with
+ * `digest` removed.
+ *
+ * @example
+ * result.digest = await computeRunDigest(result);
+ */
+declare function computeRunDigest(result: BenchRunResult): Promise<string>;
+/** Verify a result's embedded digest. Returns false when absent or wrong. */
+declare function verifyRunDigest(result: BenchRunResult): Promise<boolean>;
+
+/**
+ * Aggregation for the public leaderboard and for paper analysis. Runs group
+ * into device-class rows; medians are taken per submission first, then across
+ * submissions (median-of-medians). Nothing is ever averaged across devices.
+ */
+
+/** Device class derived from environment identity signals. */
+declare function deviceClassOf(run: BenchRunResult): string;
+/** One leaderboard row: a (deviceClass, runtime, model, workload) group. */
+interface LeaderboardRow {
+    deviceClass: string;
+    runtimeId: string;
+    benchModelId: string;
+    modelName: string;
+    workloadId: string;
+    /** Number of distinct submissions contributing. */
+    submissions: number;
+    /** Median-of-medians metrics (only those applicable to the workload). */
+    ttftMs?: number;
+    decodeCharsPerSec?: number;
+    singleLatencyMs?: number;
+    batchTextsPerSec?: number;
+    loadColdMs?: number;
+    loadWarmMs?: number;
+    qualityScore?: number;
+    resolvedBackends: string[];
+    browsers: string[];
+    /** True when any contributing submission had a high-variance metric. */
+    highVariance: boolean;
+    /** Rows below the min-N threshold are provisional. */
+    provisional: boolean;
+}
+/** Minimum concordant submissions for a non-provisional headline row. */
+declare const HEADLINE_MIN_SUBMISSIONS = 3;
+/**
+ * Aggregate validated runs into leaderboard rows.
+ *
+ * @param runs - Validated (non-quarantined) run results.
+ * @param minSubmissions - Min submissions before a row loses `provisional`.
+ * @example
+ * const rows = aggregateRuns(allRuns);
+ */
+declare function aggregateRuns(runs: readonly BenchRunResult[], minSubmissions?: number): LeaderboardRow[];
+/** Leaderboard rows as CSV. */
+declare function rowsToCSV(rows: readonly LeaderboardRow[]): string;
+/**
+ * Long-format per-iteration CSV for paper analysis (one row per timed
+ * iteration, with full environment identity columns) — feed to R/pandas.
+ */
+declare function runsToLongCSV(runs: readonly BenchRunResult[]): string;
+
+/**
+ * tinyMMLU — 100 IRT-anchored items from the MMLU benchmark, selected by the
+ * tinyBenchmarks project (Maia Polo et al., "tinyBenchmarks: evaluating LLMs
+ * with fewer examples", arXiv:2402.14992). Source dataset:
+ * https://huggingface.co/datasets/tinyBenchmarks/tinyMMLU (test split, 100 rows).
+ * Upstream MMLU (cais/mmlu) and the tinyBenchmarks repository are MIT-licensed.
+ *
+ * Used by the quality-fidelity lane: scores measure runtime/quantization
+ * fidelity of a model build, not model capability (public sets are assumed
+ * to be in training data).
+ */
+/** One multiple-choice item: four choices, `answer` is the 0-based correct index. */
+interface MMLUItem {
+    question: string;
+    subject: string;
+    choices: [string, string, string, string];
+    answer: 0 | 1 | 2 | 3;
+}
+/** The 100-item tinyMMLU test split (order preserved from the source dataset). */
+declare const TINY_MMLU: readonly MMLUItem[];
+
+/**
+ * Quality-fidelity lane. Scores measure runtime/quantization fidelity of a
+ * model build (do this runtime's weights + kernels reproduce expected answers),
+ * NOT model capability — the bundled public sets are assumed to be in every
+ * model's training data. Always run at temperature 0, outside timed regions.
+ */
+
+/** Fixed MCQ prompt template (part of the versioned protocol). */
+declare function formatMMLUPrompt(item: (typeof TINY_MMLU)[number]): string;
+/**
+ * Parse the answer letter from a model response. Accepts "B", "B.", "(B)",
+ * "Answer: B", or a response beginning with the exact choice text.
+ *
+ * @returns 0-3, or null when no unambiguous answer is present.
+ */
+declare function parseMMLUAnswer(response: string, choices: readonly string[]): number | null;
+/**
+ * Run the tinyMMLU fidelity task on a language model (temperature 0,
+ * `maxTokens` 8, greedy answer parsing). Unparseable answers count as wrong.
+ *
+ * @param model - Any structurally-compatible LanguageModel.
+ * @param items - Number of items from the 100-item set (25 or 100 in v1).
+ * @returns Accuracy in [0,1] with per-item correctness details.
+ */
+declare function runMMLUFidelity(model: BenchLanguageModel, items: number, options?: {
+    abortSignal?: AbortSignal;
+    onProgress?: (done: number, total: number) => void;
+}): Promise<QualityResult>;
+/**
+ * Run the STS-B embedding-quality task: Spearman correlation of cosine
+ * similarities against human scores over the bundled pair subset.
+ *
+ * @param model - Any structurally-compatible EmbeddingModel.
+ * @param pairs - Number of pairs from the 100-pair set.
+ * @returns Spearman rho with per-pair cosine details.
+ */
+declare function runSTSQuality(model: BenchEmbeddingModel, pairs: number, options?: {
+    abortSignal?: AbortSignal;
+    onProgress?: (done: number, total: number) => void;
+}): Promise<QualityResult>;
+
+/**
+ * STS Benchmark subset — the first 100 pairs of the STS-B test split
+ * (semantic textual similarity; human scores 0-5). Source:
+ * https://huggingface.co/datasets/mteb/stsbenchmark-sts (test split).
+ *
+ * LICENSE: CC BY-SA 4.0 (share-alike). This file and any redistribution of the
+ * pairs below remain under CC BY-SA 4.0 — see LICENSE-CC-BY-SA.md in this
+ * directory. Kept isolated from the MIT-licensed code and datasets on purpose.
+ *
+ * Used by the embedding quality lane: Spearman correlation of cosine
+ * similarities against the human scores.
+ */
+/** One sentence pair with its human similarity score (0-5). */
+interface STSPair {
+    s1: string;
+    s2: string;
+    score: number;
+}
+/** 100-pair STS-B test subset (order preserved from the source dataset). */
+declare const STSB_SUBSET: readonly STSPair[];
+
+export { type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type MMLUItem, type MemorySample, type MetricSummary, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, mean, median, memoryApiAvailable, parseMMLUAnswer, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };