@localmode/bench 0.3.1 → 0.5.0

package/dist/index.d.ts

@@ -5,16 +5,20 @@
  * published statistics can be recomputed and audited server-side.
  */
 /** Protocol identifier embedded in every result. Bump only with a spec change. */
-declare const BENCH_PROTOCOL_VERSION = "localmode-bench/2";
+declare const BENCH_PROTOCOL_VERSION = "localmode-bench/3";
 /** Result JSON schema version (independent of the protocol semantics version). */
 declare const BENCH_SCHEMA_VERSION = 2;
 /** Benchmark suite presets. `custom` = user-picked cells. */
 type BenchSuiteId = 'quick' | 'standard' | 'thorough' | 'custom';
 /**
  * A benchmark runtime lane. Transformers.js is split into two lanes because
- * WebGPU vs WASM is the paper's core comparison axis.
+ * WebGPU vs WASM is the core comparison axis.
  */
-type BenchRuntimeId = 'transformers-webgpu' | 'transformers-wasm' | 'webllm' | 'wllama' | 'litert' | 'chrome-ai' | 'mediapipe';
+type BenchRuntimeId = 'transformers-webgpu' | 'transformers-wasm' | 'webllm'
+/** llama.cpp WASM on the CPU (`n_gpu_layers: 0`); before protocol v3 this lane silently ran on WebGPU where available. */
+ | 'wllama'
+/** llama.cpp WASM with every layer offloaded to WebGPU. */
+ | 'wllama-webgpu' | 'litert' | 'chrome-ai' | 'mediapipe';
 /** Workload kinds. Quality lanes are separate from timed performance lanes. */
 type BenchWorkloadKind = 'llm-generate' | 'embed-single' | 'embed-batch' | 'quality-mmlu' | 'quality-sts';
 /** A fixed LLM generation workload (public prompt, deterministic settings). */
@@ -169,25 +173,50 @@ interface BenchCellResult {
     workloadKind: BenchWorkloadKind;
     /** Backend actually used (probed, never the requested one). */
     resolvedBackend: string;
+    /**
+     * Runtime configuration the adapter reports once the model is loaded
+     * (thread count, GPU layers requested and llama.cpp's offload report,
+     * dtype, ...): the equal-care record for the lane, per cell.
+     */
+    runtimeConfig?: Record<string, string | number | boolean>;
     load: LoadRecord | null;
     /** Untimed warmup duration (ms), when a warmup ran. */
     warmupMs?: number;
     iterations: LLMIteration[] | EmbedIteration[];
+    /**
+     * Timed iterations the tab was hidden during, kept with their gates for
+     * auditability and repeated once the tab was visible again; never scored.
+     */
+    discardedIterations?: Array<LLMIteration | EmbedIteration>;
     memory?: MemorySample;
     quality?: QualityResult;
     status: BenchCellStatus;
     invalidReasons?: string[];
+    /**
+     * Failed attempts that preceded the recorded outcome (a watchdog timeout, a
+     * provider error), oldest first. The runner retries a cell up to the
+     * policy's `maxAttempts`; nothing is retried silently.
+     */
+    attempts?: Array<{
+        error: NonNullable<BenchCellResult['error']>;
+        at: number;
+    }>;
     /** Error that ended the cell; `cause` carries the wrapped provider error's message when present. */
     error?: {
         name: string;
         message: string;
+        /** Message of the wrapped provider error, when the thrown error carried a `cause`. */
         cause?: string;
+        /** `name` of the wrapped provider error (e.g. wllama's `RuntimeError` for a WASM abort). */
+        causeName?: string;
+        /** Stack of the wrapped provider error, capped at 4,000 characters; a WASM abort names its frame only here. */
+        causeStack?: string;
     };
 }
 /** Trace events global to the suite run (validity accounting). */
 interface TraceEvent {
     t: number;
-    type: 'suite-start' | 'suite-end' | 'visibility-hidden' | 'visibility-visible' | 'wakelock-acquired' | 'wakelock-released' | 'pressure-change' | 'gpu-device-lost' | 'cooldown-start' | 'cooldown-end' | 'abort';
+    type: 'suite-start' | 'suite-end' | 'visibility-hidden' | 'visibility-visible' | 'wakelock-acquired' | 'wakelock-released' | 'pressure-change' | 'gpu-device-lost' | 'cooldown-start' | 'cooldown-end' | 'cell-timeout' | 'cell-retry' | 'iteration-redo' | 'abort';
     detail?: string;
 }
 /** Browser identification with its provenance. */
@@ -314,7 +343,7 @@ interface WasmFeatureSupport {
     maxMemoryPages?: number;
 }
 /**
- * Availability of the browser APIs the runtimes and the paper care about.
+ * Availability of the browser APIs the runtimes and the analysis care about.
  * Each entry is a plain presence check (the feature exists on this page),
  * not a functional test. Chrome Built-in AI is reported by its
  * `availability()` string where the API exists.
@@ -400,7 +429,7 @@ interface EnvironmentCapture {
     browser: BrowserInfo;
     os: OSInfo;
     hardware: {
-        /** navigator.hardwareConcurrency — clamped/randomized on Gecko/WebKit. */
+        /** navigator.hardwareConcurrency — WebKit clamps it (8 on macOS, 4 on iOS); Chromium and Firefox report the real count. */
         cores: number | null;
         coresClamped: boolean;
         /** navigator.deviceMemory (GB) — Chromium-only, capped at 8. */
@@ -607,11 +636,17 @@ declare const WORKLOADS_BY_ID: ReadonlyMap<string, LLMWorkloadSpec | EmbedWorklo
 /**
  * Fixed runtime execution order (part of the versioned protocol). This makes
  * the order deterministic so runtime interleaving is not a confounder across
- * runs; it is a reproducibility measure, not a correctness fix. (The
- * Transformers.js ORT-web lanes fail during session creation regardless of
- * where they run in the order - a separate open runtime/adapter issue.) The
- * order runs the WASM-arena runtimes first and the multi-GB-heap runtimes last.
- * Within a runtime, catalog order is preserved.
+ * runs; it is a reproducibility measure, not a correctness fix. The order runs
+ * the WASM-arena runtimes first and the multi-GB-heap runtimes last; within a
+ * runtime, catalog order is preserved.
+ *
+ * The Transformers.js WASM lane runs before its WebGPU lane: Transformers.js
+ * serializes every ONNX session creation on one promise chain and never
+ * catches a rejection on it, so the first session that fails to create (a
+ * WebGPU execution provider the browser cannot initialize, an allocation
+ * failure under memory pressure) fails every later Transformers.js session in
+ * the page with the same error. Running the WASM lane first keeps a WebGPU
+ * failure from taking the CPU measurement with it.
  */
 declare const RUNTIME_EXECUTION_ORDER: readonly BenchRuntimeId[];
 /**
@@ -639,6 +674,39 @@ interface RunPolicy {
     measureWarmReload: boolean;
     /** CV above this fraction marks a summary metric as high-variance. */
     highVarianceCv: number;
+    /**
+     * Watchdog: a load that reports no progress for this long is aborted as a
+     * timeout (stall-based, so a slow link keeps downloading as long as bytes
+     * arrive).
+     */
+    loadStallMs: number;
+    /** Watchdog: absolute cap on one load attempt. */
+    loadTimeoutMs: number;
+    /**
+     * Watchdog: a generation (warmup or timed iteration) whose stream delivers
+     * nothing for this long is aborted as a timeout. A runtime surface that
+     * flushes every chunk in a terminal burst (LiteRT-LM) stays silent for the
+     * whole request, so this is well above a full request.
+     */
+    chunkStallMs: number;
+    /** Watchdog: absolute cap on one generation or embedding call. */
+    iterationTimeoutMs: number;
+    /** Watchdog: absolute cap on one quality-lane cell (many generations). */
+    qualityTimeoutMs: number;
+    /**
+     * Attempts per cell (and per model load) before the cell is recorded as an
+     * error and the run moves on. Every failed attempt stays on the cell in
+     * `attempts`; retries are never silent.
+     */
+    maxAttempts: number;
+    /**
+     * How long the runner waits for a hidden tab to become visible again before
+     * a timed iteration starts, or before it repeats an iteration the tab hid
+     * during. An iteration measured while hidden is kept on the cell in
+     * `discardedIterations` and repeated (up to `maxAttempts` times per
+     * iteration); a tab that stays hidden past this wait leaves the cell invalid.
+     */
+    visibilityWaitMs: number;
 }
 /** Policies per suite preset. */
 declare const RUN_POLICIES: Record<'quick' | 'standard' | 'thorough', RunPolicy>;
@@ -717,12 +785,15 @@ interface LoadedLLM {
     model: BenchLanguageModel;
     /** Backend actually in use — 'webgpu' | 'wasm' | 'gpu' | 'cpu' | 'chrome-builtin'. */
     resolvedBackend: string;
+    /** Post-load runtime configuration worth recording on every cell (threads, GPU layers, dtype, ...). */
+    runtimeConfig?: Record<string, string | number | boolean>;
     dispose(): Promise<void>;
 }
 /** A loaded embedding-model handle. */
 interface LoadedEmbedder {
     model: BenchEmbeddingModel;
     resolvedBackend: string;
+    runtimeConfig?: Record<string, string | number | boolean>;
     dispose(): Promise<void>;
 }
 /** Adapter for an LLM runtime lane. */
@@ -756,6 +827,37 @@ interface EmbeddingRuntimeAdapter {
 /** Usage fidelity per runtime — how trustworthy provider token counts are. */
 declare const USAGE_FIDELITY: Record<BenchRuntimeId, ProviderUsage['fidelity']>;
 
+/**
+ * Suite-level trace recorder: validity-relevant events (tab visibility, wake
+ * lock, compute pressure, GPU device loss, aborts) with wall-clock timestamps.
+ * Iterations that overlap a hidden period are invalidated, never retried
+ * silently — the trace is part of the submitted, auditable record.
+ */
+
+/** Records trace events and tracks the current validity-gate state. */
+declare class TraceRecorder {
+    private readonly events;
+    private disposers;
+    private hidden;
+    private lastPressureState;
+    private wakeLockSentinel;
+    /** Append an event at the current timestamp. */
+    record(type: TraceEvent['type'], detail?: string): void;
+    /** True while the page is hidden (timed regions overlapping this are invalid). */
+    get isHidden(): boolean;
+    /** Last observed compute-pressure state ('nominal'|'fair'|'serious'|'critical'). */
+    get pressureState(): string | undefined;
+    /** All recorded events (live reference; snapshot with `[...events]`). */
+    get all(): readonly TraceEvent[];
+    /**
+     * Attach browser listeners (visibility, compute pressure) and request a
+     * screen wake lock. Safe to call outside a browser (records nothing).
+     */
+    attach(): Promise<void>;
+    /** Release listeners and the wake lock. */
+    dispose(): Promise<void>;
+}
+
 /**
  * The suite runner: executes (runtime x model x workload) cells under the
  * versioned run policy — cache probe, load, untimed warmup, N timed runs with
@@ -767,13 +869,43 @@ declare const USAGE_FIDELITY: Record<BenchRuntimeId, ProviderUsage['fidelity']>;
 interface PlannedCell {
     model: BenchModelRef;
     workload: LLMWorkloadSpec | EmbedWorkloadSpec | QualityWorkloadSpec;
+    /**
+     * When set, the cell is recorded as `skipped` with this reason and never
+     * touches the adapter (a lane the submitter switched off, or a build the
+     * device cannot run). Keeps every cell a suite defines in the result.
+     */
+    skipReason?: string;
+}
+/** A live-activity report: something observable happened inside a cell. */
+interface RunnerActivity {
+    cellId: string;
+    /** `waiting-visible`: the tab is hidden and the runner is waiting for it to come back before timing. */
+    phase: 'load' | 'warmup' | 'iteration' | 'quality' | 'reload' | 'waiting-visible';
+    /** Current iteration or quality item (1-based) and the total, where applicable. */
+    iteration?: number;
+    total?: number;
+    /** Characters and chunks streamed so far in the current generation. */
+    chars?: number;
+    chunks?: number;
+    /** Load progress percentage, where the provider reports one. */
+    pct?: number;
 }
 /** Progress callbacks for a host UI. */
 interface RunnerHooks {
+    /** The environment capture, before the fingerprint and the first cell (lets a host persist partial progress). */
+    onEnvironment?(environment: EnvironmentCapture): void;
     onCellStart?(cellId: string, index: number, total: number): void;
     onCellFinish?(cell: BenchCellResult): void;
     onLoadProgress?(cellId: string, pct: number | undefined): void;
     onIteration?(cellId: string, iteration: number, total: number): void;
+    /**
+     * Fires on every observable step inside a cell (each streamed chunk, each
+     * load progress event, each quality item), so a host can show that the run
+     * is alive and detect a stall at a glance.
+     */
+    onActivity?(activity: RunnerActivity): void;
+    /** A cell attempt failed and the runner is about to retry it (`attempt` is the one starting, 2-based). */
+    onCellRetry?(cellId: string, attempt: number, error: NonNullable<BenchCellResult['error']>): void;
     onPhase?(phase: string): void;
 }
 /** Inputs to a suite run. */
@@ -789,7 +921,11 @@ interface RunSuiteOptions {
     /** Skip the fingerprint microbenchmark (tests only; submissions require it). */
     skipFingerprint?: boolean;
     userReportedDevice?: string;
+    /** Trace recorder to use instead of a fresh one (tests drive its visibility). */
+    trace?: TraceLike;
 }
+/** The trace recorder surface the runner needs. */
+type TraceLike = Pick<TraceRecorder, 'attach' | 'dispose' | 'record' | 'isHidden' | 'pressureState' | 'all'>;
 /**
  * Run a benchmark suite and return the full, submittable result (raw traces
  * included; `clientSummaries` computed with the same code the server uses).
@@ -857,12 +993,16 @@ interface DeviceTypeSignals {
     formFactors?: string[];
     /** UA-CH `mobile` bit. */
     mobile?: boolean;
+    /** UA-CH `platform` (e.g. "Android"), which survives a "desktop site" UA rewrite. */
+    platform?: string;
 }
 /**
- * Derive a form factor. UA-CH form factors win when present; otherwise the UA
- * decides, with `maxTouchPoints` unmasking an iPad that reports itself as a
- * Mac (iPadOS 13+ default) and separating Android tablets (no `Mobile` token)
- * from phones.
+ * Derive a form factor. A mobile operating system settles it first: Android
+ * and iOS devices are phones or tablets whatever the form-factor hint says (an
+ * unfolded Galaxy Z Fold sends `formFactors: ["Desktop"]` with a tablet-style
+ * UA). Then UA-CH form factors; then the UA, with `maxTouchPoints` unmasking
+ * an iPad that reports itself as a Mac (iPadOS 13+ default) and separating
+ * Android tablets (no `Mobile` token) from phones.
  *
  * @example
  * deriveDeviceType({ ua: navigator.userAgent, maxTouchPoints: navigator.maxTouchPoints });
@@ -960,37 +1100,6 @@ declare function runFingerprint(options?: {
     minDurationMs?: number;
 }): Promise<FingerprintResult>;
 
-/**
- * Suite-level trace recorder: validity-relevant events (tab visibility, wake
- * lock, compute pressure, GPU device loss, aborts) with wall-clock timestamps.
- * Iterations that overlap a hidden period are invalidated, never retried
- * silently — the trace is part of the submitted, auditable record.
- */
-
-/** Records trace events and tracks the current validity-gate state. */
-declare class TraceRecorder {
-    private readonly events;
-    private disposers;
-    private hidden;
-    private lastPressureState;
-    private wakeLockSentinel;
-    /** Append an event at the current timestamp. */
-    record(type: TraceEvent['type'], detail?: string): void;
-    /** True while the page is hidden (timed regions overlapping this are invalid). */
-    get isHidden(): boolean;
-    /** Last observed compute-pressure state ('nominal'|'fair'|'serious'|'critical'). */
-    get pressureState(): string | undefined;
-    /** All recorded events (live reference; snapshot with `[...events]`). */
-    get all(): readonly TraceEvent[];
-    /**
-     * Attach browser listeners (visibility, compute pressure) and request a
-     * screen wake lock. Safe to call outside a browser (records nothing).
-     */
-    attach(): Promise<void>;
-    /** Release listeners and the wake lock. */
-    dispose(): Promise<void>;
-}
-
 /**
  * Statistics for benchmark reporting: median headline, mean ± SD, IQR, 95% CI
  * (Student-t for the small n this protocol uses), CV, and geometric mean for
@@ -1130,7 +1239,7 @@ declare function computeRunDigest(result: BenchRunResult): Promise<string>;
 declare function verifyRunDigest(result: BenchRunResult): Promise<boolean>;
 
 /**
- * Aggregation for the public leaderboard and for paper analysis. Runs group
+ * Aggregation for the public leaderboard and for offline analysis. Runs group
  * into device-class rows; medians are taken per submission first, then across
  * submissions (median-of-medians). Nothing is ever averaged across devices.
  */
@@ -1179,7 +1288,7 @@ declare function aggregateRuns(runs: readonly BenchRunResult[], minSubmissions?:
 /** Leaderboard rows as CSV. */
 declare function rowsToCSV(rows: readonly LeaderboardRow[]): string;
 /**
- * Long-format per-iteration CSV for paper analysis (one row per timed
+ * Long-format per-iteration CSV for offline analysis (one row per timed
  * iteration, with full environment identity columns) — feed to R/pandas.
  */
 declare function runsToLongCSV(runs: readonly BenchRunResult[]): string;
@@ -1285,4 +1394,4 @@ interface STSPair {
 /** 100-pair STS-B test subset (order preserved from the source dataset). */
 declare const STSB_SUBSET: readonly STSPair[];
 
-export { type APIAvailability, type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, type DeviceInfo, type DeviceType, type DeviceTypeSignals, type DisplayInfo, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type HarnessInfo, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type LocaleInfo, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type NetworkInfo, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, type WasmFeatureSupport, type WebGLInfo, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deriveDeviceType, detectEngine, detectWasmFeatures, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseGpuModel, parseMMLUAnswer, parseUserAgent, quantile, resolveGpuModel, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };
+export { type APIAvailability, type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, type DeviceInfo, type DeviceType, type DeviceTypeSignals, type DisplayInfo, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type HarnessInfo, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type LocaleInfo, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type NetworkInfo, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerActivity, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, type WasmFeatureSupport, type WebGLInfo, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deriveDeviceType, detectEngine, detectWasmFeatures, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseGpuModel, parseMMLUAnswer, parseUserAgent, quantile, resolveGpuModel, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };