@localmode/bench 0.2.0 → 0.3.0

package/dist/index.d.cts

@@ -199,6 +199,14 @@ interface BrowserInfo {
         brand: string;
         version: string;
     }>;
+    /** Rendering engine, derived from the UA (Blink / Gecko / WebKit). */
+    engine?: 'Blink' | 'Gecko' | 'WebKit' | 'unknown';
+    /** `navigator.vendor` (e.g. "Google Inc.", "Apple Computer, Inc."). */
+    vendor?: string;
+    /** `navigator.webdriver` — true under browser automation. */
+    webdriver?: boolean;
+    /** `navigator.pdfViewerEnabled`, a cheap headless/kiosk signal. */
+    pdfViewerEnabled?: boolean;
 }
 /** OS identification. Version is 'unknown-frozen' on engines with frozen UAs. */
 interface OSInfo {
@@ -206,7 +214,30 @@ interface OSInfo {
     version: string;
     architecture?: string;
     bitness?: string;
+    /** Device model from UA-CH (Android only; empty elsewhere by spec). */
     model?: string;
+    /** Whether the OS is running in a WoW64-style emulation layer (UA-CH `wow64`). */
+    wow64?: boolean;
+    /** `navigator.platform` (legacy, frozen but still informative: "MacIntel", "Win32", "iPhone"). */
+    navigatorPlatform?: string;
+}
+/** Form-factor classification, derived from UA-CH form factors, the UA, and touch. */
+type DeviceType = 'phone' | 'tablet' | 'desktop' | 'xr' | 'tv' | 'unknown';
+/** Device / form-factor signals. */
+interface DeviceInfo {
+    type: DeviceType;
+    /** UA-CH `mobile` bit (Chromium) or a UA-derived guess elsewhere. */
+    mobile: boolean;
+    /** UA-CH `formFactors` (Chromium 125+): Desktop / Mobile / Tablet / XR / EInk / Watch / Automotive. */
+    formFactors?: string[];
+    /** `navigator.maxTouchPoints`. */
+    maxTouchPoints: number;
+    /** `(pointer: coarse)` media query — primary input is a touch surface. */
+    pointerCoarse?: boolean;
+    /** `(hover: none)` media query — no hover-capable primary input. */
+    hoverNone?: boolean;
+    /** Standalone / fullscreen display mode (installed PWA or kiosk). */
+    displayMode?: string;
 }
 /** WebGPU adapter identity + selected limits (all fields may be empty strings). */
 interface GPUInfo {
@@ -215,11 +246,155 @@ interface GPUInfo {
     architecture?: string;
     device?: string;
     description?: string;
+    /** Adapter-reported subgroup sizes (Chromium 130+ `info.subgroupMinSize/MaxSize`). */
+    subgroupMinSize?: number;
+    subgroupMaxSize?: number;
     isFallbackAdapter?: boolean;
     features?: string[];
     limits?: Record<string, number>;
+    /** `navigator.gpu.getPreferredCanvasFormat()`. */
+    preferredCanvasFormat?: string;
+    /** WGSL language features the implementation reports (`navigator.gpu.wgslLanguageFeatures`). */
+    wgslLanguageFeatures?: string[];
+}
+/** WebGL identity + capacity signals (a second, older GPU identity channel). */
+interface WebGLInfo {
+    /** Best context available: 'webgl2', 'webgl', or null when neither creates. */
+    contextKind: 'webgl2' | 'webgl' | null;
+    /** UNMASKED_VENDOR_WEBGL when the debug extension exists, else VENDOR. */
+    vendor?: string;
+    /** UNMASKED_RENDERER_WEBGL when the debug extension exists, else RENDERER. */
+    renderer?: string;
+    version?: string;
+    shadingLanguageVersion?: string;
+    maxTextureSize?: number;
+    maxRenderbufferSize?: number;
+    maxVertexUniformVectors?: number;
+    maxFragmentUniformVectors?: number;
+    /** Number of extensions exposed (identity signal without shipping the whole list). */
+    extensionCount?: number;
+    /** Whether the renderer string names a software rasterizer (SwiftShader, llvmpipe, ...). */
+    softwareRenderer?: boolean;
+}
+/**
+ * WebAssembly proposal support, each probed by validating a canonical module
+ * (the same byte sequences `wasm-feature-detect` uses). Missing keys mean the
+ * probe itself failed, not that the feature is absent.
+ */
+interface WasmFeatureSupport {
+    simd: boolean;
+    relaxedSimd: boolean;
+    threads: boolean;
+    bulkMemory: boolean;
+    exceptions: boolean;
+    /** Exception handling with `exnref` (the newer, standardized form). */
+    exceptionsFinal: boolean;
+    extendedConst: boolean;
+    gc: boolean;
+    memory64: boolean;
+    multiMemory: boolean;
+    multiValue: boolean;
+    mutableGlobals: boolean;
+    referenceTypes: boolean;
+    saturatedFloatToInt: boolean;
+    signExtensions: boolean;
+    tailCall: boolean;
+    typedFunctionReferences: boolean;
+    /** 128-bit wide arithmetic (`i64.add128` family). */
+    wideArithmetic: boolean;
+    /** JavaScript Promise Integration (`WebAssembly.Suspending`). */
+    jspi: boolean;
+    /** Type reflection (`WebAssembly.Function`). */
+    typeReflection: boolean;
+    /** `WebAssembly.compileStreaming` exists. */
+    streamingCompilation: boolean;
+    /** JS String Builtins (`js-string` import module), probed via the imports/builtins option. */
+    jsStringBuiltins: boolean;
+    /** Type-level `WebAssembly.Memory` growth limit reachable in this engine (pages of 64 KiB). */
+    maxMemoryPages?: number;
+}
+/**
+ * Availability of the browser APIs the runtimes and the paper care about.
+ * Each entry is a plain presence check (the feature exists on this page),
+ * not a functional test. Chrome Built-in AI is reported by its
+ * `availability()` string where the API exists.
+ */
+interface APIAvailability {
+    webgpu: boolean;
+    webgl2: boolean;
+    webnn: boolean;
+    /** OPFS: `navigator.storage.getDirectory` exists AND resolved. */
+    opfs: boolean;
+    /** Result of `navigator.storage.persisted()` where supported. */
+    persistedStorage?: boolean;
+    indexedDB: boolean;
+    cacheApi: boolean;
+    serviceWorker: boolean;
+    webWorkers: boolean;
+    offscreenCanvas: boolean;
+    webLocks: boolean;
+    broadcastChannel: boolean;
+    wakeLock: boolean;
+    computePressure: boolean;
+    performanceMemory: boolean;
+    measureUserAgentSpecificMemory: boolean;
+    schedulerYield: boolean;
+    webCodecs: boolean;
+    audioWorklet: boolean;
+    mediaDevices: boolean;
+    webTransport: boolean;
+    /** Chrome Built-in AI (Gemini Nano) surfaces, by `availability()` verdict when reachable. */
+    promptApi?: string;
+    summarizerApi?: string;
+    translatorApi?: string;
+    languageDetectorApi?: string;
+}
+/** Network Information API (Chromium + Android) snapshot at capture time. */
+interface NetworkInfo {
+    supported: boolean;
+    effectiveType?: string;
+    /** Connection type (wifi / cellular / ethernet / ...) where the UA exposes it. */
+    type?: string;
+    downlinkMbps?: number;
+    rttMs?: number;
+    saveData?: boolean;
+    online?: boolean;
+}
+/** Display / viewport snapshot. */
+interface DisplayInfo {
+    width: number;
+    height: number;
+    availWidth?: number;
+    availHeight?: number;
+    dpr: number;
+    colorDepth?: number;
+    orientation?: string;
+    viewportWidth?: number;
+    viewportHeight?: number;
+    /** `(dynamic-range: high)` media query. */
+    hdr?: boolean;
+    /** `(color-gamut: p3)` media query. */
+    wideGamut?: boolean;
+    /** `screen.isExtended` (Window Management API) — more than one display attached. */
+    isExtended?: boolean;
+    /** Reduced-motion / forced-colors preferences, cheap OS-level signals. */
+    prefersReducedMotion?: boolean;
+    prefersColorScheme?: 'light' | 'dark' | 'no-preference';
+}
+/** Locale / clock signals. */
+interface LocaleInfo {
+    timeZone?: string;
+    /** Minutes offset from UTC at capture time (`Date#getTimezoneOffset`). */
+    timeZoneOffsetMinutes?: number;
+    locale?: string;
+    /** `Intl.DateTimeFormat().resolvedOptions().calendar`. */
+    calendar?: string;
 }
-/** Full environment capture for a run. Clamped fields are labeled as such. */
+/**
+ * Full environment capture for a run. Every field beyond the first block is
+ * additive and best-effort: a probe that fails records nothing for its key and
+ * never affects the others. Clamped or capped fields are labeled as such.
+ */
 interface EnvironmentCapture {
     capturedAt: string;
     browser: BrowserInfo;
@@ -231,23 +406,46 @@ interface EnvironmentCapture {
         /** navigator.deviceMemory (GB) — Chromium-only, capped at 8. */
         deviceMemoryGB: number | null;
         deviceMemoryCapped: boolean;
+        /** `performance.memory.jsHeapSizeLimit` (Chromium) — the V8 heap ceiling for this tab. */
+        jsHeapSizeLimitBytes?: number;
+        /** `performance.memory.usedJSHeapSize` at capture (the idle baseline). */
+        jsHeapUsedBytes?: number;
     };
     gpu: GPUInfo;
     /** WebGL renderer string, a secondary GPU identity signal. */
     webglRenderer: string | null;
+    /** Detailed WebGL identity + capacity (superset of `webglRenderer`). */
+    webgl?: WebGLInfo;
+    /**
+     * GPU model parsed from the WebGL renderer string (ANGLE unwrapped),
+     * e.g. "Apple M4", "NVIDIA GeForce RTX 4070", "Mali-G78 MP20". Absent when
+     * no WebGL context could be created.
+     */
+    gpuModel?: string;
     flags: {
         crossOriginIsolated: boolean;
         sharedArrayBuffer: boolean;
         wasmSimd: boolean;
+        /** `window.isSecureContext`. */
+        secureContext?: boolean;
+        /** Full WebAssembly proposal matrix (superset of `wasmSimd`). */
+        wasm?: WasmFeatureSupport;
     };
+    /** Presence checks for the APIs the runtimes depend on. */
+    apis?: APIAvailability;
+    device?: DeviceInfo;
     storage: {
         quotaBytes?: number;
         usageBytes?: number;
+        usageDetails?: Record<string, number>;
     } | null;
     power: {
         batterySupported: boolean;
         charging?: boolean;
         level?: number;
+        /** Seconds until full / empty (Infinity serialized as absent). */
+        chargingTimeSec?: number;
+        dischargingTimeSec?: number;
     };
     pressure: {
         supported: boolean;
@@ -260,7 +458,17 @@ interface EnvironmentCapture {
         height: number;
         dpr: number;
     } | null;
+    /** Detailed display snapshot (superset of `screen`). */
+    display?: DisplayInfo;
+    network?: NetworkInfo;
+    locale?: LocaleInfo;
     languages?: string[];
+    /** Raw `navigator.userAgent` — kept verbatim so future parsers can re-derive fields. */
+    userAgent?: string;
+    /** Page origin the run executed on (distinguishes production from local/staging). */
+    pageOrigin?: string;
+    /** `document.visibilityState` at capture; a hidden tab is throttled. */
+    visibilityState?: string;
     /** Free-text device self-report — displayed as "user-reported", never trusted. */
     userReportedDevice?: string;
 }
@@ -319,17 +527,28 @@ interface CellSummary {
     qualityParseRate?: number;
     highVariance: boolean;
 }
+/** Identity of the software that produced a run. */
+interface HarnessInfo {
+    name: string;
+    version: string;
+    appVersion?: string;
+    /**
+     * Versions of the runtime packages bundled into the harness at build time
+     * (e.g. `{ "@huggingface/transformers": "4.2.0", "@wllama/wllama": "3.5.1" }`),
+     * keyed by npm package name. Per-cell `runtimeVersion` names the same value
+     * for the runtime that produced that cell.
+     */
+    runtimeVersions?: Record<string, string>;
+    /** Git commit of the harness build where the host exposes it. */
+    commit?: string;
+}
 /** The unit of submission: one full suite run on one device. */
 interface BenchRunResult {
     protocol: typeof BENCH_PROTOCOL_VERSION;
     schemaVersion: typeof BENCH_SCHEMA_VERSION;
     runId: string;
     createdAt: string;
-    harness: {
-        name: string;
-        version: string;
-        appVersion?: string;
-    };
+    harness: HarnessInfo;
     suite: BenchSuiteId;
     environment: EnvironmentCapture;
     fingerprint: FingerprintResult | null;
@@ -564,11 +783,7 @@ interface RunSuiteOptions {
     policy: RunPolicy;
     llmAdapters: ReadonlyMap<string, LLMRuntimeAdapter>;
     embedAdapters: ReadonlyMap<string, EmbeddingRuntimeAdapter>;
-    harness: {
-        name: string;
-        version: string;
-        appVersion?: string;
-    };
+    harness: HarnessInfo;
     hooks?: RunnerHooks;
     abortSignal?: AbortSignal;
     /** Skip the fingerprint microbenchmark (tests only; submissions require it). */
@@ -595,6 +810,11 @@ declare function runBenchmarkSuite(options: RunSuiteOptions): Promise<BenchRunRe
  * provenance: UA Client Hints on Chromium; UA parsing elsewhere with the OS
  * version marked 'unknown-frozen' (UA strings are frozen by design on Gecko
  * and WebKit). Clamped fields (cores, deviceMemory) are labeled clamped.
+ *
+ * Everything a browser will disclose is recorded, whether or not the current
+ * analysis uses it: WebGPU + WebGL identity, the WebAssembly proposal matrix,
+ * API availability, form factor, display, network, locale, power. Every probe
+ * is individually guarded; a failing probe records nothing for its key.
  */
 
 /**
@@ -609,6 +829,64 @@ declare function runBenchmarkSuite(options: RunSuiteOptions): Promise<BenchRunRe
 declare function captureEnvironment(options?: {
     userReportedDevice?: string;
 }): Promise<EnvironmentCapture>;
+/**
+ * UA-string fallback for browsers without UA Client Hints (every WebKit
+ * browser). Conservative: browser name + version token, platform, and the OS
+ * version only where the UA genuinely carries one.
+ *
+ * @example
+ * parseUserAgent('... CriOS/145.0.7632.72 Mobile/15E148 Safari/604.1').browser.name; // 'Chrome iOS'
+ */
+declare function parseUserAgent(ua: string): {
+    browser: BrowserInfo;
+    os: OSInfo;
+};
+/**
+ * Rendering engine from the UA. Every iOS browser is WebKit regardless of its
+ * brand (App Store policy), so the iOS check comes first.
+ *
+ * @example
+ * detectEngine('... CriOS/145.0 ...'); // 'WebKit'
+ */
+declare function detectEngine(ua: string): BrowserInfo['engine'];
+/** Inputs to the form-factor derivation, each optional. */
+interface DeviceTypeSignals {
+    ua: string;
+    maxTouchPoints: number;
+    /** UA-CH `formFactors` high-entropy hint (Chromium 125+). */
+    formFactors?: string[];
+    /** UA-CH `mobile` bit. */
+    mobile?: boolean;
+}
+/**
+ * Derive a form factor. UA-CH form factors win when present; otherwise the UA
+ * decides, with `maxTouchPoints` unmasking an iPad that reports itself as a
+ * Mac (iPadOS 13+ default) and separating Android tablets (no `Mobile` token)
+ * from phones.
+ *
+ * @example
+ * deriveDeviceType({ ua: navigator.userAgent, maxTouchPoints: navigator.maxTouchPoints });
+ */
+declare function deriveDeviceType(signals: DeviceTypeSignals): DeviceType;
+/**
+ * GPU model from a WebGL renderer string. ANGLE wraps the model as
+ * `ANGLE (<vendor>, <model>[ (0x...)] <backend...>, <api>)`; Metal adds a
+ * `ANGLE Metal Renderer: ` prefix. Native strings are returned as-is.
+ *
+ * @example
+ * parseGpuModel('ANGLE (Apple, ANGLE Metal Renderer: Apple M4, Unspecified Version)'); // 'Apple M4'
+ */
+declare function parseGpuModel(renderer: string | null | undefined): string | undefined;
+/**
+ * WebAssembly proposal support, probed the way `wasm-feature-detect` does:
+ * validate or compile the smallest module that uses each feature. The byte
+ * sequences are those of wasm-feature-detect 1.9.0, inlined so the harness
+ * keeps its zero-dependency contract.
+ *
+ * @example
+ * const wasm = await detectWasmFeatures(); // { simd: true, threads: true, ... }
+ */
+declare function detectWasmFeatures(): Promise<WasmFeatureSupport>;
 
 /**
  * Memory sampling at protocol points (baseline / post-load / post-run).
@@ -997,4 +1275,4 @@ interface STSPair {
 /** 100-pair STS-B test subset (order preserved from the source dataset). */
 declare const STSB_SUBSET: readonly STSPair[];
 
-export { type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseMMLUAnswer, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };
+export { type APIAvailability, type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, type DeviceInfo, type DeviceType, type DeviceTypeSignals, type DisplayInfo, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type HarnessInfo, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type LocaleInfo, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type NetworkInfo, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, type WasmFeatureSupport, type WebGLInfo, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deriveDeviceType, detectEngine, detectWasmFeatures, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseGpuModel, parseMMLUAnswer, parseUserAgent, quantile, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };