@localmode/bench 0.6.1 → 0.7.0

package/dist/index.d.cts

@@ -7,7 +7,9 @@
 /** Protocol identifier embedded in every result. Bump only with a spec change. */
 declare const BENCH_PROTOCOL_VERSION = "localmode-bench/4";
 /** Result JSON schema version (independent of the protocol semantics version). */
-declare const BENCH_SCHEMA_VERSION = 2;
+declare const BENCH_SCHEMA_VERSION = 3;
+/** Schema versions a submission may carry: the current one and the one a page built before the last schema change still sends. */
+declare const ACCEPTED_SCHEMA_VERSIONS: readonly number[];
 /** Benchmark suite presets. `custom` = user-picked cells. */
 type BenchSuiteId = 'quick' | 'standard' | 'thorough' | 'custom';
 /**
@@ -406,18 +408,14 @@ interface DisplayInfo {
     wideGamut?: boolean;
     /** `screen.isExtended` (Window Management API) — more than one display attached. */
     isExtended?: boolean;
-    /** Reduced-motion / forced-colors preferences, cheap OS-level signals. */
-    prefersReducedMotion?: boolean;
-    prefersColorScheme?: 'light' | 'dark' | 'no-preference';
 }
-/** Locale / clock signals. */
+/**
+ * Locale signal: the BCP 47 tag only (`en-US`). The time zone, UTC offset,
+ * and calendar a page can also read place a device in a city, which a public
+ * dataset has no use for, so they are not captured (schema 3).
+ */
 interface LocaleInfo {
-    timeZone?: string;
-    /** Minutes offset from UTC at capture time (`Date#getTimezoneOffset`). */
-    timeZoneOffsetMinutes?: number;
     locale?: string;
-    /** `Intl.DateTimeFormat().resolvedOptions().calendar`. */
-    calendar?: string;
 }
 /**
  * Full environment capture for a run. Every field beyond the first block is
@@ -471,10 +469,12 @@ interface EnvironmentCapture {
     power: {
         batterySupported: boolean;
         charging?: boolean;
+        /**
+         * Battery level rounded to a quarter (0, 0.25, 0.5, 0.75, 1). Low levels
+         * bring power saving, which matters; the exact percentage would track a
+         * device across runs and is not kept (schema 3).
+         */
         level?: number;
-        /** Seconds until full / empty (Infinity serialized as absent). */
-        chargingTimeSec?: number;
-        dischargingTimeSec?: number;
     };
     pressure: {
         supported: boolean;
@@ -491,7 +491,6 @@ interface EnvironmentCapture {
     display?: DisplayInfo;
     network?: NetworkInfo;
     locale?: LocaleInfo;
-    languages?: string[];
     /** Raw `navigator.userAgent` — kept verbatim so future parsers can re-derive fields. */
     userAgent?: string;
     /** Page origin the run executed on (distinguishes production from local/staging). */
@@ -585,10 +584,19 @@ interface BenchRunResult {
     events: TraceEvent[];
     /** Client-computed summaries (advisory; server recomputes from the trace). */
     clientSummaries?: CellSummary[];
-    /** Server-issued anti-forgery nonce (verified tier only). */
+    /**
+     * Server-issued anti-forgery nonce (verified tier only). Present on the
+     * submission, never in the published file: the server strips it.
+     */
     nonce?: string;
-    /** SHA-256 of the canonical JSON of this object without `digest`. */
+    /** SHA-256 of the canonical JSON of this object without `digest` and `nonce`. */
     digest?: string;
+    /**
+     * ISO time at which the published file was rewritten by the publication
+     * scrub (fields removed under a later schema, digest recomputed); absent
+     * when the file is exactly what the client submitted.
+     */
+    scrubbedAt?: string;
 }
 /** A plausibility/integrity finding attached by validation. */
 interface PlausibilityFlag {
@@ -1037,6 +1045,8 @@ declare function parseGpuModel(renderer: string | null | undefined): string | un
  * const wasm = await detectWasmFeatures(); // { simd: true, threads: true, ... }
  */
 declare function detectWasmFeatures(): Promise<WasmFeatureSupport>;
+/** Battery level rounded to a quarter: enough to see power saving, too coarse to track a device. */
+declare function coarseBatteryLevel(level: number | undefined): number | undefined;
 
 /**
  * Memory sampling at protocol points (baseline / post-load / post-run).
@@ -1229,15 +1239,56 @@ declare function canonicalJson(value: unknown): string;
 declare function sha256Hex(text: string): Promise<string>;
 /**
  * Compute the run digest: SHA-256 of the canonical JSON of the result with
- * `digest` removed.
+ * `digest` and `nonce` removed. The nonce is a submission credential the
+ * server strips before publishing, so it cannot be part of what the
+ * published file attests to.
  *
  * @example
  * result.digest = await computeRunDigest(result);
  */
 declare function computeRunDigest(result: BenchRunResult): Promise<string>;
-/** Verify a result's embedded digest. Returns false when absent or wrong. */
+/**
+ * The digest rule of files published before schema 3, which covered the
+ * nonce; kept so those files still verify.
+ */
+declare function computeLegacyRunDigest(result: BenchRunResult): Promise<string>;
+/**
+ * Verify a result's embedded digest under the current rule, else under the
+ * legacy rule when the result still carries a nonce. Returns false when
+ * absent or wrong.
+ */
 declare function verifyRunDigest(result: BenchRunResult): Promise<boolean>;
 
+/**
+ * The publication scrub: what a run file may carry once it is public. The
+ * environment capture no longer reads these signals (schema 3), so on a
+ * fresh submission the scrub changes nothing; it exists for submissions from
+ * pages built before the change and for rewriting files published earlier.
+ * The digest is the caller's job (`computeRunDigest`), because a scrub that
+ * changed anything invalidates the digest the client computed.
+ */
+
+/** What the scrub did to a run. */
+interface PublicationScrub {
+    run: BenchRunResult;
+    /**
+     * True when a digested field was removed or coarsened, so the client's
+     * digest no longer applies and must be recomputed. Stripping the nonce
+     * alone does not count: the digest never covers it.
+     */
+    changed: boolean;
+    /** Dotted paths of the fields removed or coarsened, for the audit trail. */
+    removed: string[];
+}
+/**
+ * Return a copy of `run` with the fields a public file must not carry
+ * removed: the submission nonce; the time zone, UTC offset, and calendar
+ * (they locate a device); the language list; the battery's exact level and
+ * time-to-full (they track a device); and the display preferences. A run
+ * that already lacks them is returned unchanged (`changed: false`).
+ */
+declare function scrubRunForPublication(run: BenchRunResult): PublicationScrub;
+
 /**
  * Aggregation for the public leaderboard and for offline analysis. Runs group
  * into device-class rows; medians are taken per submission first, then across
@@ -1394,4 +1445,4 @@ interface STSPair {
 /** 100-pair STS-B test subset (order preserved from the source dataset). */
 declare const STSB_SUBSET: readonly STSPair[];
 
-export { type APIAvailability, type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, type DeviceInfo, type DeviceType, type DeviceTypeSignals, type DisplayInfo, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type HarnessInfo, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type LocaleInfo, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type NetworkInfo, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerActivity, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, type WasmFeatureSupport, type WebGLInfo, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deriveDeviceType, detectEngine, detectWasmFeatures, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseGpuModel, parseMMLUAnswer, parseUserAgent, quantile, resolveGpuModel, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };
+export { ACCEPTED_SCHEMA_VERSIONS, type APIAvailability, type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, type DeviceInfo, type DeviceType, type DeviceTypeSignals, type DisplayInfo, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type HarnessInfo, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type LocaleInfo, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type NetworkInfo, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, type PublicationScrub, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerActivity, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, type WasmFeatureSupport, type WebGLInfo, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, coarseBatteryLevel, computeLegacyRunDigest, computeRunDigest, deriveDeviceType, detectEngine, detectWasmFeatures, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseGpuModel, parseMMLUAnswer, parseUserAgent, quantile, resolveGpuModel, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, scrubRunForPublication, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };

package/dist/index.d.ts

@@ -7,7 +7,9 @@
 /** Protocol identifier embedded in every result. Bump only with a spec change. */
 declare const BENCH_PROTOCOL_VERSION = "localmode-bench/4";
 /** Result JSON schema version (independent of the protocol semantics version). */
-declare const BENCH_SCHEMA_VERSION = 2;
+declare const BENCH_SCHEMA_VERSION = 3;
+/** Schema versions a submission may carry: the current one and the one a page built before the last schema change still sends. */
+declare const ACCEPTED_SCHEMA_VERSIONS: readonly number[];
 /** Benchmark suite presets. `custom` = user-picked cells. */
 type BenchSuiteId = 'quick' | 'standard' | 'thorough' | 'custom';
 /**
@@ -406,18 +408,14 @@ interface DisplayInfo {
     wideGamut?: boolean;
     /** `screen.isExtended` (Window Management API) — more than one display attached. */
     isExtended?: boolean;
-    /** Reduced-motion / forced-colors preferences, cheap OS-level signals. */
-    prefersReducedMotion?: boolean;
-    prefersColorScheme?: 'light' | 'dark' | 'no-preference';
 }
-/** Locale / clock signals. */
+/**
+ * Locale signal: the BCP 47 tag only (`en-US`). The time zone, UTC offset,
+ * and calendar a page can also read place a device in a city, which a public
+ * dataset has no use for, so they are not captured (schema 3).
+ */
 interface LocaleInfo {
-    timeZone?: string;
-    /** Minutes offset from UTC at capture time (`Date#getTimezoneOffset`). */
-    timeZoneOffsetMinutes?: number;
     locale?: string;
-    /** `Intl.DateTimeFormat().resolvedOptions().calendar`. */
-    calendar?: string;
 }
 /**
  * Full environment capture for a run. Every field beyond the first block is
@@ -471,10 +469,12 @@ interface EnvironmentCapture {
     power: {
         batterySupported: boolean;
         charging?: boolean;
+        /**
+         * Battery level rounded to a quarter (0, 0.25, 0.5, 0.75, 1). Low levels
+         * bring power saving, which matters; the exact percentage would track a
+         * device across runs and is not kept (schema 3).
+         */
         level?: number;
-        /** Seconds until full / empty (Infinity serialized as absent). */
-        chargingTimeSec?: number;
-        dischargingTimeSec?: number;
     };
     pressure: {
         supported: boolean;
@@ -491,7 +491,6 @@ interface EnvironmentCapture {
     display?: DisplayInfo;
     network?: NetworkInfo;
     locale?: LocaleInfo;
-    languages?: string[];
     /** Raw `navigator.userAgent` — kept verbatim so future parsers can re-derive fields. */
     userAgent?: string;
     /** Page origin the run executed on (distinguishes production from local/staging). */
@@ -585,10 +584,19 @@ interface BenchRunResult {
     events: TraceEvent[];
     /** Client-computed summaries (advisory; server recomputes from the trace). */
     clientSummaries?: CellSummary[];
-    /** Server-issued anti-forgery nonce (verified tier only). */
+    /**
+     * Server-issued anti-forgery nonce (verified tier only). Present on the
+     * submission, never in the published file: the server strips it.
+     */
     nonce?: string;
-    /** SHA-256 of the canonical JSON of this object without `digest`. */
+    /** SHA-256 of the canonical JSON of this object without `digest` and `nonce`. */
     digest?: string;
+    /**
+     * ISO time at which the published file was rewritten by the publication
+     * scrub (fields removed under a later schema, digest recomputed); absent
+     * when the file is exactly what the client submitted.
+     */
+    scrubbedAt?: string;
 }
 /** A plausibility/integrity finding attached by validation. */
 interface PlausibilityFlag {
@@ -1037,6 +1045,8 @@ declare function parseGpuModel(renderer: string | null | undefined): string | un
  * const wasm = await detectWasmFeatures(); // { simd: true, threads: true, ... }
  */
 declare function detectWasmFeatures(): Promise<WasmFeatureSupport>;
+/** Battery level rounded to a quarter: enough to see power saving, too coarse to track a device. */
+declare function coarseBatteryLevel(level: number | undefined): number | undefined;
 
 /**
  * Memory sampling at protocol points (baseline / post-load / post-run).
@@ -1229,15 +1239,56 @@ declare function canonicalJson(value: unknown): string;
 declare function sha256Hex(text: string): Promise<string>;
 /**
  * Compute the run digest: SHA-256 of the canonical JSON of the result with
- * `digest` removed.
+ * `digest` and `nonce` removed. The nonce is a submission credential the
+ * server strips before publishing, so it cannot be part of what the
+ * published file attests to.
  *
  * @example
  * result.digest = await computeRunDigest(result);
  */
 declare function computeRunDigest(result: BenchRunResult): Promise<string>;
-/** Verify a result's embedded digest. Returns false when absent or wrong. */
+/**
+ * The digest rule of files published before schema 3, which covered the
+ * nonce; kept so those files still verify.
+ */
+declare function computeLegacyRunDigest(result: BenchRunResult): Promise<string>;
+/**
+ * Verify a result's embedded digest under the current rule, else under the
+ * legacy rule when the result still carries a nonce. Returns false when
+ * absent or wrong.
+ */
 declare function verifyRunDigest(result: BenchRunResult): Promise<boolean>;
 
+/**
+ * The publication scrub: what a run file may carry once it is public. The
+ * environment capture no longer reads these signals (schema 3), so on a
+ * fresh submission the scrub changes nothing; it exists for submissions from
+ * pages built before the change and for rewriting files published earlier.
+ * The digest is the caller's job (`computeRunDigest`), because a scrub that
+ * changed anything invalidates the digest the client computed.
+ */
+
+/** What the scrub did to a run. */
+interface PublicationScrub {
+    run: BenchRunResult;
+    /**
+     * True when a digested field was removed or coarsened, so the client's
+     * digest no longer applies and must be recomputed. Stripping the nonce
+     * alone does not count: the digest never covers it.
+     */
+    changed: boolean;
+    /** Dotted paths of the fields removed or coarsened, for the audit trail. */
+    removed: string[];
+}
+/**
+ * Return a copy of `run` with the fields a public file must not carry
+ * removed: the submission nonce; the time zone, UTC offset, and calendar
+ * (they locate a device); the language list; the battery's exact level and
+ * time-to-full (they track a device); and the display preferences. A run
+ * that already lacks them is returned unchanged (`changed: false`).
+ */
+declare function scrubRunForPublication(run: BenchRunResult): PublicationScrub;
+
 /**
  * Aggregation for the public leaderboard and for offline analysis. Runs group
  * into device-class rows; medians are taken per submission first, then across
@@ -1394,4 +1445,4 @@ interface STSPair {
 /** 100-pair STS-B test subset (order preserved from the source dataset). */
 declare const STSB_SUBSET: readonly STSPair[];
 
-export { type APIAvailability, type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, type DeviceInfo, type DeviceType, type DeviceTypeSignals, type DisplayInfo, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type HarnessInfo, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type LocaleInfo, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type NetworkInfo, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerActivity, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, type WasmFeatureSupport, type WebGLInfo, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, computeRunDigest, deriveDeviceType, detectEngine, detectWasmFeatures, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseGpuModel, parseMMLUAnswer, parseUserAgent, quantile, resolveGpuModel, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };
+export { ACCEPTED_SCHEMA_VERSIONS, type APIAvailability, type AdapterAvailability, type AdapterLoadProgress, BENCH_PROTOCOL_VERSION, BENCH_SCHEMA_VERSION, type BenchCellResult, type BenchCellStatus, type BenchChunk, type BenchEmbeddingModel, type BenchLanguageModel, type BenchModelRef, type BenchRunResult, type BenchRuntimeId, type BenchStreamChunk, type BenchSuiteId, type BenchWorkloadKind, type BenchWorkloadSpec, type BrowserInfo, type CellSummary, type DeviceInfo, type DeviceType, type DeviceTypeSignals, type DisplayInfo, EMBED_WORKLOADS, type EmbedIteration, type EmbedWorkloadSpec, type EmbeddingRuntimeAdapter, type EnvironmentCapture, type FingerprintResult, GENERATION_BUDGET, type GPUInfo, HEADLINE_MIN_SUBMISSIONS, type HarnessInfo, type LLMIteration, type LLMRuntimeAdapter, type LLMWorkloadSpec, LLM_WORKLOADS, type LeaderboardRow, type LoadRecord, type LoadedEmbedder, type LoadedLLM, type LocaleInfo, MIN_GENERATED_CHARS, type MMLUItem, MMLU_MAX_TOKENS, MMLU_OUTPUT_CAP, type MemorySample, type MetricSummary, type NetworkInfo, type OSInfo, PLAUSIBILITY_RULES_VERSION, type PlannedCell, type PlausibilityFlag, type ProviderUsage, type PublicationScrub, QUALITY_WORKLOADS, type QualityResult, type QualityWorkloadSpec, RUNTIME_EXECUTION_ORDER, RUN_POLICIES, type RunPolicy, type RunSuiteOptions, type RunnerActivity, type RunnerHooks, STREAM_COHERENCE_MIN_SPAN_RATIO, STSB_SUBSET, type STSPair, TINY_MMLU, type TraceEvent, TraceRecorder, USAGE_FIDELITY, type ValidationReport, WORKLOADS_BY_ID, type WasmFeatureSupport, type WebGLInfo, aggregateRuns, canonicalJson, captureEnvironment, checkPlausibility, coarseBatteryLevel, computeLegacyRunDigest, computeRunDigest, deriveDeviceType, detectEngine, detectWasmFeatures, deviceClassOf, formatMMLUPrompt, geomean, hrNow, inferTimerResolutionUs, isIncrementalStream, mean, median, memoryApiAvailable, orderCells, parseGpuModel, parseMMLUAnswer, parseUserAgent, quantile, resolveGpuModel, rowsToCSV, runBenchmarkSuite, runFingerprint, runMMLUFidelity, runSTSQuality, runsToLongCSV, sampleMemoryBytes, scrubRunForPublication, sha256Hex, sleep, spearman, stddev, summarize, summarizeCell, summarizeRun, validateRunShape, validateSubmission, verifyRunDigest };