@localmode/bench 0.8.0 → 0.8.2

package/dist/index.d.cts

@@ -1189,7 +1189,17 @@ declare const STREAM_COHERENCE_MIN_SPAN_RATIO = 0.2;
  * (empty array = shape ok). Deliberately hand-rolled: this package is
  * zero-dependency and the checks double as executable schema documentation.
  */
-declare function validateRunShape(value: unknown): string[];
+/** Shape-validation options. */
+interface ValidateOptions {
+    /**
+     * Accept any `localmode-bench/<n>` protocol identifier instead of only the
+     * current one. Submissions never set this; offline analysis of the archive
+     * does, because every archived version stays readable and each row carries
+     * its version.
+     */
+    anyProtocol?: boolean;
+}
+declare function validateRunShape(value: unknown, options?: ValidateOptions): string[];
 /**
  * True when an iteration's chunk trace is genuinely incremental: at least two
  * chunks, and the visible stream spans at least
@@ -1226,7 +1236,7 @@ declare function checkPlausibility(run: BenchRunResult): PlausibilityFlag[];
  * summary recompute, client-summary agreement (>1% relative disagreement on
  * medians is flagged), and plausibility rules.
  */
-declare function validateSubmission(run: BenchRunResult): ValidationReport;
+declare function validateSubmission(run: BenchRunResult, options?: ValidateOptions): ValidationReport;
 
 /**
  * Canonical JSON serialization + SHA-256 digest for run results. The digest

package/dist/index.d.ts

@@ -1189,7 +1189,17 @@ declare const STREAM_COHERENCE_MIN_SPAN_RATIO = 0.2;
  * (empty array = shape ok). Deliberately hand-rolled: this package is
  * zero-dependency and the checks double as executable schema documentation.
  */
-declare function validateRunShape(value: unknown): string[];
+/** Shape-validation options. */
+interface ValidateOptions {
+    /**
+     * Accept any `localmode-bench/<n>` protocol identifier instead of only the
+     * current one. Submissions never set this; offline analysis of the archive
+     * does, because every archived version stays readable and each row carries
+     * its version.
+     */
+    anyProtocol?: boolean;
+}
+declare function validateRunShape(value: unknown, options?: ValidateOptions): string[];
 /**
  * True when an iteration's chunk trace is genuinely incremental: at least two
  * chunks, and the visible stream spans at least
@@ -1226,7 +1236,7 @@ declare function checkPlausibility(run: BenchRunResult): PlausibilityFlag[];
  * summary recompute, client-summary agreement (>1% relative disagreement on
  * medians is flagged), and plausibility rules.
  */
-declare function validateSubmission(run: BenchRunResult): ValidationReport;
+declare function validateSubmission(run: BenchRunResult, options?: ValidateOptions): ValidationReport;
 
 /**
  * Canonical JSON serialization + SHA-256 digest for run results. The digest

package/dist/index.js

@@ -31,9 +31,9 @@ ${e.question}
 ${o}
 
 Answer with only the letter (A, B, C, or D) of the correct choice.${t??""}
-Answer:`}function xt(e){let t=e.replace(/<think>[\s\S]*?<\/think>/gi," "),o=t.search(/<think>/i);return o===-1?t:t.slice(0,o)}function J(e,t){let o=xt(e).trim(),n=o.replace(/[*`_]/g,"").trim(),r=n.match(/(?:^|(?:[Aa]nswer|ANSWER)\s*(?:is)?\s*[:-]?\s*|(?:[Oo]ption|OPTION|[Cc]hoice|CHOICE)\s*[:-]?\s*)\(?([ABCD])\)?(?=[\s.,)!]|$)/);if(r)return he.indexOf(r[1]);let a=n.match(/^\(?([abcd])(?:\)|\.|$)/);if(a)return he.indexOf(a[1].toUpperCase());let i=o.toLowerCase();for(let s=0;s<t.length;s++)if(i.startsWith(t[s].toLowerCase().slice(0,Math.max(8,t[s].length))))return s;return null}async function pe(e,t,o){let n=H.slice(0,t),r=[],a=[],i=0,s=0;for(let l=0;l<n.length;l++){if(o?.abortSignal?.aborted)throw T();let c=n[l],m=(await e.doGenerate({prompt:We(c,o?.promptSuffix),maxTokens:qe,temperature:0,abortSignal:o?.abortSignal})).text.slice(0,je),f=J(m,c.choices);f!==null&&s++;let u=f===c.answer?1:0;i+=u,r.push(u),a.push(m),o?.onProgress?.(l+1,n.length);}return {taskId:`tinymmlu-${t}`,score:n.length===0?0:i/n.length,n:n.length,details:r,outputs:a,parseRate:n.length===0?0:s/n.length}}function _t(e,t){let o=0,n=0,r=0;for(let i=0;i<e.length;i++)o+=e[i]*t[i],n+=e[i]*e[i],r+=t[i]*t[i];let a=Math.sqrt(n)*Math.sqrt(r);return a===0?0:o/a}async function fe(e,t,o){let n=me.slice(0,t),r=[],a=[];for(let i=0;i<n.length;i++){if(o?.abortSignal?.aborted)throw T();let s=n[i],{embeddings:l}=await e.doEmbed({values:[s.s1,s.s2],abortSignal:o?.abortSignal});r.push(_t(l[0],l[1])),a.push(s.score),o?.onProgress?.(i+1,n.length);}return {taskId:`stsb-${t}`,score:r.length>=2?ue(r,a):0,n:n.length,details:r.map(i=>Math.round(i*1e4)/1e4)}}var Et=2,Ne=.2,Pt=[{maxBytes:5e8,max:4e3},{maxBytes:1e9,max:2500},{maxBytes:2e9,max:1500},{maxBytes:Number.POSITIVE_INFINITY,max:1e3}],Ct=2e3,Lt=[/swiftshader/i,/llvmpipe/i,/softpipe/i,/\bwarp\b/i,/microsoft basic render/i,/parallels/i,/vmware/i,/virtio/i,/virtualbox/i];function Oe(e){let t=[],o=i=>{t.length<50&&t.push(i);};if(typeof e!="object"||e===null)return ["result must be an object"];let n=e;n.protocol!==j&&o(`protocol must be "${j}"`),z.includes(n.schemaVersion)||o(`schemaVersion must be one of ${z.join(", ")}`),(typeof n.runId!="string"||n.runId.length<8||n.runId.length>64)&&o("runId must be a string (8-64 chars)"),(typeof n.createdAt!="string"||Number.isNaN(Date.parse(n.createdAt)))&&o("createdAt must be an ISO date string");let r=n.harness;(!r||typeof r.name!="string"||typeof r.version!="string")&&o("harness {name, version} is required"),["quick","standard","thorough","custom"].includes(n.suite)||o("suite must be quick|standard|thorough|custom");let a=n.environment;if(!a||typeof a!="object")o("environment is required");else {let i=a.browser;(!i||typeof i.name!="string")&&o("environment.browser.name is required"),(!a.os||typeof a.os.platform!="string")&&o("environment.os.platform is required");}return Array.isArray(n.cells)?n.cells.length===0?o("cells must not be empty"):n.cells.length>500?o("cells exceeds the 500-cell limit"):n.cells.forEach((i,s)=>Bt(i,s,o)):o("cells must be an array"),Array.isArray(n.events)||o("events must be an array"),t}function Bt(e,t,o){if(typeof e!="object"||e===null){o(`cells[${t}] must be an object`);return}let n=e,r=`cells[${t}]`;typeof n.cellId!="string"&&o(`${r}.cellId must be a string`),typeof n.runtimeId!="string"&&o(`${r}.runtimeId must be a string`),typeof n.workloadId!="string"&&o(`${r}.workloadId must be a string`),["ok","invalid","error","skipped"].includes(n.status)||o(`${r}.status must be ok|invalid|error|skipped`);let a=n.model;(!a||typeof a.benchModelId!="string"||typeof a.providerModelId!="string")&&o(`${r}.model {benchModelId, providerModelId} is required`);let i=n.quality;if(i&&Array.isArray(i.outputs)&&i.outputs.some(s=>typeof s!="string"||s.length>500)&&o(`${r}.quality.outputs must be strings of at most 500 chars`),!Array.isArray(n.iterations)){o(`${r}.iterations must be an array`);return}n.iterations.forEach((s,l)=>{if(typeof s!="object"||s===null){o(`${r}.iterations[${l}] must be an object`);return}let c=s;(typeof c.startT!="number"||typeof c.endT!="number")&&o(`${r}.iterations[${l}] needs numeric startT/endT`),"chunks"in c&&!Array.isArray(c.chunks)&&o(`${r}.iterations[${l}].chunks must be an array`),"text"in c&&typeof c.text=="string"&&c.text.length>2e4&&o(`${r}.iterations[${l}].text exceeds 20000 chars`);});}function De(e){return e.length>0&&"chunks"in e[0]}function ee(e){let t=e.chunks.filter(r=>r.c>0);if(t.length<2)return  false;let o=t[t.length-1].t-t[0].t,n=e.endT-e.startT;return n>0&&o/n>=Ne}function Z(e,t=.05,o){let n={cellId:e.cellId,status:e.status,highVariance:false};if(e.load&&(n.loadMs=e.load.endT-e.load.startT,n.loadCached=e.load.cached),e.quality&&(n.qualityScore=e.quality.score,e.quality.parseRate!==void 0&&(n.qualityParseRate=e.quality.parseRate)),e.iterations.length===0)return n;if(De(e.iterations)){let a=e.iterations.every(ee);n.streamIncremental=a;let i=[],s=[],l=[],c=[],d=[],m=[],f=[];for(let u of e.iterations){let y=u.chunks.reduce((g,k)=>g+k.c,0),p=u.endT-u.startT;if(p>0&&(m.push(p),y>0&&f.push(y/p*1e3)),c.push(y),!a)continue;let h=u.chunks.find(g=>g.c>0);if(!h)continue;let b=h.t-u.startT;i.push(b);let S=u.chunks[u.chunks.length-1],M=y-h.c,v=S.t-h.t;v>0&&M>0&&(s.push(M/v*1e3),l.push((u.chunks.length-1)/v*1e3)),o&&b>0&&d.push(o/b*1e3);}i.length>0&&(n.ttftMs=_(i)),s.length>0&&(n.decodeCharsPerSec=_(s)),l.length>0&&(n.decodeChunksPerSec=_(l)),c.length>0&&(n.generatedChars=_(c)),d.length>0&&(n.prefillTokPerSecApprox=_(d)),m.length>0&&(n.totalMs=_(m)),f.length>0&&(n.overallCharsPerSec=_(f));}else {let a=e.iterations,i=a.map(s=>s.endT-s.startT).filter(s=>s>0);i.length>0&&(a[0].count===1?n.singleLatencyMs=_(i):n.batchTextsPerSec=_(a.filter(s=>s.endT-s.startT>0).map(s=>s.count/(s.endT-s.startT)*1e3)));}let r=[n.ttftMs,n.decodeCharsPerSec,n.overallCharsPerSec,n.singleLatencyMs,n.batchTextsPerSec];return n.highVariance=r.some(a=>a!==void 0&&a.n>=2&&a.cv>t),n}function N(e,t=.05,o){let r=o??new Map([["chat-pp128-tg128",128],["chat-pp512-tg128",512]]);return e.cells.map(a=>Z(a,t,r.get(a.workloadId)))}function Ue(e){let t=[],o=(s,l,c,d)=>t.push({code:s,message:l,severity:c,...d?{cellId:d}:{}});e.fingerprint?(e.fingerprint.mflops<=1||e.fingerprint.mflops>1e5||!Number.isFinite(e.fingerprint.checksum))&&o("fingerprint-implausible",`fingerprint mflops=${e.fingerprint.mflops}`,"reject"):o("fingerprint-missing","no hardware fingerprint in submission","reject");let n=e.environment.timerResolutionUs;for(let s of e.cells)if(s.status==="ok"){if(s.load&&s.load.endT<s.load.startT&&o("load-time-reversed","load endT precedes startT","reject",s.cellId),De(s.iterations)){for(let m of s.iterations){let f=m.startT,u=true;for(let p of m.chunks)p.t<f&&(u=false),f=p.t;if(!u||m.endT<f){o("trace-not-monotonic","chunk timestamps are not monotonic","reject",s.cellId);break}let y=m.chunks.reduce((p,h)=>p+h.c,0);if(m.text.length!==y){o("text-chunk-mismatch",`text length ${m.text.length} != chunk sum ${y}`,"reject",s.cellId);break}}let l=Z(s);s.iterations.some(m=>m.text.length<L)&&o("degenerate-generation",`an ok cell has a timed iteration with fewer than ${L} generated chars`,"reject",s.cellId);let c=s.model.sizeBytes??Number.POSITIVE_INFINITY,d=Pt.find(m=>c<=m.maxBytes);if(d&&l.decodeCharsPerSec&&l.decodeCharsPerSec.median>d.max&&o("decode-rate-envelope",`decode ${Math.round(l.decodeCharsPerSec.median)} chars/s exceeds envelope ${d.max} for model size`,"reject",s.cellId),d&&l.overallCharsPerSec&&l.overallCharsPerSec.median>d.max&&o("overall-rate-envelope",`overall ${Math.round(l.overallCharsPerSec.median)} chars/s exceeds envelope ${d.max} for model size`,"reject",s.cellId),l.ttftMs&&l.ttftMs.median<1&&o("ttft-implausible",`median TTFT ${l.ttftMs.median}ms < 1ms`,"reject",s.cellId),n!=null&&n>=4){let m=n/1e3,f=0,u=0;for(let y of s.iterations)for(let p of y.chunks)u++,Math.abs(p.t/m-Math.round(p.t/m))>.01&&f++;u>=20&&f/u>.5&&o("timer-grid",`${f}/${u} chunk timestamps off the ${n}us timer grid`,"warn",s.cellId);}}else if(s.quality?.taskId.startsWith("tinymmlu")&&s.quality.outputs){let{outputs:l,details:c,score:d,n:m}=s.quality;if(l.length!==m||!c||c.length!==m)o("quality-outputs-mismatch","quality outputs/details length disagrees with n","reject",s.cellId);else {let f=0,u=true;for(let y=0;y<m;y++){let p=H[y];if(!p)break;let h=J(l[y],p.choices)===p.answer?1:0;f+=h,h!==c[y]&&(u=false);}(!u||Math.abs(f/m-d)>1e-9)&&o("quality-details-mismatch",`submitted MMLU score ${d} disagrees with recompute ${f/m} from raw outputs`,"reject",s.cellId);}}else if(s.iterations.length>0){let l=Z(s);l.batchTextsPerSec&&l.batchTextsPerSec.median>Ct&&o("embed-rate-envelope",`embedding throughput ${Math.round(l.batchTextsPerSec.median)} texts/s exceeds envelope`,"reject",s.cellId),l.singleLatencyMs&&l.singleLatencyMs.median<.05&&o("embed-latency-implausible","sub-50us embedding latency","reject",s.cellId);}}let r=e.environment.gpu.vendor?.toLowerCase()??"",a=e.environment.os.platform.toLowerCase();r==="apple"&&a.includes("windows")&&o("env-cross-field","Apple GPU adapter with Windows platform","reject");let i=[e.environment.gpu.vendor,e.environment.gpu.architecture,e.environment.gpu.device,e.environment.gpu.description,e.environment.webglRenderer].filter(Boolean).join(" ");if(Lt.some(s=>s.test(i))){let s=e.cells.some(l=>l.status==="ok"&&/webgpu|^gpu$/.test(l.resolvedBackend));o("software-renderer",`GPU identity matches a software/virtual renderer: "${i.slice(0,120)}"`,s?"reject":"warn");}return t}function qt(e){let t=Oe(e);if(t.length>0)return {ok:false,shapeErrors:t,flags:[],summaries:[]};let o=N(e),n=Ue(e);if(e.clientSummaries){let r=new Map(o.map(a=>[a.cellId,a]));for(let a of e.clientSummaries){let i=r.get(a.cellId);if(i)for(let s of ["ttftMs","decodeCharsPerSec","totalMs","overallCharsPerSec","singleLatencyMs","batchTextsPerSec"]){let l=a[s]?.median,c=i[s]?.median;l!==void 0&&c!==void 0&&c!==0&&Math.abs(l-c)/Math.abs(c)>.01&&n.push({code:"client-summary-disagrees",cellId:a.cellId,message:`${s} client median ${l} vs recomputed ${c}`,severity:"reject"});}}}return {ok:n.every(r=>r.severity!=="reject"),shapeErrors:[],flags:n,summaries:o}}async function jt(e){let{policy:t,hooks:o,abortSignal:n}=e,r=e.trace??new $;await r.attach(),r.record("suite-start",e.suite);let a=K(),i=[];try{o?.onPhase?.("environment");let s=await ce({userReportedDevice:e.userReportedDevice});o?.onEnvironment?.(s),o?.onPhase?.("fingerprint"),P(n);let l=e.skipFingerprint?null:await de(),c=a!=="none"?await W():null,d=Wt(ie(e.cells)),m=e.cells.length,f=0;for(let y of d){P(n);let p=await Nt(y,{...e,trace:r,memApi:a,baselineMemory:c,cellIndexRef:()=>f++,totalCells:m});i.push(...p),p.some(h=>h.status!=="skipped")&&(r.record("cooldown-start"),await F(t.cooldownMs,n).catch(h=>{if(n?.aborted)throw h}),t.pressureGate&&await Ut(r,t.pressureGateTimeoutMs,n),r.record("cooldown-end"));}r.record("suite-end");let u={protocol:j,schemaVersion:3,runId:Vt(),createdAt:new Date().toISOString(),harness:e.harness,suite:e.suite,environment:s,fingerprint:l,cells:i,events:[...r.all]};return u.clientSummaries=N(u,t.highVarianceCv),u}finally{await r.dispose();}}function Wt(e){let t=new Map;for(let o of e){let n=`${o.model.runtimeId}/${o.model.benchModelId}/${o.model.providerModelId}`,r=t.get(n);r||(r={key:n,model:o.model,workloads:[],skipped:[]},t.set(n,r)),o.skipReason?r.skipped.push({workload:o.workload,reason:o.skipReason}):r.workloads.push(o.workload);}return [...t.values()]}async function Nt(e,t){let{model:o}=e,n=o.task==="embedding",r=n?t.embedAdapters.get(o.runtimeId):t.llmAdapters.get(o.runtimeId),a=[],i=p=>({cellId:`${o.runtimeId}/${o.benchModelId}/${p.id}`,runtimeId:o.runtimeId,runtimeVersion:r?.runtimeVersion,model:o,workloadId:p.id,workloadKind:p.kind,resolvedBackend:"unknown",load:null,iterations:[],status:"skipped"});for(let{workload:p,reason:h}of e.skipped){let b=i(p);b.invalidReasons=[h],a.push(O(b,t));}if(e.workloads.length===0)return a;if(!r){for(let p of e.workloads){let h=i(p);h.invalidReasons=[`no adapter registered for ${o.runtimeId}`],a.push(O(h,t));}return a}let s=await r.isAvailable();if(!s.ok){for(let p of e.workloads){let h=i(p);h.invalidReasons=[`runtime unavailable: ${s.reason??"unknown"}`],a.push(O(h,t));}return a}let l=null,c=null,d,m=null,f=[],u=`${o.runtimeId}/${o.benchModelId}/${e.workloads[0].id}`;for(let p=1;p<=Math.max(1,t.policy.maxAttempts);p++)try{P(t.abortSignal);let h=await r.isModelCached(o),b=[],S=w();t.hooks?.onActivity?.({cellId:u,phase:"load"}),l=await B((v,g)=>$e(r.load(o,{abortSignal:v,onProgress:k=>{g(),typeof k.pct=="number"&&(b.length===0||k.pct-b[b.length-1].pct>=2)&&b.push({t:w(),pct:Math.round(k.pct*100)/100}),t.hooks?.onLoadProgress?.(u,k.pct),t.hooks?.onActivity?.({cellId:u,phase:"load",pct:k.pct});}}),v),{parent:t.abortSignal,stallMs:t.policy.loadStallMs,timeoutMs:t.policy.loadTimeoutMs,what:"load"});let M=w();if(c={cached:h,startT:S,endT:M,progress:b.length>0?Ft(b,50):void 0,declaredBytes:o.sizeBytes},t.memApi!=="none"&&(m=await W()),t.policy.warmupRuns>0){let v=w();for(let g=0;g<t.policy.warmupRuns;g++){P(t.abortSignal);let k=l;t.hooks?.onActivity?.({cellId:u,phase:"warmup",iteration:g+1,total:t.policy.warmupRuns,chars:0,chunks:0}),await B((A,E)=>n?k.model.doEmbed({values:["warmup probe"],abortSignal:A}):Ge(k.model,{prompt:"Reply with the single word: ready",maxTokens:4,temperature:0},A,(Ze,et)=>{E(),t.hooks?.onActivity?.({cellId:u,phase:"warmup",chars:Ze,chunks:et});}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.iterationTimeoutMs,what:"warmup"}),t.hooks?.onActivity?.({cellId:u,phase:"warmup",iteration:g+1,total:t.policy.warmupRuns});}d=w()-v;}break}catch(h){if(t.abortSignal?.aborted)throw l&&await D(l),T();l&&await D(l),l=null;let b=Fe(h);if(b.name==="TimeoutError"&&t.trace.record("cell-timeout",`${u}: ${b.message}`),p<Math.max(1,t.policy.maxAttempts)){f.push({error:b,at:w()}),t.trace.record("cell-retry",`${u} attempt ${p+1}`),t.hooks?.onCellRetry?.(u,p+1,b);continue}let S=t.memApi!=="none"?await W(5e3):null;for(let M of e.workloads){let v=i(M);v.status="error",v.load=c,v.error=b,f.length>0&&(v.attempts=[...f]),t.memApi!=="none"&&(v.memory={api:t.memApi,baseline:t.baselineMemory??void 0,postLoad:m??void 0,atError:S??void 0}),a.push(O(v,t));}return a}if(!l)return a;let y=false;for(let p of e.workloads){P(t.abortSignal);let h=i(p);h.resolvedBackend=l.resolvedBackend,l.runtimeConfig&&(h.runtimeConfig={...l.runtimeConfig}),y||(h.load=c,h.warmupMs=d,f.length>0&&(h.attempts=[...f]),y=true);let b=t.cellIndexRef();t.hooks?.onCellStart?.(h.cellId,b,t.totalCells);let S=l,M=async()=>{switch(p.kind){case "llm-generate":await Ot(h,S,p,t);break;case "embed-single":case "embed-batch":await Dt(h,S,p,t);break;case "quality-mmlu":{let g=p.items;h.quality=await B((k,A)=>pe(S.model,g,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onIteration?.(h.cellId,E,g),t.hooks?.onActivity?.({cellId:h.cellId,phase:"quality",iteration:E,total:g});},promptSuffix:o.qualityPromptSuffix}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.qualityTimeoutMs,what:"quality lane"}),h.status="ok";break}case "quality-sts":{let g=p.items;h.quality=await B((k,A)=>fe(S.model,g,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onIteration?.(h.cellId,E,g),t.hooks?.onActivity?.({cellId:h.cellId,phase:"quality",iteration:E,total:g});}}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.qualityTimeoutMs,what:"quality lane"}),h.status="ok";break}}},v=Math.max(1,t.policy.maxAttempts);for(let g=1;g<=v;g++)try{await M();break}catch(k){if(t.abortSignal?.aborted)throw await D(l),T();let A=Fe(k);if(A.name==="TimeoutError"&&t.trace.record("cell-timeout",`${h.cellId}: ${A.message}`),g<v){h.attempts=[...h.attempts??[],{error:A,at:w()}],h.iterations=[],h.quality=void 0,h.invalidReasons=void 0,h.status="skipped",t.trace.record("cell-retry",`${h.cellId} attempt ${g+1}`),t.hooks?.onCellRetry?.(h.cellId,g+1,A);continue}h.status="error",h.error=A;}if(t.memApi!=="none"&&(h.status==="ok"||h.status==="error")){let g=await W(5e3);h.memory={api:t.memApi,baseline:t.baselineMemory??void 0,postLoad:m??void 0,...h.status==="ok"?{postRun:g??void 0}:{atError:g??void 0}};}a.push(O(h,t));}if(await D(l),t.policy.measureWarmReload&&c?.cached===false)try{P(t.abortSignal);let p=`${o.runtimeId}/${o.benchModelId}/warm-reload`;t.hooks?.onActivity?.({cellId:p,phase:"reload"});let h=w(),b=await B((k,A)=>$e(r.load(o,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onActivity?.({cellId:p,phase:"reload",pct:E.pct});}}),k),{parent:t.abortSignal,stallMs:t.policy.loadStallMs,timeoutMs:t.policy.loadTimeoutMs,what:"load"}),S=w(),M=b.resolvedBackend,v=b.runtimeConfig?{...b.runtimeConfig}:void 0;await D(b);let g={cellId:`${o.runtimeId}/${o.benchModelId}/warm-reload`,runtimeId:o.runtimeId,runtimeVersion:r.runtimeVersion,model:o,workloadId:"warm-reload",workloadKind:"llm-generate",resolvedBackend:M,...v?{runtimeConfig:v}:{},load:{cached:!0,startT:h,endT:S,declaredBytes:o.sizeBytes},iterations:[],status:"ok"};a.push(O(g,t));}catch{if(t.abortSignal?.aborted)throw T()}return a}async function Ot(e,t,o,n){let r=[],a=[];for(let i=0;i<n.policy.timedRuns;i++){let s=i+1;for(let l=0;;l++){P(n.abortSignal),n.hooks?.onIteration?.(e.cellId,s,n.policy.timedRuns),await ze(n,e.cellId);let c=[];n.trace.isHidden&&c.push("started-hidden");let d=ne(n.trace);n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:s,total:n.policy.timedRuns,chars:0,chunks:0});let m=w(),{chunks:f,text:u,providerUsage:y,finishReason:p}=await B((S,M)=>Ge(t.model,{prompt:o.prompt,systemPrompt:o.systemPrompt,maxTokens:o.maxTokens,temperature:o.temperature},S,(v,g)=>{M(),n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:s,total:n.policy.timedRuns,chars:v,chunks:g});}),{parent:n.abortSignal,stallMs:n.policy.chunkStallMs,timeoutMs:n.policy.iterationTimeoutMs,what:`iteration ${s}`}),h=w();ne(n.trace)>d&&c.push("hidden-during-run"),u.length<L&&c.push("degenerate-output");let b={startT:m,chunks:f,endT:h,text:u,providerUsage:y?{...y,fidelity:se[e.runtimeId]}:void 0,finishReason:p,gates:c};if(Ve(c)&&l<Math.max(1,n.policy.maxAttempts)){a.push(b),n.trace.record("iteration-redo",`${e.cellId} iteration ${s}: ${c.join(",")}`);continue}r.push(b);break}}e.iterations=r,a.length>0&&(e.discardedIterations=a),He(e,r);}function Ve(e){return e.includes("started-hidden")||e.includes("hidden-during-run")}function He(e,t){let o=t.some(n=>n.gates.length>0);if(e.status=o?"invalid":"ok",o){let n=new Set;for(let r of t)for(let a of r.gates)n.add(a==="degenerate-output"?`degenerate output: fewer than ${L} generated chars in a timed iteration`:a==="started-hidden"||a==="hidden-during-run"?"tab hidden during a timed iteration (the tab did not return in time for a repeat)":"validity gate fired during a timed region");e.invalidReasons=[...n];}}async function ze(e,t){if(!e.trace.isHidden)return  true;e.hooks?.onActivity?.({cellId:t,phase:"waiting-visible"});let o=w()+e.policy.visibilityWaitMs;for(;e.trace.isHidden&&w()<o;)P(e.abortSignal),await F(Math.min(250,e.policy.visibilityWaitMs),e.abortSignal);return !e.trace.isHidden}async function Dt(e,t,o,n){let r=[],a=[];for(let i=0;i<n.policy.timedRuns;i++)for(let s=0;;s++){P(n.abortSignal),n.hooks?.onIteration?.(e.cellId,i+1,n.policy.timedRuns),await ze(n,e.cellId);let l=[];n.trace.isHidden&&l.push("started-hidden");let c=ne(n.trace),d=w(),{embeddings:m}=await B(y=>t.model.doEmbed({values:o.texts,abortSignal:y}),{parent:n.abortSignal,stallMs:n.policy.iterationTimeoutMs,timeoutMs:n.policy.iterationTimeoutMs,what:`iteration ${i+1}`}),f=w();ne(n.trace)>c&&l.push("hidden-during-run"),n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:i+1,total:n.policy.timedRuns});let u={startT:d,endT:f,count:o.texts.length,dimensions:m[0]?.length??t.model.dimensions,gates:l};if(Ve(l)&&s<Math.max(1,n.policy.maxAttempts)){a.push(u),n.trace.record("iteration-redo",`${e.cellId} iteration ${i+1}: ${l.join(",")}`);continue}r.push(u);break}e.iterations=r,a.length>0&&(e.discardedIterations=a),He(e,r);}async function Ge(e,t,o,n){let r=[],a="",i,s;if(e.doStream)for await(let l of e.doStream({...t,abortSignal:o})){let c=w();l.text.length>0&&(r.push({t:c,c:l.text.length}),a+=l.text,n?.(a.length,r.length)),l.done&&(i=l.usage,s=l.finishReason);}else {let l=await e.doGenerate({...t,abortSignal:o}),c=w();a=l.text,r.push({t:c,c:l.text.length}),n?.(a.length,r.length),i=l.usage,s=l.finishReason;}return {chunks:r,text:a,providerUsage:i,finishReason:s}}var te=class extends Error{constructor(t){super(t),this.name="TimeoutError";}};async function B(e,t){let o=new AbortController,n=()=>{},r=new Promise((u,y)=>{n=y;}),a=()=>{o.abort(t.parent?.reason),n(T());};t.parent?.aborted&&a(),t.parent?.addEventListener("abort",a,{once:true});let i=w(),s=i,l=false,c=u=>{l||(l=true,o.abort(new te(u)),n(new te(u)));},m=setInterval(()=>{if(l)return;let u=w();if(u-s>=t.stallMs){c(t.what==="load"?`load made no progress for ${t.stallMs} ms`:`${t.what}: no stream progress for ${t.stallMs} ms`);return}u-i>=t.timeoutMs&&c(`${t.what}: exceeded the ${t.timeoutMs} ms budget`);},Math.max(10,Math.min(t.stallMs,t.timeoutMs)/4)),f=()=>{s=w();};try{let u=e(o.signal,f);return u.catch(()=>{}),await Promise.race([u,r])}finally{l=true,clearInterval(m),t.parent?.removeEventListener("abort",a);}}function ne(e){let t=0;for(let o of e.all)o.type==="visibility-hidden"&&t++;return t}async function Ut(e,t,o){let n=w()+t;for(;w()<n;){let r=e.pressureState;if(r===void 0||r==="nominal"||r==="fair")return;await F(1e3,o);}}function Ft(e,t){if(e.length<=t)return e;let o=[],n=(e.length-1)/(t-1);for(let r=0;r<t;r++)o.push(e[Math.round(r*n)]);return o}var $t=4e3;function Fe(e){let t=e,o=t?.cause,n=o instanceof Error?o.message:typeof o=="string"?o:void 0,r=o instanceof Error&&o.name&&o.name!=="Error"?o.name:void 0,a=o instanceof Error&&typeof o.stack=="string"&&o.stack.length>0?o.stack.slice(0,$t):void 0;return {name:t?.name??"Error",message:t?.message??String(e),...n!==void 0?{cause:n}:{},...r?{causeName:r}:{},...a?{causeStack:a}:{}}}function O(e,t){return t.hooks?.onCellFinish?.(e),e}async function $e(e,t){let o=await e;if(t.aborted)throw await D(o),T();return o}async function D(e){try{await e.dispose();}catch{}}function P(e){if(e?.aborted)throw T()}function Vt(){let e=globalThis.crypto;if(e?.randomUUID)return e.randomUUID();let t=new Uint8Array(16);if(e?.getRandomValues)e.getRandomValues(t);else for(let n=0;n<16;n++)t[n]=Math.floor(Math.random()*256);t[6]=t[6]&15|64,t[8]=t[8]&63|128;let o=[...t].map(n=>n.toString(16).padStart(2,"0")).join("");return `${o.slice(0,8)}-${o.slice(8,12)}-${o.slice(12,16)}-${o.slice(16,20)}-${o.slice(20)}`}function be(e){return ge(e)}function ge(e){if(e===null||typeof e=="number"||typeof e=="boolean"||typeof e=="string")return JSON.stringify(e);if(typeof e>"u")return "null";if(Array.isArray(e))return `[${e.map(t=>ge(t)).join(",")}]`;if(typeof e=="object")return `{${Object.entries(e).filter(([,o])=>o!==void 0).sort(([o],[n])=>o<n?-1:o>n?1:0).map(([o,n])=>`${JSON.stringify(o)}:${ge(n)}`).join(",")}}`;throw new TypeError(`canonicalJson: unsupported value type ${typeof e}`)}async function ye(e){let t=globalThis.crypto?.subtle;if(!t)throw new Error("Web Crypto (crypto.subtle) is not available in this environment");let o=new TextEncoder().encode(e),n=await t.digest("SHA-256",o);return [...new Uint8Array(n)].map(r=>r.toString(16).padStart(2,"0")).join("")}async function Ke(e){let{digest:t,nonce:o,...n}=e;return ye(be(n))}async function Ye(e){let{digest:t,...o}=e;return ye(be(o))}async function Ht(e){return e.digest?await Ke(e)===e.digest?true:e.nonce!==void 0?await Ye(e)===e.digest:false:false}function zt(e){let t=[],o={...e,environment:{...e.environment}};"nonce"in o&&(delete o.nonce,t.push("nonce"));let n=o.environment;if(n.locale&&typeof n.locale=="object"){let a={...n.locale};for(let i of ["timeZone","timeZoneOffsetMinutes","calendar"])i in a&&(delete a[i],t.push(`environment.locale.${i}`));n.locale=a;}if("languages"in n&&(delete n.languages,t.push("environment.languages")),n.power&&typeof n.power=="object"){let a={...n.power};for(let i of ["chargingTimeSec","dischargingTimeSec"])i in a&&(delete a[i],t.push(`environment.power.${i}`));if(typeof a.level=="number"){let i=Y(a.level);i!==a.level&&(a.level=i,t.push("environment.power.level (coarsened)"));}n.power=a;}if(n.display&&typeof n.display=="object"){let a={...n.display};for(let i of ["prefersReducedMotion","prefersColorScheme"])i in a&&(delete a[i],t.push(`environment.display.${i}`));n.display=a;}let r=t.some(a=>a!=="nonce");return r&&o.schemaVersion<3&&(o.schemaVersion=3,t.push("schemaVersion (raised)")),{run:o,changed:r,removed:t}}function oe(e){let t=e.environment,o=t.gpu.available?[t.gpu.vendor??"gpu",t.gpu.architecture].filter(Boolean).join("-"):"no-webgpu";return [t.os.platform.toLowerCase().replace(/\s+/g,"-"),o].join("/")}function ve(e,t){let o=t?.trim();if(!o||e.endsWith("/no-webgpu")||/or similar/i.test(o)||!/\d/.test(o))return e;let n=o.toLowerCase().replace(/\((tm|r|c)\)/g,"").replace(/^mesa\s+/,"").replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"");return n?`${e.split("/")[0]}/${n}`:e}function Xe(e){return ve(oe(e),e.environment.gpuModel)}var Je=3;function Gt(e,t=Je){let o=new Map;for(let r of e){let a=r.protocol,i=oe(r),s=ve(i,r.environment.gpuModel),l=r.clientSummaries??N(r),c=new Map(l.map(d=>[d.cellId,d]));for(let d of r.cells){if(d.status!=="ok")continue;let m=c.get(d.cellId);if(!m)continue;let f=[a,s,d.runtimeId,d.model.benchModelId,d.workloadId].join("|"),u=o.get(f);u||(u={row:{protocol:a,deviceClass:i,deviceSubclass:s,runtimeId:d.runtimeId,benchModelId:d.model.benchModelId,modelName:d.model.displayName,workloadId:d.workloadId,resolvedBackends:[],browsers:[],highVariance:false},ttft:[],decode:[],overall:[],single:[],batch:[],loadCold:[],loadWarm:[],quality:[],parseRate:[],runIds:new Set},o.set(f,u)),u.runIds.add(r.runId),Qe(u.row.resolvedBackends,d.resolvedBackend),Qe(u.row.browsers,r.environment.browser.name),u.row.highVariance||=m.highVariance,m.ttftMs&&u.ttft.push(m.ttftMs.median),m.decodeCharsPerSec&&u.decode.push(m.decodeCharsPerSec.median),m.overallCharsPerSec&&u.overall.push(m.overallCharsPerSec.median),m.singleLatencyMs&&u.single.push(m.singleLatencyMs.median),m.batchTextsPerSec&&u.batch.push(m.batchTextsPerSec.median),m.loadMs!==void 0&&(m.loadCached?u.loadWarm:u.loadCold).push(m.loadMs),m.qualityScore!==void 0&&u.quality.push(m.qualityScore),m.qualityParseRate!==void 0&&u.parseRate.push(m.qualityParseRate);}}let n=[];for(let r of o.values())n.push({...r.row,submissions:r.runIds.size,ttftMs:C(r.ttft),decodeCharsPerSec:C(r.decode),overallCharsPerSec:C(r.overall),singleLatencyMs:C(r.single),batchTextsPerSec:C(r.batch),loadColdMs:C(r.loadCold),loadWarmMs:C(r.loadWarm),qualityScore:C(r.quality),qualityParseRate:C(r.parseRate),provisional:r.runIds.size<t});return n.sort((r,a)=>a.protocol.localeCompare(r.protocol,void 0,{numeric:true})||r.deviceClass.localeCompare(a.deviceClass)||r.deviceSubclass.localeCompare(a.deviceSubclass)||r.benchModelId.localeCompare(a.benchModelId)||r.runtimeId.localeCompare(a.runtimeId)||r.workloadId.localeCompare(a.workloadId)),n}function Qe(e,t){t&&!e.includes(t)&&e.push(t);}function C(e){return e.length>0?q(X(e)):void 0}function q(e){return Math.round(e*100)/100}function we(e){let t=e==null?"":String(e);return /[",\n]/.test(t)?`"${t.replace(/"/g,'""')}"`:t}function Kt(e){let o=[["protocol","deviceClass","deviceSubclass","runtimeId","benchModelId","modelName","workloadId","submissions","ttftMs","decodeCharsPerSec","overallCharsPerSec","singleLatencyMs","batchTextsPerSec","loadColdMs","loadWarmMs","qualityScore","qualityParseRate","resolvedBackends","browsers","highVariance","provisional"].join(",")];for(let n of e)o.push([n.protocol,n.deviceClass,n.deviceSubclass,n.runtimeId,n.benchModelId,n.modelName,n.workloadId,n.submissions,n.ttftMs,n.decodeCharsPerSec,n.overallCharsPerSec,n.singleLatencyMs,n.batchTextsPerSec,n.loadColdMs,n.loadWarmMs,n.qualityScore,n.qualityParseRate,n.resolvedBackends.join(";"),n.browsers.join(";"),n.highVariance,n.provisional].map(we).join(","));return o.join(`
+Answer:`}function xt(e){let t=e.replace(/<think>[\s\S]*?<\/think>/gi," "),o=t.search(/<think>/i);return o===-1?t:t.slice(0,o)}function J(e,t){let o=xt(e).trim(),n=o.replace(/[*`_]/g,"").trim(),r=n.match(/(?:^|(?:[Aa]nswer|ANSWER)\s*(?:is)?\s*[:-]?\s*|(?:[Oo]ption|OPTION|[Cc]hoice|CHOICE)\s*[:-]?\s*)\(?([ABCD])\)?(?=[\s.,)!]|$)/);if(r)return he.indexOf(r[1]);let a=n.match(/^\(?([abcd])(?:\)|\.|$)/);if(a)return he.indexOf(a[1].toUpperCase());let i=o.toLowerCase();for(let s=0;s<t.length;s++)if(i.startsWith(t[s].toLowerCase().slice(0,Math.max(8,t[s].length))))return s;return null}async function pe(e,t,o){let n=H.slice(0,t),r=[],a=[],i=0,s=0;for(let l=0;l<n.length;l++){if(o?.abortSignal?.aborted)throw T();let c=n[l],m=(await e.doGenerate({prompt:We(c,o?.promptSuffix),maxTokens:qe,temperature:0,abortSignal:o?.abortSignal})).text.slice(0,je),f=J(m,c.choices);f!==null&&s++;let u=f===c.answer?1:0;i+=u,r.push(u),a.push(m),o?.onProgress?.(l+1,n.length);}return {taskId:`tinymmlu-${t}`,score:n.length===0?0:i/n.length,n:n.length,details:r,outputs:a,parseRate:n.length===0?0:s/n.length}}function _t(e,t){let o=0,n=0,r=0;for(let i=0;i<e.length;i++)o+=e[i]*t[i],n+=e[i]*e[i],r+=t[i]*t[i];let a=Math.sqrt(n)*Math.sqrt(r);return a===0?0:o/a}async function fe(e,t,o){let n=me.slice(0,t),r=[],a=[];for(let i=0;i<n.length;i++){if(o?.abortSignal?.aborted)throw T();let s=n[i],{embeddings:l}=await e.doEmbed({values:[s.s1,s.s2],abortSignal:o?.abortSignal});r.push(_t(l[0],l[1])),a.push(s.score),o?.onProgress?.(i+1,n.length);}return {taskId:`stsb-${t}`,score:r.length>=2?ue(r,a):0,n:n.length,details:r.map(i=>Math.round(i*1e4)/1e4)}}var Et=2,Oe=.2,Pt=[{maxBytes:5e8,max:4e3},{maxBytes:1e9,max:2500},{maxBytes:2e9,max:1500},{maxBytes:Number.POSITIVE_INFINITY,max:1e3}],Ct=2e3,Lt=[/swiftshader/i,/llvmpipe/i,/softpipe/i,/\bwarp\b/i,/microsoft basic render/i,/parallels/i,/vmware/i,/virtio/i,/virtualbox/i];function Ne(e,t={}){let o=[],n=s=>{o.length<50&&o.push(s);};if(typeof e!="object"||e===null)return ["result must be an object"];let r=e;t.anyProtocol?(typeof r.protocol!="string"||!/^localmode-bench\/\d+$/.test(r.protocol))&&n('protocol must be a "localmode-bench/<n>" identifier'):r.protocol!==j&&n(`protocol must be "${j}"`),z.includes(r.schemaVersion)||n(`schemaVersion must be one of ${z.join(", ")}`),(typeof r.runId!="string"||r.runId.length<8||r.runId.length>64)&&n("runId must be a string (8-64 chars)"),(typeof r.createdAt!="string"||Number.isNaN(Date.parse(r.createdAt)))&&n("createdAt must be an ISO date string");let a=r.harness;(!a||typeof a.name!="string"||typeof a.version!="string")&&n("harness {name, version} is required"),["quick","standard","thorough","custom"].includes(r.suite)||n("suite must be quick|standard|thorough|custom");let i=r.environment;if(!i||typeof i!="object")n("environment is required");else {let s=i.browser;(!s||typeof s.name!="string")&&n("environment.browser.name is required"),(!i.os||typeof i.os.platform!="string")&&n("environment.os.platform is required");}return Array.isArray(r.cells)?r.cells.length===0?n("cells must not be empty"):r.cells.length>500?n("cells exceeds the 500-cell limit"):r.cells.forEach((s,l)=>Bt(s,l,n)):n("cells must be an array"),Array.isArray(r.events)||n("events must be an array"),o}function Bt(e,t,o){if(typeof e!="object"||e===null){o(`cells[${t}] must be an object`);return}let n=e,r=`cells[${t}]`;typeof n.cellId!="string"&&o(`${r}.cellId must be a string`),typeof n.runtimeId!="string"&&o(`${r}.runtimeId must be a string`),typeof n.workloadId!="string"&&o(`${r}.workloadId must be a string`),["ok","invalid","error","skipped"].includes(n.status)||o(`${r}.status must be ok|invalid|error|skipped`);let a=n.model;(!a||typeof a.benchModelId!="string"||typeof a.providerModelId!="string")&&o(`${r}.model {benchModelId, providerModelId} is required`);let i=n.quality;if(i&&Array.isArray(i.outputs)&&i.outputs.some(s=>typeof s!="string"||s.length>500)&&o(`${r}.quality.outputs must be strings of at most 500 chars`),!Array.isArray(n.iterations)){o(`${r}.iterations must be an array`);return}n.iterations.forEach((s,l)=>{if(typeof s!="object"||s===null){o(`${r}.iterations[${l}] must be an object`);return}let c=s;(typeof c.startT!="number"||typeof c.endT!="number")&&o(`${r}.iterations[${l}] needs numeric startT/endT`),"chunks"in c&&!Array.isArray(c.chunks)&&o(`${r}.iterations[${l}].chunks must be an array`),"text"in c&&typeof c.text=="string"&&c.text.length>2e4&&o(`${r}.iterations[${l}].text exceeds 20000 chars`);});}function De(e){return e.length>0&&"chunks"in e[0]}function ee(e){let t=e.chunks.filter(r=>r.c>0);if(t.length<2)return  false;let o=t[t.length-1].t-t[0].t,n=e.endT-e.startT;return n>0&&o/n>=Oe}function Z(e,t=.05,o){let n={cellId:e.cellId,status:e.status,highVariance:false};if(e.load&&(n.loadMs=e.load.endT-e.load.startT,n.loadCached=e.load.cached),e.quality&&(n.qualityScore=e.quality.score,e.quality.parseRate!==void 0&&(n.qualityParseRate=e.quality.parseRate)),e.iterations.length===0)return n;if(De(e.iterations)){let a=e.iterations.every(ee);n.streamIncremental=a;let i=[],s=[],l=[],c=[],d=[],m=[],f=[];for(let u of e.iterations){let y=u.chunks.reduce((g,k)=>g+k.c,0),p=u.endT-u.startT;if(p>0&&(m.push(p),y>0&&f.push(y/p*1e3)),c.push(y),!a)continue;let h=u.chunks.find(g=>g.c>0);if(!h)continue;let b=h.t-u.startT;i.push(b);let S=u.chunks[u.chunks.length-1],M=y-h.c,v=S.t-h.t;v>0&&M>0&&(s.push(M/v*1e3),l.push((u.chunks.length-1)/v*1e3)),o&&b>0&&d.push(o/b*1e3);}i.length>0&&(n.ttftMs=_(i)),s.length>0&&(n.decodeCharsPerSec=_(s)),l.length>0&&(n.decodeChunksPerSec=_(l)),c.length>0&&(n.generatedChars=_(c)),d.length>0&&(n.prefillTokPerSecApprox=_(d)),m.length>0&&(n.totalMs=_(m)),f.length>0&&(n.overallCharsPerSec=_(f));}else {let a=e.iterations,i=a.map(s=>s.endT-s.startT).filter(s=>s>0);i.length>0&&(a[0].count===1?n.singleLatencyMs=_(i):n.batchTextsPerSec=_(a.filter(s=>s.endT-s.startT>0).map(s=>s.count/(s.endT-s.startT)*1e3)));}let r=[n.ttftMs,n.decodeCharsPerSec,n.overallCharsPerSec,n.singleLatencyMs,n.batchTextsPerSec];return n.highVariance=r.some(a=>a!==void 0&&a.n>=2&&a.cv>t),n}function O(e,t=.05,o){let r=o??new Map([["chat-pp128-tg128",128],["chat-pp512-tg128",512]]);return e.cells.map(a=>Z(a,t,r.get(a.workloadId)))}function Ue(e){let t=[],o=(s,l,c,d)=>t.push({code:s,message:l,severity:c,...d?{cellId:d}:{}});e.fingerprint?(e.fingerprint.mflops<=1||e.fingerprint.mflops>1e5||!Number.isFinite(e.fingerprint.checksum))&&o("fingerprint-implausible",`fingerprint mflops=${e.fingerprint.mflops}`,"reject"):o("fingerprint-missing","no hardware fingerprint in submission","reject");let n=e.environment.timerResolutionUs;for(let s of e.cells)if(s.status==="ok"){if(s.load&&s.load.endT<s.load.startT&&o("load-time-reversed","load endT precedes startT","reject",s.cellId),De(s.iterations)){for(let m of s.iterations){let f=m.startT,u=true;for(let p of m.chunks)p.t<f&&(u=false),f=p.t;if(!u||m.endT<f){o("trace-not-monotonic","chunk timestamps are not monotonic","reject",s.cellId);break}let y=m.chunks.reduce((p,h)=>p+h.c,0);if(m.text.length!==y){o("text-chunk-mismatch",`text length ${m.text.length} != chunk sum ${y}`,"reject",s.cellId);break}}let l=Z(s);s.iterations.some(m=>m.text.length<L)&&o("degenerate-generation",`an ok cell has a timed iteration with fewer than ${L} generated chars`,"reject",s.cellId);let c=s.model.sizeBytes??Number.POSITIVE_INFINITY,d=Pt.find(m=>c<=m.maxBytes);if(d&&l.decodeCharsPerSec&&l.decodeCharsPerSec.median>d.max&&o("decode-rate-envelope",`decode ${Math.round(l.decodeCharsPerSec.median)} chars/s exceeds envelope ${d.max} for model size`,"reject",s.cellId),d&&l.overallCharsPerSec&&l.overallCharsPerSec.median>d.max&&o("overall-rate-envelope",`overall ${Math.round(l.overallCharsPerSec.median)} chars/s exceeds envelope ${d.max} for model size`,"reject",s.cellId),l.ttftMs&&l.ttftMs.median<1&&o("ttft-implausible",`median TTFT ${l.ttftMs.median}ms < 1ms`,"reject",s.cellId),n!=null&&n>=4){let m=n/1e3,f=0,u=0;for(let y of s.iterations)for(let p of y.chunks)u++,Math.abs(p.t/m-Math.round(p.t/m))>.01&&f++;u>=20&&f/u>.5&&o("timer-grid",`${f}/${u} chunk timestamps off the ${n}us timer grid`,"warn",s.cellId);}}else if(s.quality?.taskId.startsWith("tinymmlu")&&s.quality.outputs){let{outputs:l,details:c,score:d,n:m}=s.quality;if(l.length!==m||!c||c.length!==m)o("quality-outputs-mismatch","quality outputs/details length disagrees with n","reject",s.cellId);else {let f=0,u=true;for(let y=0;y<m;y++){let p=H[y];if(!p)break;let h=J(l[y],p.choices)===p.answer?1:0;f+=h,h!==c[y]&&(u=false);}(!u||Math.abs(f/m-d)>1e-9)&&o("quality-details-mismatch",`submitted MMLU score ${d} disagrees with recompute ${f/m} from raw outputs`,"reject",s.cellId);}}else if(s.iterations.length>0){let l=Z(s);l.batchTextsPerSec&&l.batchTextsPerSec.median>Ct&&o("embed-rate-envelope",`embedding throughput ${Math.round(l.batchTextsPerSec.median)} texts/s exceeds envelope`,"reject",s.cellId),l.singleLatencyMs&&l.singleLatencyMs.median<.05&&o("embed-latency-implausible","sub-50us embedding latency","reject",s.cellId);}}let r=e.environment.gpu.vendor?.toLowerCase()??"",a=e.environment.os.platform.toLowerCase();r==="apple"&&a.includes("windows")&&o("env-cross-field","Apple GPU adapter with Windows platform","reject");let i=[e.environment.gpu.vendor,e.environment.gpu.architecture,e.environment.gpu.device,e.environment.gpu.description,e.environment.webglRenderer].filter(Boolean).join(" ");if(Lt.some(s=>s.test(i))){let s=e.cells.some(l=>l.status==="ok"&&/webgpu|^gpu$/.test(l.resolvedBackend));o("software-renderer",`GPU identity matches a software/virtual renderer: "${i.slice(0,120)}"`,s?"reject":"warn");}return t}function qt(e,t={}){let o=Ne(e,t);if(o.length>0)return {ok:false,shapeErrors:o,flags:[],summaries:[]};let n=O(e),r=Ue(e);if(e.clientSummaries){let a=new Map(n.map(i=>[i.cellId,i]));for(let i of e.clientSummaries){let s=a.get(i.cellId);if(s)for(let l of ["ttftMs","decodeCharsPerSec","totalMs","overallCharsPerSec","singleLatencyMs","batchTextsPerSec"]){let c=i[l]?.median,d=s[l]?.median;c!==void 0&&d!==void 0&&d!==0&&Math.abs(c-d)/Math.abs(d)>.01&&r.push({code:"client-summary-disagrees",cellId:i.cellId,message:`${l} client median ${c} vs recomputed ${d}`,severity:"reject"});}}}return {ok:r.every(a=>a.severity!=="reject"),shapeErrors:[],flags:r,summaries:n}}async function jt(e){let{policy:t,hooks:o,abortSignal:n}=e,r=e.trace??new $;await r.attach(),r.record("suite-start",e.suite);let a=K(),i=[];try{o?.onPhase?.("environment");let s=await ce({userReportedDevice:e.userReportedDevice});o?.onEnvironment?.(s),o?.onPhase?.("fingerprint"),P(n);let l=e.skipFingerprint?null:await de(),c=a!=="none"?await W():null,d=Wt(ie(e.cells)),m=e.cells.length,f=0;for(let y of d){P(n);let p=await Ot(y,{...e,trace:r,memApi:a,baselineMemory:c,cellIndexRef:()=>f++,totalCells:m});i.push(...p),p.some(h=>h.status!=="skipped")&&(r.record("cooldown-start"),await F(t.cooldownMs,n).catch(h=>{if(n?.aborted)throw h}),t.pressureGate&&await Ut(r,t.pressureGateTimeoutMs,n),r.record("cooldown-end"));}r.record("suite-end");let u={protocol:j,schemaVersion:3,runId:Vt(),createdAt:new Date().toISOString(),harness:e.harness,suite:e.suite,environment:s,fingerprint:l,cells:i,events:[...r.all]};return u.clientSummaries=O(u,t.highVarianceCv),u}finally{await r.dispose();}}function Wt(e){let t=new Map;for(let o of e){let n=`${o.model.runtimeId}/${o.model.benchModelId}/${o.model.providerModelId}`,r=t.get(n);r||(r={key:n,model:o.model,workloads:[],skipped:[]},t.set(n,r)),o.skipReason?r.skipped.push({workload:o.workload,reason:o.skipReason}):r.workloads.push(o.workload);}return [...t.values()]}async function Ot(e,t){let{model:o}=e,n=o.task==="embedding",r=n?t.embedAdapters.get(o.runtimeId):t.llmAdapters.get(o.runtimeId),a=[],i=p=>({cellId:`${o.runtimeId}/${o.benchModelId}/${p.id}`,runtimeId:o.runtimeId,runtimeVersion:r?.runtimeVersion,model:o,workloadId:p.id,workloadKind:p.kind,resolvedBackend:"unknown",load:null,iterations:[],status:"skipped"});for(let{workload:p,reason:h}of e.skipped){let b=i(p);b.invalidReasons=[h],a.push(N(b,t));}if(e.workloads.length===0)return a;if(!r){for(let p of e.workloads){let h=i(p);h.invalidReasons=[`no adapter registered for ${o.runtimeId}`],a.push(N(h,t));}return a}let s=await r.isAvailable();if(!s.ok){for(let p of e.workloads){let h=i(p);h.invalidReasons=[`runtime unavailable: ${s.reason??"unknown"}`],a.push(N(h,t));}return a}let l=null,c=null,d,m=null,f=[],u=`${o.runtimeId}/${o.benchModelId}/${e.workloads[0].id}`;for(let p=1;p<=Math.max(1,t.policy.maxAttempts);p++)try{P(t.abortSignal);let h=await r.isModelCached(o),b=[],S=w();t.hooks?.onActivity?.({cellId:u,phase:"load"}),l=await B((v,g)=>$e(r.load(o,{abortSignal:v,onProgress:k=>{g(),typeof k.pct=="number"&&(b.length===0||k.pct-b[b.length-1].pct>=2)&&b.push({t:w(),pct:Math.round(k.pct*100)/100}),t.hooks?.onLoadProgress?.(u,k.pct),t.hooks?.onActivity?.({cellId:u,phase:"load",pct:k.pct});}}),v),{parent:t.abortSignal,stallMs:t.policy.loadStallMs,timeoutMs:t.policy.loadTimeoutMs,what:"load"});let M=w();if(c={cached:h,startT:S,endT:M,progress:b.length>0?Ft(b,50):void 0,declaredBytes:o.sizeBytes},t.memApi!=="none"&&(m=await W()),t.policy.warmupRuns>0){let v=w();for(let g=0;g<t.policy.warmupRuns;g++){P(t.abortSignal);let k=l;t.hooks?.onActivity?.({cellId:u,phase:"warmup",iteration:g+1,total:t.policy.warmupRuns,chars:0,chunks:0}),await B((A,E)=>n?k.model.doEmbed({values:["warmup probe"],abortSignal:A}):Ge(k.model,{prompt:"Reply with the single word: ready",maxTokens:4,temperature:0},A,(Ze,et)=>{E(),t.hooks?.onActivity?.({cellId:u,phase:"warmup",chars:Ze,chunks:et});}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.iterationTimeoutMs,what:"warmup"}),t.hooks?.onActivity?.({cellId:u,phase:"warmup",iteration:g+1,total:t.policy.warmupRuns});}d=w()-v;}break}catch(h){if(t.abortSignal?.aborted)throw l&&await D(l),T();l&&await D(l),l=null;let b=Fe(h);if(b.name==="TimeoutError"&&t.trace.record("cell-timeout",`${u}: ${b.message}`),p<Math.max(1,t.policy.maxAttempts)){f.push({error:b,at:w()}),t.trace.record("cell-retry",`${u} attempt ${p+1}`),t.hooks?.onCellRetry?.(u,p+1,b);continue}let S=t.memApi!=="none"?await W(5e3):null;for(let M of e.workloads){let v=i(M);v.status="error",v.load=c,v.error=b,f.length>0&&(v.attempts=[...f]),t.memApi!=="none"&&(v.memory={api:t.memApi,baseline:t.baselineMemory??void 0,postLoad:m??void 0,atError:S??void 0}),a.push(N(v,t));}return a}if(!l)return a;let y=false;for(let p of e.workloads){P(t.abortSignal);let h=i(p);h.resolvedBackend=l.resolvedBackend,l.runtimeConfig&&(h.runtimeConfig={...l.runtimeConfig}),y||(h.load=c,h.warmupMs=d,f.length>0&&(h.attempts=[...f]),y=true);let b=t.cellIndexRef();t.hooks?.onCellStart?.(h.cellId,b,t.totalCells);let S=l,M=async()=>{switch(p.kind){case "llm-generate":await Nt(h,S,p,t);break;case "embed-single":case "embed-batch":await Dt(h,S,p,t);break;case "quality-mmlu":{let g=p.items;h.quality=await B((k,A)=>pe(S.model,g,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onIteration?.(h.cellId,E,g),t.hooks?.onActivity?.({cellId:h.cellId,phase:"quality",iteration:E,total:g});},promptSuffix:o.qualityPromptSuffix}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.qualityTimeoutMs,what:"quality lane"}),h.status="ok";break}case "quality-sts":{let g=p.items;h.quality=await B((k,A)=>fe(S.model,g,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onIteration?.(h.cellId,E,g),t.hooks?.onActivity?.({cellId:h.cellId,phase:"quality",iteration:E,total:g});}}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.qualityTimeoutMs,what:"quality lane"}),h.status="ok";break}}},v=Math.max(1,t.policy.maxAttempts);for(let g=1;g<=v;g++)try{await M();break}catch(k){if(t.abortSignal?.aborted)throw await D(l),T();let A=Fe(k);if(A.name==="TimeoutError"&&t.trace.record("cell-timeout",`${h.cellId}: ${A.message}`),g<v){h.attempts=[...h.attempts??[],{error:A,at:w()}],h.iterations=[],h.quality=void 0,h.invalidReasons=void 0,h.status="skipped",t.trace.record("cell-retry",`${h.cellId} attempt ${g+1}`),t.hooks?.onCellRetry?.(h.cellId,g+1,A);continue}h.status="error",h.error=A;}if(t.memApi!=="none"&&(h.status==="ok"||h.status==="error")){let g=await W(5e3);h.memory={api:t.memApi,baseline:t.baselineMemory??void 0,postLoad:m??void 0,...h.status==="ok"?{postRun:g??void 0}:{atError:g??void 0}};}a.push(N(h,t));}if(await D(l),t.policy.measureWarmReload&&c?.cached===false)try{P(t.abortSignal);let p=`${o.runtimeId}/${o.benchModelId}/warm-reload`;t.hooks?.onActivity?.({cellId:p,phase:"reload"});let h=w(),b=await B((k,A)=>$e(r.load(o,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onActivity?.({cellId:p,phase:"reload",pct:E.pct});}}),k),{parent:t.abortSignal,stallMs:t.policy.loadStallMs,timeoutMs:t.policy.loadTimeoutMs,what:"load"}),S=w(),M=b.resolvedBackend,v=b.runtimeConfig?{...b.runtimeConfig}:void 0;await D(b);let g={cellId:`${o.runtimeId}/${o.benchModelId}/warm-reload`,runtimeId:o.runtimeId,runtimeVersion:r.runtimeVersion,model:o,workloadId:"warm-reload",workloadKind:"llm-generate",resolvedBackend:M,...v?{runtimeConfig:v}:{},load:{cached:!0,startT:h,endT:S,declaredBytes:o.sizeBytes},iterations:[],status:"ok"};a.push(N(g,t));}catch{if(t.abortSignal?.aborted)throw T()}return a}async function Nt(e,t,o,n){let r=[],a=[];for(let i=0;i<n.policy.timedRuns;i++){let s=i+1;for(let l=0;;l++){P(n.abortSignal),n.hooks?.onIteration?.(e.cellId,s,n.policy.timedRuns),await ze(n,e.cellId);let c=[];n.trace.isHidden&&c.push("started-hidden");let d=ne(n.trace);n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:s,total:n.policy.timedRuns,chars:0,chunks:0});let m=w(),{chunks:f,text:u,providerUsage:y,finishReason:p}=await B((S,M)=>Ge(t.model,{prompt:o.prompt,systemPrompt:o.systemPrompt,maxTokens:o.maxTokens,temperature:o.temperature},S,(v,g)=>{M(),n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:s,total:n.policy.timedRuns,chars:v,chunks:g});}),{parent:n.abortSignal,stallMs:n.policy.chunkStallMs,timeoutMs:n.policy.iterationTimeoutMs,what:`iteration ${s}`}),h=w();ne(n.trace)>d&&c.push("hidden-during-run"),u.length<L&&c.push("degenerate-output");let b={startT:m,chunks:f,endT:h,text:u,providerUsage:y?{...y,fidelity:se[e.runtimeId]}:void 0,finishReason:p,gates:c};if(Ve(c)&&l<Math.max(1,n.policy.maxAttempts)){a.push(b),n.trace.record("iteration-redo",`${e.cellId} iteration ${s}: ${c.join(",")}`);continue}r.push(b);break}}e.iterations=r,a.length>0&&(e.discardedIterations=a),He(e,r);}function Ve(e){return e.includes("started-hidden")||e.includes("hidden-during-run")}function He(e,t){let o=t.some(n=>n.gates.length>0);if(e.status=o?"invalid":"ok",o){let n=new Set;for(let r of t)for(let a of r.gates)n.add(a==="degenerate-output"?`degenerate output: fewer than ${L} generated chars in a timed iteration`:a==="started-hidden"||a==="hidden-during-run"?"tab hidden during a timed iteration (the tab did not return in time for a repeat)":"validity gate fired during a timed region");e.invalidReasons=[...n];}}async function ze(e,t){if(!e.trace.isHidden)return  true;e.hooks?.onActivity?.({cellId:t,phase:"waiting-visible"});let o=w()+e.policy.visibilityWaitMs;for(;e.trace.isHidden&&w()<o;)P(e.abortSignal),await F(Math.min(250,e.policy.visibilityWaitMs),e.abortSignal);return !e.trace.isHidden}async function Dt(e,t,o,n){let r=[],a=[];for(let i=0;i<n.policy.timedRuns;i++)for(let s=0;;s++){P(n.abortSignal),n.hooks?.onIteration?.(e.cellId,i+1,n.policy.timedRuns),await ze(n,e.cellId);let l=[];n.trace.isHidden&&l.push("started-hidden");let c=ne(n.trace),d=w(),{embeddings:m}=await B(y=>t.model.doEmbed({values:o.texts,abortSignal:y}),{parent:n.abortSignal,stallMs:n.policy.iterationTimeoutMs,timeoutMs:n.policy.iterationTimeoutMs,what:`iteration ${i+1}`}),f=w();ne(n.trace)>c&&l.push("hidden-during-run"),n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:i+1,total:n.policy.timedRuns});let u={startT:d,endT:f,count:o.texts.length,dimensions:m[0]?.length??t.model.dimensions,gates:l};if(Ve(l)&&s<Math.max(1,n.policy.maxAttempts)){a.push(u),n.trace.record("iteration-redo",`${e.cellId} iteration ${i+1}: ${l.join(",")}`);continue}r.push(u);break}e.iterations=r,a.length>0&&(e.discardedIterations=a),He(e,r);}async function Ge(e,t,o,n){let r=[],a="",i,s;if(e.doStream)for await(let l of e.doStream({...t,abortSignal:o})){let c=w();l.text.length>0&&(r.push({t:c,c:l.text.length}),a+=l.text,n?.(a.length,r.length)),l.done&&(i=l.usage,s=l.finishReason);}else {let l=await e.doGenerate({...t,abortSignal:o}),c=w();a=l.text,r.push({t:c,c:l.text.length}),n?.(a.length,r.length),i=l.usage,s=l.finishReason;}return {chunks:r,text:a,providerUsage:i,finishReason:s}}var te=class extends Error{constructor(t){super(t),this.name="TimeoutError";}};async function B(e,t){let o=new AbortController,n=()=>{},r=new Promise((u,y)=>{n=y;}),a=()=>{o.abort(t.parent?.reason),n(T());};t.parent?.aborted&&a(),t.parent?.addEventListener("abort",a,{once:true});let i=w(),s=i,l=false,c=u=>{l||(l=true,o.abort(new te(u)),n(new te(u)));},m=setInterval(()=>{if(l)return;let u=w();if(u-s>=t.stallMs){c(t.what==="load"?`load made no progress for ${t.stallMs} ms`:`${t.what}: no stream progress for ${t.stallMs} ms`);return}u-i>=t.timeoutMs&&c(`${t.what}: exceeded the ${t.timeoutMs} ms budget`);},Math.max(10,Math.min(t.stallMs,t.timeoutMs)/4)),f=()=>{s=w();};try{let u=e(o.signal,f);return u.catch(()=>{}),await Promise.race([u,r])}finally{l=true,clearInterval(m),t.parent?.removeEventListener("abort",a);}}function ne(e){let t=0;for(let o of e.all)o.type==="visibility-hidden"&&t++;return t}async function Ut(e,t,o){let n=w()+t;for(;w()<n;){let r=e.pressureState;if(r===void 0||r==="nominal"||r==="fair")return;await F(1e3,o);}}function Ft(e,t){if(e.length<=t)return e;let o=[],n=(e.length-1)/(t-1);for(let r=0;r<t;r++)o.push(e[Math.round(r*n)]);return o}var $t=4e3;function Fe(e){let t=e,o=t?.cause,n=o instanceof Error?o.message:typeof o=="string"?o:void 0,r=o instanceof Error&&o.name&&o.name!=="Error"?o.name:void 0,a=o instanceof Error&&typeof o.stack=="string"&&o.stack.length>0?o.stack.slice(0,$t):void 0;return {name:t?.name??"Error",message:t?.message??String(e),...n!==void 0?{cause:n}:{},...r?{causeName:r}:{},...a?{causeStack:a}:{}}}function N(e,t){return t.hooks?.onCellFinish?.(e),e}async function $e(e,t){let o=await e;if(t.aborted)throw await D(o),T();return o}async function D(e){try{await e.dispose();}catch{}}function P(e){if(e?.aborted)throw T()}function Vt(){let e=globalThis.crypto;if(e?.randomUUID)return e.randomUUID();let t=new Uint8Array(16);if(e?.getRandomValues)e.getRandomValues(t);else for(let n=0;n<16;n++)t[n]=Math.floor(Math.random()*256);t[6]=t[6]&15|64,t[8]=t[8]&63|128;let o=[...t].map(n=>n.toString(16).padStart(2,"0")).join("");return `${o.slice(0,8)}-${o.slice(8,12)}-${o.slice(12,16)}-${o.slice(16,20)}-${o.slice(20)}`}function be(e){return ge(e)}function ge(e){if(e===null||typeof e=="number"||typeof e=="boolean"||typeof e=="string")return JSON.stringify(e);if(typeof e>"u")return "null";if(Array.isArray(e))return `[${e.map(t=>ge(t)).join(",")}]`;if(typeof e=="object")return `{${Object.entries(e).filter(([,o])=>o!==void 0).sort(([o],[n])=>o<n?-1:o>n?1:0).map(([o,n])=>`${JSON.stringify(o)}:${ge(n)}`).join(",")}}`;throw new TypeError(`canonicalJson: unsupported value type ${typeof e}`)}async function ye(e){let t=globalThis.crypto?.subtle;if(!t)throw new Error("Web Crypto (crypto.subtle) is not available in this environment");let o=new TextEncoder().encode(e),n=await t.digest("SHA-256",o);return [...new Uint8Array(n)].map(r=>r.toString(16).padStart(2,"0")).join("")}async function Ke(e){let{digest:t,nonce:o,...n}=e;return ye(be(n))}async function Ye(e){let{digest:t,...o}=e;return ye(be(o))}async function Ht(e){return e.digest?await Ke(e)===e.digest?true:e.nonce!==void 0?await Ye(e)===e.digest:false:false}function zt(e){let t=[],o={...e,environment:{...e.environment}};"nonce"in o&&(delete o.nonce,t.push("nonce"));let n=o.environment;if(n.locale&&typeof n.locale=="object"){let a={...n.locale};for(let i of ["timeZone","timeZoneOffsetMinutes","calendar"])i in a&&(delete a[i],t.push(`environment.locale.${i}`));n.locale=a;}if("languages"in n&&(delete n.languages,t.push("environment.languages")),n.power&&typeof n.power=="object"){let a={...n.power};for(let i of ["chargingTimeSec","dischargingTimeSec"])i in a&&(delete a[i],t.push(`environment.power.${i}`));if(typeof a.level=="number"){let i=Y(a.level);i!==a.level&&(a.level=i,t.push("environment.power.level (coarsened)"));}n.power=a;}if(n.display&&typeof n.display=="object"){let a={...n.display};for(let i of ["prefersReducedMotion","prefersColorScheme"])i in a&&(delete a[i],t.push(`environment.display.${i}`));n.display=a;}let r=t.some(a=>a!=="nonce");return r&&o.schemaVersion<3&&(o.schemaVersion=3,t.push("schemaVersion (raised)")),{run:o,changed:r,removed:t}}function oe(e){let t=e.environment,o=t.gpu.available?[t.gpu.vendor??"gpu",t.gpu.architecture].filter(Boolean).join("-"):"no-webgpu";return [t.os.platform.toLowerCase().replace(/\s+/g,"-"),o].join("/")}function ve(e,t){let o=t?.trim();if(!o||e.endsWith("/no-webgpu")||/or similar/i.test(o)||!/\d/.test(o))return e;let n=o.toLowerCase().replace(/\((tm|r|c)\)/g,"").replace(/^mesa\s+/,"").replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"");return n?`${e.split("/")[0]}/${n}`:e}function Xe(e){return ve(oe(e),e.environment.gpuModel)}var Je=3;function Gt(e,t=Je){let o=new Map;for(let r of e){let a=r.protocol,i=oe(r),s=ve(i,r.environment.gpuModel),l=r.clientSummaries??O(r),c=new Map(l.map(d=>[d.cellId,d]));for(let d of r.cells){if(d.status!=="ok")continue;let m=c.get(d.cellId);if(!m)continue;let f=[a,s,d.runtimeId,d.model.benchModelId,d.workloadId].join("|"),u=o.get(f);u||(u={row:{protocol:a,deviceClass:i,deviceSubclass:s,runtimeId:d.runtimeId,benchModelId:d.model.benchModelId,modelName:d.model.displayName,workloadId:d.workloadId,resolvedBackends:[],browsers:[],highVariance:false},ttft:[],decode:[],overall:[],single:[],batch:[],loadCold:[],loadWarm:[],quality:[],parseRate:[],runIds:new Set},o.set(f,u)),u.runIds.add(r.runId),Qe(u.row.resolvedBackends,d.resolvedBackend),Qe(u.row.browsers,r.environment.browser.name),u.row.highVariance||=m.highVariance,m.ttftMs&&u.ttft.push(m.ttftMs.median),m.decodeCharsPerSec&&u.decode.push(m.decodeCharsPerSec.median),m.overallCharsPerSec&&u.overall.push(m.overallCharsPerSec.median),m.singleLatencyMs&&u.single.push(m.singleLatencyMs.median),m.batchTextsPerSec&&u.batch.push(m.batchTextsPerSec.median),m.loadMs!==void 0&&(m.loadCached?u.loadWarm:u.loadCold).push(m.loadMs),m.qualityScore!==void 0&&u.quality.push(m.qualityScore),m.qualityParseRate!==void 0&&u.parseRate.push(m.qualityParseRate);}}let n=[];for(let r of o.values())n.push({...r.row,submissions:r.runIds.size,ttftMs:C(r.ttft),decodeCharsPerSec:C(r.decode),overallCharsPerSec:C(r.overall),singleLatencyMs:C(r.single),batchTextsPerSec:C(r.batch),loadColdMs:C(r.loadCold),loadWarmMs:C(r.loadWarm),qualityScore:C(r.quality),qualityParseRate:C(r.parseRate),provisional:r.runIds.size<t});return n.sort((r,a)=>a.protocol.localeCompare(r.protocol,void 0,{numeric:true})||r.deviceClass.localeCompare(a.deviceClass)||r.deviceSubclass.localeCompare(a.deviceSubclass)||r.benchModelId.localeCompare(a.benchModelId)||r.runtimeId.localeCompare(a.runtimeId)||r.workloadId.localeCompare(a.workloadId)),n}function Qe(e,t){t&&!e.includes(t)&&e.push(t);}function C(e){return e.length>0?q(X(e)):void 0}function q(e){return Math.round(e*100)/100}function we(e){let t=e==null?"":String(e);return /[",\n]/.test(t)?`"${t.replace(/"/g,'""')}"`:t}function Kt(e){let o=[["protocol","deviceClass","deviceSubclass","runtimeId","benchModelId","modelName","workloadId","submissions","ttftMs","decodeCharsPerSec","overallCharsPerSec","singleLatencyMs","batchTextsPerSec","loadColdMs","loadWarmMs","qualityScore","qualityParseRate","resolvedBackends","browsers","highVariance","provisional"].join(",")];for(let n of e)o.push([n.protocol,n.deviceClass,n.deviceSubclass,n.runtimeId,n.benchModelId,n.modelName,n.workloadId,n.submissions,n.ttftMs,n.decodeCharsPerSec,n.overallCharsPerSec,n.singleLatencyMs,n.batchTextsPerSec,n.loadColdMs,n.loadWarmMs,n.qualityScore,n.qualityParseRate,n.resolvedBackends.join(";"),n.browsers.join(";"),n.highVariance,n.provisional].map(we).join(","));return o.join(`
 `)+`
 `}function Yt(e){let o=[["runId","createdAt","protocol","suite","deviceClass","deviceSubclass","browser","browserVersion","os","gpuVendor","gpuArchitecture","cores","deviceMemoryGB","crossOriginIsolated","fingerprintMflops","runtimeId","runtimeVersion","benchModelId","providerModelId","quantization","sizeBytes","workloadId","resolvedBackend","iteration","ttftMs","decodeCharsPerSec","generatedChars","overallCharsPerSec","streamIncremental","durationMs","loadMs","loadCached","status"].join(",")];for(let n of e){let r=n.environment,a=[n.runId,n.createdAt,n.protocol,n.suite,oe(n),Xe(n),r.browser.name,r.browser.version,r.os.platform,r.gpu.vendor??"",r.gpu.architecture??"",r.hardware.cores??"",r.hardware.deviceMemoryGB??"",r.flags.crossOriginIsolated,n.fingerprint?q(n.fingerprint.mflops):""];for(let i of n.cells){let s=i.load?q(i.load.endT-i.load.startT):"",l=i.load?String(i.load.cached):"",c=[i.runtimeId,i.runtimeVersion??"",i.model.benchModelId,i.model.providerModelId,i.model.quantization??"",i.model.sizeBytes??"",i.workloadId,i.resolvedBackend];if(i.iterations.length===0){o.push([...a,...c,"","","","","","","",s,l,i.status].map(we).join(","));continue}i.iterations.forEach((d,m)=>{let f="",u="",y="",p="",h="";if("chunks"in d){let b=d.chunks.reduce((g,k)=>g+k.c,0);y=String(b);let S=d.endT-d.startT;S>0&&b>0&&(p=String(q(b/S*1e3)));let M=ee(d);h=String(M);let v=d.chunks.find(g=>g.c>0);if(M&&v){f=String(q(v.t-d.startT));let g=d.chunks[d.chunks.length-1],k=b-v.c,A=g.t-v.t;A>0&&(u=String(q(k/A*1e3)));}}o.push([...a,...c,m+1,f,u,y,p,h,q(d.endT-d.startT),s,l,i.status].map(we).join(","));});}}return o.join(`
 `)+`
-`}export{z as ACCEPTED_SCHEMA_VERSIONS,j as BENCH_PROTOCOL_VERSION,U as BENCH_SCHEMA_VERSION,Ae as EMBED_WORKLOADS,ae as GENERATION_BUDGET,Je as HEADLINE_MIN_SUBMISSIONS,tt as LEADERBOARD_PROTOCOL_VERSIONS,Se as LLM_WORKLOADS,L as MIN_GENERATED_CHARS,qe as MMLU_MAX_TOKENS,je as MMLU_OUTPUT_CAP,Et as PLAUSIBILITY_RULES_VERSION,Ie as QUALITY_WORKLOADS,G as RUNTIME_EXECUTION_ORDER,it as RUN_POLICIES,Ne as STREAM_COHERENCE_MIN_SPAN_RATIO,me as STSB_SUBSET,H as TINY_MMLU,$ as TraceRecorder,se as USAGE_FIDELITY,at as WORKLOADS_BY_ID,Gt as aggregateRuns,be as canonicalJson,ce as captureEnvironment,Ue as checkPlausibility,Y as coarseBatteryLevel,Ye as computeLegacyRunDigest,Ke as computeRunDigest,Te as deriveDeviceType,Re as detectEngine,Ee as detectWasmFeatures,oe as deviceClassOf,Xe as deviceSubclassOf,We as formatMMLUPrompt,Rt as geomean,w as hrNow,le as inferTimerResolutionUs,ee as isIncrementalStream,V as mean,X as median,K as memoryApiAvailable,ie as orderCells,_e as parseGpuModel,J as parseMMLUAnswer,Me as parseUserAgent,Q as quantile,ve as refineDeviceClass,xe as resolveGpuModel,Kt as rowsToCSV,jt as runBenchmarkSuite,de as runFingerprint,pe as runMMLUFidelity,fe as runSTSQuality,Yt as runsToLongCSV,W as sampleMemoryBytes,zt as scrubRunForPublication,ye as sha256Hex,F as sleep,ue as spearman,Be as stddev,_ as summarize,Z as summarizeCell,N as summarizeRun,Oe as validateRunShape,qt as validateSubmission,Ht as verifyRunDigest};//# sourceMappingURL=index.js.map
+`}export{z as ACCEPTED_SCHEMA_VERSIONS,j as BENCH_PROTOCOL_VERSION,U as BENCH_SCHEMA_VERSION,Ae as EMBED_WORKLOADS,ae as GENERATION_BUDGET,Je as HEADLINE_MIN_SUBMISSIONS,tt as LEADERBOARD_PROTOCOL_VERSIONS,Se as LLM_WORKLOADS,L as MIN_GENERATED_CHARS,qe as MMLU_MAX_TOKENS,je as MMLU_OUTPUT_CAP,Et as PLAUSIBILITY_RULES_VERSION,Ie as QUALITY_WORKLOADS,G as RUNTIME_EXECUTION_ORDER,it as RUN_POLICIES,Oe as STREAM_COHERENCE_MIN_SPAN_RATIO,me as STSB_SUBSET,H as TINY_MMLU,$ as TraceRecorder,se as USAGE_FIDELITY,at as WORKLOADS_BY_ID,Gt as aggregateRuns,be as canonicalJson,ce as captureEnvironment,Ue as checkPlausibility,Y as coarseBatteryLevel,Ye as computeLegacyRunDigest,Ke as computeRunDigest,Te as deriveDeviceType,Re as detectEngine,Ee as detectWasmFeatures,oe as deviceClassOf,Xe as deviceSubclassOf,We as formatMMLUPrompt,Rt as geomean,w as hrNow,le as inferTimerResolutionUs,ee as isIncrementalStream,V as mean,X as median,K as memoryApiAvailable,ie as orderCells,_e as parseGpuModel,J as parseMMLUAnswer,Me as parseUserAgent,Q as quantile,ve as refineDeviceClass,xe as resolveGpuModel,Kt as rowsToCSV,jt as runBenchmarkSuite,de as runFingerprint,pe as runMMLUFidelity,fe as runSTSQuality,Yt as runsToLongCSV,W as sampleMemoryBytes,zt as scrubRunForPublication,ye as sha256Hex,F as sleep,ue as spearman,Be as stddev,_ as summarize,Z as summarizeCell,O as summarizeRun,Ne as validateRunShape,qt as validateSubmission,Ht as verifyRunDigest};//# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map