@localmode/bench 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/index.cjs +2 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +12 -2
- package/dist/index.d.ts +12 -2
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -20,9 +20,12 @@ the previous one) in separate rows that never mix (`aggregateRuns()` groups by
|
|
|
20
20
|
v5 changes only the llama.cpp lanes: they request half the browser's logical
|
|
21
21
|
thread count (at least two) instead of all of it, because a pool over every
|
|
22
22
|
logical thread ran at half speed with high variance on hybrid and SMT
|
|
23
|
-
processors, and load with a 2,048-token context
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
processors, and load with a 2,048-token context sized to the workloads (about
|
|
24
|
+
700 tokens) instead of the provider's 8,192 default, which shrinks the KV
|
|
25
|
+
cache of the 3.46 GB Gemma 4 E2B GGUF inside the CPU lane's 4 GB wasm heap
|
|
26
|
+
(that lane's Gemma quality cell still fails there on the per-request state
|
|
27
|
+
allocation, recorded as an error as under v4); `n_ctx`, `n_threads`,
|
|
28
|
+
`multithread` and `n_threads_used` are recorded per cell. v3 split the llama.cpp lane: `wllama` is llama.cpp
|
|
26
29
|
WASM on the CPU (`n_gpu_layers: 0`) and `wllama-webgpu` offloads every layer
|
|
27
30
|
to WebGPU, over the same GGUF files; under v2 the single `wllama` lane ran on
|
|
28
31
|
WebGPU wherever the browser had it while recording `wasm` (wllama 3.5's
|
package/dist/index.cjs
CHANGED
|
@@ -31,9 +31,9 @@ ${e.question}
|
|
|
31
31
|
${o}
|
|
32
32
|
|
|
33
33
|
Answer with only the letter (A, B, C, or D) of the correct choice.${t??""}
|
|
34
|
-
Answer:`}function xt(e){let t=e.replace(/<think>[\s\S]*?<\/think>/gi," "),o=t.search(/<think>/i);return o===-1?t:t.slice(0,o)}function J(e,t){let o=xt(e).trim(),n=o.replace(/[*`_]/g,"").trim(),r=n.match(/(?:^|(?:[Aa]nswer|ANSWER)\s*(?:is)?\s*[:-]?\s*|(?:[Oo]ption|OPTION|[Cc]hoice|CHOICE)\s*[:-]?\s*)\(?([ABCD])\)?(?=[\s.,)!]|$)/);if(r)return he.indexOf(r[1]);let a=n.match(/^\(?([abcd])(?:\)|\.|$)/);if(a)return he.indexOf(a[1].toUpperCase());let i=o.toLowerCase();for(let s=0;s<t.length;s++)if(i.startsWith(t[s].toLowerCase().slice(0,Math.max(8,t[s].length))))return s;return null}async function pe(e,t,o){let n=H.slice(0,t),r=[],a=[],i=0,s=0;for(let l=0;l<n.length;l++){if(o?.abortSignal?.aborted)throw T();let c=n[l],m=(await e.doGenerate({prompt:We(c,o?.promptSuffix),maxTokens:qe,temperature:0,abortSignal:o?.abortSignal})).text.slice(0,je),f=J(m,c.choices);f!==null&&s++;let u=f===c.answer?1:0;i+=u,r.push(u),a.push(m),o?.onProgress?.(l+1,n.length);}return {taskId:`tinymmlu-${t}`,score:n.length===0?0:i/n.length,n:n.length,details:r,outputs:a,parseRate:n.length===0?0:s/n.length}}function _t(e,t){let o=0,n=0,r=0;for(let i=0;i<e.length;i++)o+=e[i]*t[i],n+=e[i]*e[i],r+=t[i]*t[i];let a=Math.sqrt(n)*Math.sqrt(r);return a===0?0:o/a}async function fe(e,t,o){let n=me.slice(0,t),r=[],a=[];for(let i=0;i<n.length;i++){if(o?.abortSignal?.aborted)throw T();let s=n[i],{embeddings:l}=await e.doEmbed({values:[s.s1,s.s2],abortSignal:o?.abortSignal});r.push(_t(l[0],l[1])),a.push(s.score),o?.onProgress?.(i+1,n.length);}return {taskId:`stsb-${t}`,score:r.length>=2?ue(r,a):0,n:n.length,details:r.map(i=>Math.round(i*1e4)/1e4)}}var Et=2,Ne=.2,Pt=[{maxBytes:5e8,max:4e3},{maxBytes:1e9,max:2500},{maxBytes:2e9,max:1500},{maxBytes:Number.POSITIVE_INFINITY,max:1e3}],Ct=2e3,Lt=[/swiftshader/i,/llvmpipe/i,/softpipe/i,/\bwarp\b/i,/microsoft basic render/i,/parallels/i,/vmware/i,/virtio/i,/virtualbox/i];function Oe(e){let t=[],o=i=>{t.length<50&&t.push(i);};if(typeof e!="object"||e===null)return ["result must be an object"];let n=e;n.protocol!==j&&o(`protocol must be "${j}"`),z.includes(n.schemaVersion)||o(`schemaVersion must be one of ${z.join(", ")}`),(typeof n.runId!="string"||n.runId.length<8||n.runId.length>64)&&o("runId must be a string (8-64 chars)"),(typeof n.createdAt!="string"||Number.isNaN(Date.parse(n.createdAt)))&&o("createdAt must be an ISO date string");let r=n.harness;(!r||typeof r.name!="string"||typeof r.version!="string")&&o("harness {name, version} is required"),["quick","standard","thorough","custom"].includes(n.suite)||o("suite must be quick|standard|thorough|custom");let a=n.environment;if(!a||typeof a!="object")o("environment is required");else {let i=a.browser;(!i||typeof i.name!="string")&&o("environment.browser.name is required"),(!a.os||typeof a.os.platform!="string")&&o("environment.os.platform is required");}return Array.isArray(n.cells)?n.cells.length===0?o("cells must not be empty"):n.cells.length>500?o("cells exceeds the 500-cell limit"):n.cells.forEach((i,s)=>Bt(i,s,o)):o("cells must be an array"),Array.isArray(n.events)||o("events must be an array"),t}function Bt(e,t,o){if(typeof e!="object"||e===null){o(`cells[${t}] must be an object`);return}let n=e,r=`cells[${t}]`;typeof n.cellId!="string"&&o(`${r}.cellId must be a string`),typeof n.runtimeId!="string"&&o(`${r}.runtimeId must be a string`),typeof n.workloadId!="string"&&o(`${r}.workloadId must be a string`),["ok","invalid","error","skipped"].includes(n.status)||o(`${r}.status must be ok|invalid|error|skipped`);let a=n.model;(!a||typeof a.benchModelId!="string"||typeof a.providerModelId!="string")&&o(`${r}.model {benchModelId, providerModelId} is required`);let i=n.quality;if(i&&Array.isArray(i.outputs)&&i.outputs.some(s=>typeof s!="string"||s.length>500)&&o(`${r}.quality.outputs must be strings of at most 500 chars`),!Array.isArray(n.iterations)){o(`${r}.iterations must be an array`);return}n.iterations.forEach((s,l)=>{if(typeof s!="object"||s===null){o(`${r}.iterations[${l}] must be an object`);return}let c=s;(typeof c.startT!="number"||typeof c.endT!="number")&&o(`${r}.iterations[${l}] needs numeric startT/endT`),"chunks"in c&&!Array.isArray(c.chunks)&&o(`${r}.iterations[${l}].chunks must be an array`),"text"in c&&typeof c.text=="string"&&c.text.length>2e4&&o(`${r}.iterations[${l}].text exceeds 20000 chars`);});}function De(e){return e.length>0&&"chunks"in e[0]}function ee(e){let t=e.chunks.filter(r=>r.c>0);if(t.length<2)return false;let o=t[t.length-1].t-t[0].t,n=e.endT-e.startT;return n>0&&o/n>=Ne}function Z(e,t=.05,o){let n={cellId:e.cellId,status:e.status,highVariance:false};if(e.load&&(n.loadMs=e.load.endT-e.load.startT,n.loadCached=e.load.cached),e.quality&&(n.qualityScore=e.quality.score,e.quality.parseRate!==void 0&&(n.qualityParseRate=e.quality.parseRate)),e.iterations.length===0)return n;if(De(e.iterations)){let a=e.iterations.every(ee);n.streamIncremental=a;let i=[],s=[],l=[],c=[],d=[],m=[],f=[];for(let u of e.iterations){let y=u.chunks.reduce((g,k)=>g+k.c,0),p=u.endT-u.startT;if(p>0&&(m.push(p),y>0&&f.push(y/p*1e3)),c.push(y),!a)continue;let h=u.chunks.find(g=>g.c>0);if(!h)continue;let b=h.t-u.startT;i.push(b);let S=u.chunks[u.chunks.length-1],M=y-h.c,v=S.t-h.t;v>0&&M>0&&(s.push(M/v*1e3),l.push((u.chunks.length-1)/v*1e3)),o&&b>0&&d.push(o/b*1e3);}i.length>0&&(n.ttftMs=_(i)),s.length>0&&(n.decodeCharsPerSec=_(s)),l.length>0&&(n.decodeChunksPerSec=_(l)),c.length>0&&(n.generatedChars=_(c)),d.length>0&&(n.prefillTokPerSecApprox=_(d)),m.length>0&&(n.totalMs=_(m)),f.length>0&&(n.overallCharsPerSec=_(f));}else {let a=e.iterations,i=a.map(s=>s.endT-s.startT).filter(s=>s>0);i.length>0&&(a[0].count===1?n.singleLatencyMs=_(i):n.batchTextsPerSec=_(a.filter(s=>s.endT-s.startT>0).map(s=>s.count/(s.endT-s.startT)*1e3)));}let r=[n.ttftMs,n.decodeCharsPerSec,n.overallCharsPerSec,n.singleLatencyMs,n.batchTextsPerSec];return n.highVariance=r.some(a=>a!==void 0&&a.n>=2&&a.cv>t),n}function N(e,t=.05,o){let r=o??new Map([["chat-pp128-tg128",128],["chat-pp512-tg128",512]]);return e.cells.map(a=>Z(a,t,r.get(a.workloadId)))}function Ue(e){let t=[],o=(s,l,c,d)=>t.push({code:s,message:l,severity:c,...d?{cellId:d}:{}});e.fingerprint?(e.fingerprint.mflops<=1||e.fingerprint.mflops>1e5||!Number.isFinite(e.fingerprint.checksum))&&o("fingerprint-implausible",`fingerprint mflops=${e.fingerprint.mflops}`,"reject"):o("fingerprint-missing","no hardware fingerprint in submission","reject");let n=e.environment.timerResolutionUs;for(let s of e.cells)if(s.status==="ok"){if(s.load&&s.load.endT<s.load.startT&&o("load-time-reversed","load endT precedes startT","reject",s.cellId),De(s.iterations)){for(let m of s.iterations){let f=m.startT,u=true;for(let p of m.chunks)p.t<f&&(u=false),f=p.t;if(!u||m.endT<f){o("trace-not-monotonic","chunk timestamps are not monotonic","reject",s.cellId);break}let y=m.chunks.reduce((p,h)=>p+h.c,0);if(m.text.length!==y){o("text-chunk-mismatch",`text length ${m.text.length} != chunk sum ${y}`,"reject",s.cellId);break}}let l=Z(s);s.iterations.some(m=>m.text.length<L)&&o("degenerate-generation",`an ok cell has a timed iteration with fewer than ${L} generated chars`,"reject",s.cellId);let c=s.model.sizeBytes??Number.POSITIVE_INFINITY,d=Pt.find(m=>c<=m.maxBytes);if(d&&l.decodeCharsPerSec&&l.decodeCharsPerSec.median>d.max&&o("decode-rate-envelope",`decode ${Math.round(l.decodeCharsPerSec.median)} chars/s exceeds envelope ${d.max} for model size`,"reject",s.cellId),d&&l.overallCharsPerSec&&l.overallCharsPerSec.median>d.max&&o("overall-rate-envelope",`overall ${Math.round(l.overallCharsPerSec.median)} chars/s exceeds envelope ${d.max} for model size`,"reject",s.cellId),l.ttftMs&&l.ttftMs.median<1&&o("ttft-implausible",`median TTFT ${l.ttftMs.median}ms < 1ms`,"reject",s.cellId),n!=null&&n>=4){let m=n/1e3,f=0,u=0;for(let y of s.iterations)for(let p of y.chunks)u++,Math.abs(p.t/m-Math.round(p.t/m))>.01&&f++;u>=20&&f/u>.5&&o("timer-grid",`${f}/${u} chunk timestamps off the ${n}us timer grid`,"warn",s.cellId);}}else if(s.quality?.taskId.startsWith("tinymmlu")&&s.quality.outputs){let{outputs:l,details:c,score:d,n:m}=s.quality;if(l.length!==m||!c||c.length!==m)o("quality-outputs-mismatch","quality outputs/details length disagrees with n","reject",s.cellId);else {let f=0,u=true;for(let y=0;y<m;y++){let p=H[y];if(!p)break;let h=J(l[y],p.choices)===p.answer?1:0;f+=h,h!==c[y]&&(u=false);}(!u||Math.abs(f/m-d)>1e-9)&&o("quality-details-mismatch",`submitted MMLU score ${d} disagrees with recompute ${f/m} from raw outputs`,"reject",s.cellId);}}else if(s.iterations.length>0){let l=Z(s);l.batchTextsPerSec&&l.batchTextsPerSec.median>Ct&&o("embed-rate-envelope",`embedding throughput ${Math.round(l.batchTextsPerSec.median)} texts/s exceeds envelope`,"reject",s.cellId),l.singleLatencyMs&&l.singleLatencyMs.median<.05&&o("embed-latency-implausible","sub-50us embedding latency","reject",s.cellId);}}let r=e.environment.gpu.vendor?.toLowerCase()??"",a=e.environment.os.platform.toLowerCase();r==="apple"&&a.includes("windows")&&o("env-cross-field","Apple GPU adapter with Windows platform","reject");let i=[e.environment.gpu.vendor,e.environment.gpu.architecture,e.environment.gpu.device,e.environment.gpu.description,e.environment.webglRenderer].filter(Boolean).join(" ");if(Lt.some(s=>s.test(i))){let s=e.cells.some(l=>l.status==="ok"&&/webgpu|^gpu$/.test(l.resolvedBackend));o("software-renderer",`GPU identity matches a software/virtual renderer: "${i.slice(0,120)}"`,s?"reject":"warn");}return t}function qt(e){let t=Oe(e);if(t.length>0)return {ok:false,shapeErrors:t,flags:[],summaries:[]};let o=N(e),n=Ue(e);if(e.clientSummaries){let r=new Map(o.map(a=>[a.cellId,a]));for(let a of e.clientSummaries){let i=r.get(a.cellId);if(i)for(let s of ["ttftMs","decodeCharsPerSec","totalMs","overallCharsPerSec","singleLatencyMs","batchTextsPerSec"]){let l=a[s]?.median,c=i[s]?.median;l!==void 0&&c!==void 0&&c!==0&&Math.abs(l-c)/Math.abs(c)>.01&&n.push({code:"client-summary-disagrees",cellId:a.cellId,message:`${s} client median ${l} vs recomputed ${c}`,severity:"reject"});}}}return {ok:n.every(r=>r.severity!=="reject"),shapeErrors:[],flags:n,summaries:o}}async function jt(e){let{policy:t,hooks:o,abortSignal:n}=e,r=e.trace??new $;await r.attach(),r.record("suite-start",e.suite);let a=K(),i=[];try{o?.onPhase?.("environment");let s=await ce({userReportedDevice:e.userReportedDevice});o?.onEnvironment?.(s),o?.onPhase?.("fingerprint"),P(n);let l=e.skipFingerprint?null:await de(),c=a!=="none"?await W():null,d=Wt(ie(e.cells)),m=e.cells.length,f=0;for(let y of d){P(n);let p=await Nt(y,{...e,trace:r,memApi:a,baselineMemory:c,cellIndexRef:()=>f++,totalCells:m});i.push(...p),p.some(h=>h.status!=="skipped")&&(r.record("cooldown-start"),await F(t.cooldownMs,n).catch(h=>{if(n?.aborted)throw h}),t.pressureGate&&await Ut(r,t.pressureGateTimeoutMs,n),r.record("cooldown-end"));}r.record("suite-end");let u={protocol:j,schemaVersion:3,runId:Vt(),createdAt:new Date().toISOString(),harness:e.harness,suite:e.suite,environment:s,fingerprint:l,cells:i,events:[...r.all]};return u.clientSummaries=N(u,t.highVarianceCv),u}finally{await r.dispose();}}function Wt(e){let t=new Map;for(let o of e){let n=`${o.model.runtimeId}/${o.model.benchModelId}/${o.model.providerModelId}`,r=t.get(n);r||(r={key:n,model:o.model,workloads:[],skipped:[]},t.set(n,r)),o.skipReason?r.skipped.push({workload:o.workload,reason:o.skipReason}):r.workloads.push(o.workload);}return [...t.values()]}async function Nt(e,t){let{model:o}=e,n=o.task==="embedding",r=n?t.embedAdapters.get(o.runtimeId):t.llmAdapters.get(o.runtimeId),a=[],i=p=>({cellId:`${o.runtimeId}/${o.benchModelId}/${p.id}`,runtimeId:o.runtimeId,runtimeVersion:r?.runtimeVersion,model:o,workloadId:p.id,workloadKind:p.kind,resolvedBackend:"unknown",load:null,iterations:[],status:"skipped"});for(let{workload:p,reason:h}of e.skipped){let b=i(p);b.invalidReasons=[h],a.push(O(b,t));}if(e.workloads.length===0)return a;if(!r){for(let p of e.workloads){let h=i(p);h.invalidReasons=[`no adapter registered for ${o.runtimeId}`],a.push(O(h,t));}return a}let s=await r.isAvailable();if(!s.ok){for(let p of e.workloads){let h=i(p);h.invalidReasons=[`runtime unavailable: ${s.reason??"unknown"}`],a.push(O(h,t));}return a}let l=null,c=null,d,m=null,f=[],u=`${o.runtimeId}/${o.benchModelId}/${e.workloads[0].id}`;for(let p=1;p<=Math.max(1,t.policy.maxAttempts);p++)try{P(t.abortSignal);let h=await r.isModelCached(o),b=[],S=w();t.hooks?.onActivity?.({cellId:u,phase:"load"}),l=await B((v,g)=>$e(r.load(o,{abortSignal:v,onProgress:k=>{g(),typeof k.pct=="number"&&(b.length===0||k.pct-b[b.length-1].pct>=2)&&b.push({t:w(),pct:Math.round(k.pct*100)/100}),t.hooks?.onLoadProgress?.(u,k.pct),t.hooks?.onActivity?.({cellId:u,phase:"load",pct:k.pct});}}),v),{parent:t.abortSignal,stallMs:t.policy.loadStallMs,timeoutMs:t.policy.loadTimeoutMs,what:"load"});let M=w();if(c={cached:h,startT:S,endT:M,progress:b.length>0?Ft(b,50):void 0,declaredBytes:o.sizeBytes},t.memApi!=="none"&&(m=await W()),t.policy.warmupRuns>0){let v=w();for(let g=0;g<t.policy.warmupRuns;g++){P(t.abortSignal);let k=l;t.hooks?.onActivity?.({cellId:u,phase:"warmup",iteration:g+1,total:t.policy.warmupRuns,chars:0,chunks:0}),await B((A,E)=>n?k.model.doEmbed({values:["warmup probe"],abortSignal:A}):Ge(k.model,{prompt:"Reply with the single word: ready",maxTokens:4,temperature:0},A,(Ze,et)=>{E(),t.hooks?.onActivity?.({cellId:u,phase:"warmup",chars:Ze,chunks:et});}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.iterationTimeoutMs,what:"warmup"}),t.hooks?.onActivity?.({cellId:u,phase:"warmup",iteration:g+1,total:t.policy.warmupRuns});}d=w()-v;}break}catch(h){if(t.abortSignal?.aborted)throw l&&await D(l),T();l&&await D(l),l=null;let b=Fe(h);if(b.name==="TimeoutError"&&t.trace.record("cell-timeout",`${u}: ${b.message}`),p<Math.max(1,t.policy.maxAttempts)){f.push({error:b,at:w()}),t.trace.record("cell-retry",`${u} attempt ${p+1}`),t.hooks?.onCellRetry?.(u,p+1,b);continue}let S=t.memApi!=="none"?await W(5e3):null;for(let M of e.workloads){let v=i(M);v.status="error",v.load=c,v.error=b,f.length>0&&(v.attempts=[...f]),t.memApi!=="none"&&(v.memory={api:t.memApi,baseline:t.baselineMemory??void 0,postLoad:m??void 0,atError:S??void 0}),a.push(O(v,t));}return a}if(!l)return a;let y=false;for(let p of e.workloads){P(t.abortSignal);let h=i(p);h.resolvedBackend=l.resolvedBackend,l.runtimeConfig&&(h.runtimeConfig={...l.runtimeConfig}),y||(h.load=c,h.warmupMs=d,f.length>0&&(h.attempts=[...f]),y=true);let b=t.cellIndexRef();t.hooks?.onCellStart?.(h.cellId,b,t.totalCells);let S=l,M=async()=>{switch(p.kind){case "llm-generate":await Ot(h,S,p,t);break;case "embed-single":case "embed-batch":await Dt(h,S,p,t);break;case "quality-mmlu":{let g=p.items;h.quality=await B((k,A)=>pe(S.model,g,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onIteration?.(h.cellId,E,g),t.hooks?.onActivity?.({cellId:h.cellId,phase:"quality",iteration:E,total:g});},promptSuffix:o.qualityPromptSuffix}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.qualityTimeoutMs,what:"quality lane"}),h.status="ok";break}case "quality-sts":{let g=p.items;h.quality=await B((k,A)=>fe(S.model,g,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onIteration?.(h.cellId,E,g),t.hooks?.onActivity?.({cellId:h.cellId,phase:"quality",iteration:E,total:g});}}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.qualityTimeoutMs,what:"quality lane"}),h.status="ok";break}}},v=Math.max(1,t.policy.maxAttempts);for(let g=1;g<=v;g++)try{await M();break}catch(k){if(t.abortSignal?.aborted)throw await D(l),T();let A=Fe(k);if(A.name==="TimeoutError"&&t.trace.record("cell-timeout",`${h.cellId}: ${A.message}`),g<v){h.attempts=[...h.attempts??[],{error:A,at:w()}],h.iterations=[],h.quality=void 0,h.invalidReasons=void 0,h.status="skipped",t.trace.record("cell-retry",`${h.cellId} attempt ${g+1}`),t.hooks?.onCellRetry?.(h.cellId,g+1,A);continue}h.status="error",h.error=A;}if(t.memApi!=="none"&&(h.status==="ok"||h.status==="error")){let g=await W(5e3);h.memory={api:t.memApi,baseline:t.baselineMemory??void 0,postLoad:m??void 0,...h.status==="ok"?{postRun:g??void 0}:{atError:g??void 0}};}a.push(O(h,t));}if(await D(l),t.policy.measureWarmReload&&c?.cached===false)try{P(t.abortSignal);let p=`${o.runtimeId}/${o.benchModelId}/warm-reload`;t.hooks?.onActivity?.({cellId:p,phase:"reload"});let h=w(),b=await B((k,A)=>$e(r.load(o,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onActivity?.({cellId:p,phase:"reload",pct:E.pct});}}),k),{parent:t.abortSignal,stallMs:t.policy.loadStallMs,timeoutMs:t.policy.loadTimeoutMs,what:"load"}),S=w(),M=b.resolvedBackend,v=b.runtimeConfig?{...b.runtimeConfig}:void 0;await D(b);let g={cellId:`${o.runtimeId}/${o.benchModelId}/warm-reload`,runtimeId:o.runtimeId,runtimeVersion:r.runtimeVersion,model:o,workloadId:"warm-reload",workloadKind:"llm-generate",resolvedBackend:M,...v?{runtimeConfig:v}:{},load:{cached:!0,startT:h,endT:S,declaredBytes:o.sizeBytes},iterations:[],status:"ok"};a.push(O(g,t));}catch{if(t.abortSignal?.aborted)throw T()}return a}async function Ot(e,t,o,n){let r=[],a=[];for(let i=0;i<n.policy.timedRuns;i++){let s=i+1;for(let l=0;;l++){P(n.abortSignal),n.hooks?.onIteration?.(e.cellId,s,n.policy.timedRuns),await ze(n,e.cellId);let c=[];n.trace.isHidden&&c.push("started-hidden");let d=ne(n.trace);n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:s,total:n.policy.timedRuns,chars:0,chunks:0});let m=w(),{chunks:f,text:u,providerUsage:y,finishReason:p}=await B((S,M)=>Ge(t.model,{prompt:o.prompt,systemPrompt:o.systemPrompt,maxTokens:o.maxTokens,temperature:o.temperature},S,(v,g)=>{M(),n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:s,total:n.policy.timedRuns,chars:v,chunks:g});}),{parent:n.abortSignal,stallMs:n.policy.chunkStallMs,timeoutMs:n.policy.iterationTimeoutMs,what:`iteration ${s}`}),h=w();ne(n.trace)>d&&c.push("hidden-during-run"),u.length<L&&c.push("degenerate-output");let b={startT:m,chunks:f,endT:h,text:u,providerUsage:y?{...y,fidelity:se[e.runtimeId]}:void 0,finishReason:p,gates:c};if(Ve(c)&&l<Math.max(1,n.policy.maxAttempts)){a.push(b),n.trace.record("iteration-redo",`${e.cellId} iteration ${s}: ${c.join(",")}`);continue}r.push(b);break}}e.iterations=r,a.length>0&&(e.discardedIterations=a),He(e,r);}function Ve(e){return e.includes("started-hidden")||e.includes("hidden-during-run")}function He(e,t){let o=t.some(n=>n.gates.length>0);if(e.status=o?"invalid":"ok",o){let n=new Set;for(let r of t)for(let a of r.gates)n.add(a==="degenerate-output"?`degenerate output: fewer than ${L} generated chars in a timed iteration`:a==="started-hidden"||a==="hidden-during-run"?"tab hidden during a timed iteration (the tab did not return in time for a repeat)":"validity gate fired during a timed region");e.invalidReasons=[...n];}}async function ze(e,t){if(!e.trace.isHidden)return true;e.hooks?.onActivity?.({cellId:t,phase:"waiting-visible"});let o=w()+e.policy.visibilityWaitMs;for(;e.trace.isHidden&&w()<o;)P(e.abortSignal),await F(Math.min(250,e.policy.visibilityWaitMs),e.abortSignal);return !e.trace.isHidden}async function Dt(e,t,o,n){let r=[],a=[];for(let i=0;i<n.policy.timedRuns;i++)for(let s=0;;s++){P(n.abortSignal),n.hooks?.onIteration?.(e.cellId,i+1,n.policy.timedRuns),await ze(n,e.cellId);let l=[];n.trace.isHidden&&l.push("started-hidden");let c=ne(n.trace),d=w(),{embeddings:m}=await B(y=>t.model.doEmbed({values:o.texts,abortSignal:y}),{parent:n.abortSignal,stallMs:n.policy.iterationTimeoutMs,timeoutMs:n.policy.iterationTimeoutMs,what:`iteration ${i+1}`}),f=w();ne(n.trace)>c&&l.push("hidden-during-run"),n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:i+1,total:n.policy.timedRuns});let u={startT:d,endT:f,count:o.texts.length,dimensions:m[0]?.length??t.model.dimensions,gates:l};if(Ve(l)&&s<Math.max(1,n.policy.maxAttempts)){a.push(u),n.trace.record("iteration-redo",`${e.cellId} iteration ${i+1}: ${l.join(",")}`);continue}r.push(u);break}e.iterations=r,a.length>0&&(e.discardedIterations=a),He(e,r);}async function Ge(e,t,o,n){let r=[],a="",i,s;if(e.doStream)for await(let l of e.doStream({...t,abortSignal:o})){let c=w();l.text.length>0&&(r.push({t:c,c:l.text.length}),a+=l.text,n?.(a.length,r.length)),l.done&&(i=l.usage,s=l.finishReason);}else {let l=await e.doGenerate({...t,abortSignal:o}),c=w();a=l.text,r.push({t:c,c:l.text.length}),n?.(a.length,r.length),i=l.usage,s=l.finishReason;}return {chunks:r,text:a,providerUsage:i,finishReason:s}}var te=class extends Error{constructor(t){super(t),this.name="TimeoutError";}};async function B(e,t){let o=new AbortController,n=()=>{},r=new Promise((u,y)=>{n=y;}),a=()=>{o.abort(t.parent?.reason),n(T());};t.parent?.aborted&&a(),t.parent?.addEventListener("abort",a,{once:true});let i=w(),s=i,l=false,c=u=>{l||(l=true,o.abort(new te(u)),n(new te(u)));},m=setInterval(()=>{if(l)return;let u=w();if(u-s>=t.stallMs){c(t.what==="load"?`load made no progress for ${t.stallMs} ms`:`${t.what}: no stream progress for ${t.stallMs} ms`);return}u-i>=t.timeoutMs&&c(`${t.what}: exceeded the ${t.timeoutMs} ms budget`);},Math.max(10,Math.min(t.stallMs,t.timeoutMs)/4)),f=()=>{s=w();};try{let u=e(o.signal,f);return u.catch(()=>{}),await Promise.race([u,r])}finally{l=true,clearInterval(m),t.parent?.removeEventListener("abort",a);}}function ne(e){let t=0;for(let o of e.all)o.type==="visibility-hidden"&&t++;return t}async function Ut(e,t,o){let n=w()+t;for(;w()<n;){let r=e.pressureState;if(r===void 0||r==="nominal"||r==="fair")return;await F(1e3,o);}}function Ft(e,t){if(e.length<=t)return e;let o=[],n=(e.length-1)/(t-1);for(let r=0;r<t;r++)o.push(e[Math.round(r*n)]);return o}var $t=4e3;function Fe(e){let t=e,o=t?.cause,n=o instanceof Error?o.message:typeof o=="string"?o:void 0,r=o instanceof Error&&o.name&&o.name!=="Error"?o.name:void 0,a=o instanceof Error&&typeof o.stack=="string"&&o.stack.length>0?o.stack.slice(0,$t):void 0;return {name:t?.name??"Error",message:t?.message??String(e),...n!==void 0?{cause:n}:{},...r?{causeName:r}:{},...a?{causeStack:a}:{}}}function O(e,t){return t.hooks?.onCellFinish?.(e),e}async function $e(e,t){let o=await e;if(t.aborted)throw await D(o),T();return o}async function D(e){try{await e.dispose();}catch{}}function P(e){if(e?.aborted)throw T()}function Vt(){let e=globalThis.crypto;if(e?.randomUUID)return e.randomUUID();let t=new Uint8Array(16);if(e?.getRandomValues)e.getRandomValues(t);else for(let n=0;n<16;n++)t[n]=Math.floor(Math.random()*256);t[6]=t[6]&15|64,t[8]=t[8]&63|128;let o=[...t].map(n=>n.toString(16).padStart(2,"0")).join("");return `${o.slice(0,8)}-${o.slice(8,12)}-${o.slice(12,16)}-${o.slice(16,20)}-${o.slice(20)}`}function be(e){return ge(e)}function ge(e){if(e===null||typeof e=="number"||typeof e=="boolean"||typeof e=="string")return JSON.stringify(e);if(typeof e>"u")return "null";if(Array.isArray(e))return `[${e.map(t=>ge(t)).join(",")}]`;if(typeof e=="object")return `{${Object.entries(e).filter(([,o])=>o!==void 0).sort(([o],[n])=>o<n?-1:o>n?1:0).map(([o,n])=>`${JSON.stringify(o)}:${ge(n)}`).join(",")}}`;throw new TypeError(`canonicalJson: unsupported value type ${typeof e}`)}async function ye(e){let t=globalThis.crypto?.subtle;if(!t)throw new Error("Web Crypto (crypto.subtle) is not available in this environment");let o=new TextEncoder().encode(e),n=await t.digest("SHA-256",o);return [...new Uint8Array(n)].map(r=>r.toString(16).padStart(2,"0")).join("")}async function Ke(e){let{digest:t,nonce:o,...n}=e;return ye(be(n))}async function Ye(e){let{digest:t,...o}=e;return ye(be(o))}async function Ht(e){return e.digest?await Ke(e)===e.digest?true:e.nonce!==void 0?await Ye(e)===e.digest:false:false}function zt(e){let t=[],o={...e,environment:{...e.environment}};"nonce"in o&&(delete o.nonce,t.push("nonce"));let n=o.environment;if(n.locale&&typeof n.locale=="object"){let a={...n.locale};for(let i of ["timeZone","timeZoneOffsetMinutes","calendar"])i in a&&(delete a[i],t.push(`environment.locale.${i}`));n.locale=a;}if("languages"in n&&(delete n.languages,t.push("environment.languages")),n.power&&typeof n.power=="object"){let a={...n.power};for(let i of ["chargingTimeSec","dischargingTimeSec"])i in a&&(delete a[i],t.push(`environment.power.${i}`));if(typeof a.level=="number"){let i=Y(a.level);i!==a.level&&(a.level=i,t.push("environment.power.level (coarsened)"));}n.power=a;}if(n.display&&typeof n.display=="object"){let a={...n.display};for(let i of ["prefersReducedMotion","prefersColorScheme"])i in a&&(delete a[i],t.push(`environment.display.${i}`));n.display=a;}let r=t.some(a=>a!=="nonce");return r&&o.schemaVersion<3&&(o.schemaVersion=3,t.push("schemaVersion (raised)")),{run:o,changed:r,removed:t}}function oe(e){let t=e.environment,o=t.gpu.available?[t.gpu.vendor??"gpu",t.gpu.architecture].filter(Boolean).join("-"):"no-webgpu";return [t.os.platform.toLowerCase().replace(/\s+/g,"-"),o].join("/")}function ve(e,t){let o=t?.trim();if(!o||e.endsWith("/no-webgpu")||/or similar/i.test(o)||!/\d/.test(o))return e;let n=o.toLowerCase().replace(/\((tm|r|c)\)/g,"").replace(/^mesa\s+/,"").replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"");return n?`${e.split("/")[0]}/${n}`:e}function Xe(e){return ve(oe(e),e.environment.gpuModel)}var Je=3;function Gt(e,t=Je){let o=new Map;for(let r of e){let a=r.protocol,i=oe(r),s=ve(i,r.environment.gpuModel),l=r.clientSummaries??N(r),c=new Map(l.map(d=>[d.cellId,d]));for(let d of r.cells){if(d.status!=="ok")continue;let m=c.get(d.cellId);if(!m)continue;let f=[a,s,d.runtimeId,d.model.benchModelId,d.workloadId].join("|"),u=o.get(f);u||(u={row:{protocol:a,deviceClass:i,deviceSubclass:s,runtimeId:d.runtimeId,benchModelId:d.model.benchModelId,modelName:d.model.displayName,workloadId:d.workloadId,resolvedBackends:[],browsers:[],highVariance:false},ttft:[],decode:[],overall:[],single:[],batch:[],loadCold:[],loadWarm:[],quality:[],parseRate:[],runIds:new Set},o.set(f,u)),u.runIds.add(r.runId),Qe(u.row.resolvedBackends,d.resolvedBackend),Qe(u.row.browsers,r.environment.browser.name),u.row.highVariance||=m.highVariance,m.ttftMs&&u.ttft.push(m.ttftMs.median),m.decodeCharsPerSec&&u.decode.push(m.decodeCharsPerSec.median),m.overallCharsPerSec&&u.overall.push(m.overallCharsPerSec.median),m.singleLatencyMs&&u.single.push(m.singleLatencyMs.median),m.batchTextsPerSec&&u.batch.push(m.batchTextsPerSec.median),m.loadMs!==void 0&&(m.loadCached?u.loadWarm:u.loadCold).push(m.loadMs),m.qualityScore!==void 0&&u.quality.push(m.qualityScore),m.qualityParseRate!==void 0&&u.parseRate.push(m.qualityParseRate);}}let n=[];for(let r of o.values())n.push({...r.row,submissions:r.runIds.size,ttftMs:C(r.ttft),decodeCharsPerSec:C(r.decode),overallCharsPerSec:C(r.overall),singleLatencyMs:C(r.single),batchTextsPerSec:C(r.batch),loadColdMs:C(r.loadCold),loadWarmMs:C(r.loadWarm),qualityScore:C(r.quality),qualityParseRate:C(r.parseRate),provisional:r.runIds.size<t});return n.sort((r,a)=>a.protocol.localeCompare(r.protocol,void 0,{numeric:true})||r.deviceClass.localeCompare(a.deviceClass)||r.deviceSubclass.localeCompare(a.deviceSubclass)||r.benchModelId.localeCompare(a.benchModelId)||r.runtimeId.localeCompare(a.runtimeId)||r.workloadId.localeCompare(a.workloadId)),n}function Qe(e,t){t&&!e.includes(t)&&e.push(t);}function C(e){return e.length>0?q(X(e)):void 0}function q(e){return Math.round(e*100)/100}function we(e){let t=e==null?"":String(e);return /[",\n]/.test(t)?`"${t.replace(/"/g,'""')}"`:t}function Kt(e){let o=[["protocol","deviceClass","deviceSubclass","runtimeId","benchModelId","modelName","workloadId","submissions","ttftMs","decodeCharsPerSec","overallCharsPerSec","singleLatencyMs","batchTextsPerSec","loadColdMs","loadWarmMs","qualityScore","qualityParseRate","resolvedBackends","browsers","highVariance","provisional"].join(",")];for(let n of e)o.push([n.protocol,n.deviceClass,n.deviceSubclass,n.runtimeId,n.benchModelId,n.modelName,n.workloadId,n.submissions,n.ttftMs,n.decodeCharsPerSec,n.overallCharsPerSec,n.singleLatencyMs,n.batchTextsPerSec,n.loadColdMs,n.loadWarmMs,n.qualityScore,n.qualityParseRate,n.resolvedBackends.join(";"),n.browsers.join(";"),n.highVariance,n.provisional].map(we).join(","));return o.join(`
|
|
34
|
+
Answer:`}function xt(e){let t=e.replace(/<think>[\s\S]*?<\/think>/gi," "),o=t.search(/<think>/i);return o===-1?t:t.slice(0,o)}function J(e,t){let o=xt(e).trim(),n=o.replace(/[*`_]/g,"").trim(),r=n.match(/(?:^|(?:[Aa]nswer|ANSWER)\s*(?:is)?\s*[:-]?\s*|(?:[Oo]ption|OPTION|[Cc]hoice|CHOICE)\s*[:-]?\s*)\(?([ABCD])\)?(?=[\s.,)!]|$)/);if(r)return he.indexOf(r[1]);let a=n.match(/^\(?([abcd])(?:\)|\.|$)/);if(a)return he.indexOf(a[1].toUpperCase());let i=o.toLowerCase();for(let s=0;s<t.length;s++)if(i.startsWith(t[s].toLowerCase().slice(0,Math.max(8,t[s].length))))return s;return null}async function pe(e,t,o){let n=H.slice(0,t),r=[],a=[],i=0,s=0;for(let l=0;l<n.length;l++){if(o?.abortSignal?.aborted)throw T();let c=n[l],m=(await e.doGenerate({prompt:We(c,o?.promptSuffix),maxTokens:qe,temperature:0,abortSignal:o?.abortSignal})).text.slice(0,je),f=J(m,c.choices);f!==null&&s++;let u=f===c.answer?1:0;i+=u,r.push(u),a.push(m),o?.onProgress?.(l+1,n.length);}return {taskId:`tinymmlu-${t}`,score:n.length===0?0:i/n.length,n:n.length,details:r,outputs:a,parseRate:n.length===0?0:s/n.length}}function _t(e,t){let o=0,n=0,r=0;for(let i=0;i<e.length;i++)o+=e[i]*t[i],n+=e[i]*e[i],r+=t[i]*t[i];let a=Math.sqrt(n)*Math.sqrt(r);return a===0?0:o/a}async function fe(e,t,o){let n=me.slice(0,t),r=[],a=[];for(let i=0;i<n.length;i++){if(o?.abortSignal?.aborted)throw T();let s=n[i],{embeddings:l}=await e.doEmbed({values:[s.s1,s.s2],abortSignal:o?.abortSignal});r.push(_t(l[0],l[1])),a.push(s.score),o?.onProgress?.(i+1,n.length);}return {taskId:`stsb-${t}`,score:r.length>=2?ue(r,a):0,n:n.length,details:r.map(i=>Math.round(i*1e4)/1e4)}}var Et=2,Oe=.2,Pt=[{maxBytes:5e8,max:4e3},{maxBytes:1e9,max:2500},{maxBytes:2e9,max:1500},{maxBytes:Number.POSITIVE_INFINITY,max:1e3}],Ct=2e3,Lt=[/swiftshader/i,/llvmpipe/i,/softpipe/i,/\bwarp\b/i,/microsoft basic render/i,/parallels/i,/vmware/i,/virtio/i,/virtualbox/i];function Ne(e,t={}){let o=[],n=s=>{o.length<50&&o.push(s);};if(typeof e!="object"||e===null)return ["result must be an object"];let r=e;t.anyProtocol?(typeof r.protocol!="string"||!/^localmode-bench\/\d+$/.test(r.protocol))&&n('protocol must be a "localmode-bench/<n>" identifier'):r.protocol!==j&&n(`protocol must be "${j}"`),z.includes(r.schemaVersion)||n(`schemaVersion must be one of ${z.join(", ")}`),(typeof r.runId!="string"||r.runId.length<8||r.runId.length>64)&&n("runId must be a string (8-64 chars)"),(typeof r.createdAt!="string"||Number.isNaN(Date.parse(r.createdAt)))&&n("createdAt must be an ISO date string");let a=r.harness;(!a||typeof a.name!="string"||typeof a.version!="string")&&n("harness {name, version} is required"),["quick","standard","thorough","custom"].includes(r.suite)||n("suite must be quick|standard|thorough|custom");let i=r.environment;if(!i||typeof i!="object")n("environment is required");else {let s=i.browser;(!s||typeof s.name!="string")&&n("environment.browser.name is required"),(!i.os||typeof i.os.platform!="string")&&n("environment.os.platform is required");}return Array.isArray(r.cells)?r.cells.length===0?n("cells must not be empty"):r.cells.length>500?n("cells exceeds the 500-cell limit"):r.cells.forEach((s,l)=>Bt(s,l,n)):n("cells must be an array"),Array.isArray(r.events)||n("events must be an array"),o}function Bt(e,t,o){if(typeof e!="object"||e===null){o(`cells[${t}] must be an object`);return}let n=e,r=`cells[${t}]`;typeof n.cellId!="string"&&o(`${r}.cellId must be a string`),typeof n.runtimeId!="string"&&o(`${r}.runtimeId must be a string`),typeof n.workloadId!="string"&&o(`${r}.workloadId must be a string`),["ok","invalid","error","skipped"].includes(n.status)||o(`${r}.status must be ok|invalid|error|skipped`);let a=n.model;(!a||typeof a.benchModelId!="string"||typeof a.providerModelId!="string")&&o(`${r}.model {benchModelId, providerModelId} is required`);let i=n.quality;if(i&&Array.isArray(i.outputs)&&i.outputs.some(s=>typeof s!="string"||s.length>500)&&o(`${r}.quality.outputs must be strings of at most 500 chars`),!Array.isArray(n.iterations)){o(`${r}.iterations must be an array`);return}n.iterations.forEach((s,l)=>{if(typeof s!="object"||s===null){o(`${r}.iterations[${l}] must be an object`);return}let c=s;(typeof c.startT!="number"||typeof c.endT!="number")&&o(`${r}.iterations[${l}] needs numeric startT/endT`),"chunks"in c&&!Array.isArray(c.chunks)&&o(`${r}.iterations[${l}].chunks must be an array`),"text"in c&&typeof c.text=="string"&&c.text.length>2e4&&o(`${r}.iterations[${l}].text exceeds 20000 chars`);});}function De(e){return e.length>0&&"chunks"in e[0]}function ee(e){let t=e.chunks.filter(r=>r.c>0);if(t.length<2)return false;let o=t[t.length-1].t-t[0].t,n=e.endT-e.startT;return n>0&&o/n>=Oe}function Z(e,t=.05,o){let n={cellId:e.cellId,status:e.status,highVariance:false};if(e.load&&(n.loadMs=e.load.endT-e.load.startT,n.loadCached=e.load.cached),e.quality&&(n.qualityScore=e.quality.score,e.quality.parseRate!==void 0&&(n.qualityParseRate=e.quality.parseRate)),e.iterations.length===0)return n;if(De(e.iterations)){let a=e.iterations.every(ee);n.streamIncremental=a;let i=[],s=[],l=[],c=[],d=[],m=[],f=[];for(let u of e.iterations){let y=u.chunks.reduce((g,k)=>g+k.c,0),p=u.endT-u.startT;if(p>0&&(m.push(p),y>0&&f.push(y/p*1e3)),c.push(y),!a)continue;let h=u.chunks.find(g=>g.c>0);if(!h)continue;let b=h.t-u.startT;i.push(b);let S=u.chunks[u.chunks.length-1],M=y-h.c,v=S.t-h.t;v>0&&M>0&&(s.push(M/v*1e3),l.push((u.chunks.length-1)/v*1e3)),o&&b>0&&d.push(o/b*1e3);}i.length>0&&(n.ttftMs=_(i)),s.length>0&&(n.decodeCharsPerSec=_(s)),l.length>0&&(n.decodeChunksPerSec=_(l)),c.length>0&&(n.generatedChars=_(c)),d.length>0&&(n.prefillTokPerSecApprox=_(d)),m.length>0&&(n.totalMs=_(m)),f.length>0&&(n.overallCharsPerSec=_(f));}else {let a=e.iterations,i=a.map(s=>s.endT-s.startT).filter(s=>s>0);i.length>0&&(a[0].count===1?n.singleLatencyMs=_(i):n.batchTextsPerSec=_(a.filter(s=>s.endT-s.startT>0).map(s=>s.count/(s.endT-s.startT)*1e3)));}let r=[n.ttftMs,n.decodeCharsPerSec,n.overallCharsPerSec,n.singleLatencyMs,n.batchTextsPerSec];return n.highVariance=r.some(a=>a!==void 0&&a.n>=2&&a.cv>t),n}function O(e,t=.05,o){let r=o??new Map([["chat-pp128-tg128",128],["chat-pp512-tg128",512]]);return e.cells.map(a=>Z(a,t,r.get(a.workloadId)))}function Ue(e){let t=[],o=(s,l,c,d)=>t.push({code:s,message:l,severity:c,...d?{cellId:d}:{}});e.fingerprint?(e.fingerprint.mflops<=1||e.fingerprint.mflops>1e5||!Number.isFinite(e.fingerprint.checksum))&&o("fingerprint-implausible",`fingerprint mflops=${e.fingerprint.mflops}`,"reject"):o("fingerprint-missing","no hardware fingerprint in submission","reject");let n=e.environment.timerResolutionUs;for(let s of e.cells)if(s.status==="ok"){if(s.load&&s.load.endT<s.load.startT&&o("load-time-reversed","load endT precedes startT","reject",s.cellId),De(s.iterations)){for(let m of s.iterations){let f=m.startT,u=true;for(let p of m.chunks)p.t<f&&(u=false),f=p.t;if(!u||m.endT<f){o("trace-not-monotonic","chunk timestamps are not monotonic","reject",s.cellId);break}let y=m.chunks.reduce((p,h)=>p+h.c,0);if(m.text.length!==y){o("text-chunk-mismatch",`text length ${m.text.length} != chunk sum ${y}`,"reject",s.cellId);break}}let l=Z(s);s.iterations.some(m=>m.text.length<L)&&o("degenerate-generation",`an ok cell has a timed iteration with fewer than ${L} generated chars`,"reject",s.cellId);let c=s.model.sizeBytes??Number.POSITIVE_INFINITY,d=Pt.find(m=>c<=m.maxBytes);if(d&&l.decodeCharsPerSec&&l.decodeCharsPerSec.median>d.max&&o("decode-rate-envelope",`decode ${Math.round(l.decodeCharsPerSec.median)} chars/s exceeds envelope ${d.max} for model size`,"reject",s.cellId),d&&l.overallCharsPerSec&&l.overallCharsPerSec.median>d.max&&o("overall-rate-envelope",`overall ${Math.round(l.overallCharsPerSec.median)} chars/s exceeds envelope ${d.max} for model size`,"reject",s.cellId),l.ttftMs&&l.ttftMs.median<1&&o("ttft-implausible",`median TTFT ${l.ttftMs.median}ms < 1ms`,"reject",s.cellId),n!=null&&n>=4){let m=n/1e3,f=0,u=0;for(let y of s.iterations)for(let p of y.chunks)u++,Math.abs(p.t/m-Math.round(p.t/m))>.01&&f++;u>=20&&f/u>.5&&o("timer-grid",`${f}/${u} chunk timestamps off the ${n}us timer grid`,"warn",s.cellId);}}else if(s.quality?.taskId.startsWith("tinymmlu")&&s.quality.outputs){let{outputs:l,details:c,score:d,n:m}=s.quality;if(l.length!==m||!c||c.length!==m)o("quality-outputs-mismatch","quality outputs/details length disagrees with n","reject",s.cellId);else {let f=0,u=true;for(let y=0;y<m;y++){let p=H[y];if(!p)break;let h=J(l[y],p.choices)===p.answer?1:0;f+=h,h!==c[y]&&(u=false);}(!u||Math.abs(f/m-d)>1e-9)&&o("quality-details-mismatch",`submitted MMLU score ${d} disagrees with recompute ${f/m} from raw outputs`,"reject",s.cellId);}}else if(s.iterations.length>0){let l=Z(s);l.batchTextsPerSec&&l.batchTextsPerSec.median>Ct&&o("embed-rate-envelope",`embedding throughput ${Math.round(l.batchTextsPerSec.median)} texts/s exceeds envelope`,"reject",s.cellId),l.singleLatencyMs&&l.singleLatencyMs.median<.05&&o("embed-latency-implausible","sub-50us embedding latency","reject",s.cellId);}}let r=e.environment.gpu.vendor?.toLowerCase()??"",a=e.environment.os.platform.toLowerCase();r==="apple"&&a.includes("windows")&&o("env-cross-field","Apple GPU adapter with Windows platform","reject");let i=[e.environment.gpu.vendor,e.environment.gpu.architecture,e.environment.gpu.device,e.environment.gpu.description,e.environment.webglRenderer].filter(Boolean).join(" ");if(Lt.some(s=>s.test(i))){let s=e.cells.some(l=>l.status==="ok"&&/webgpu|^gpu$/.test(l.resolvedBackend));o("software-renderer",`GPU identity matches a software/virtual renderer: "${i.slice(0,120)}"`,s?"reject":"warn");}return t}function qt(e,t={}){let o=Ne(e,t);if(o.length>0)return {ok:false,shapeErrors:o,flags:[],summaries:[]};let n=O(e),r=Ue(e);if(e.clientSummaries){let a=new Map(n.map(i=>[i.cellId,i]));for(let i of e.clientSummaries){let s=a.get(i.cellId);if(s)for(let l of ["ttftMs","decodeCharsPerSec","totalMs","overallCharsPerSec","singleLatencyMs","batchTextsPerSec"]){let c=i[l]?.median,d=s[l]?.median;c!==void 0&&d!==void 0&&d!==0&&Math.abs(c-d)/Math.abs(d)>.01&&r.push({code:"client-summary-disagrees",cellId:i.cellId,message:`${l} client median ${c} vs recomputed ${d}`,severity:"reject"});}}}return {ok:r.every(a=>a.severity!=="reject"),shapeErrors:[],flags:r,summaries:n}}async function jt(e){let{policy:t,hooks:o,abortSignal:n}=e,r=e.trace??new $;await r.attach(),r.record("suite-start",e.suite);let a=K(),i=[];try{o?.onPhase?.("environment");let s=await ce({userReportedDevice:e.userReportedDevice});o?.onEnvironment?.(s),o?.onPhase?.("fingerprint"),P(n);let l=e.skipFingerprint?null:await de(),c=a!=="none"?await W():null,d=Wt(ie(e.cells)),m=e.cells.length,f=0;for(let y of d){P(n);let p=await Ot(y,{...e,trace:r,memApi:a,baselineMemory:c,cellIndexRef:()=>f++,totalCells:m});i.push(...p),p.some(h=>h.status!=="skipped")&&(r.record("cooldown-start"),await F(t.cooldownMs,n).catch(h=>{if(n?.aborted)throw h}),t.pressureGate&&await Ut(r,t.pressureGateTimeoutMs,n),r.record("cooldown-end"));}r.record("suite-end");let u={protocol:j,schemaVersion:3,runId:Vt(),createdAt:new Date().toISOString(),harness:e.harness,suite:e.suite,environment:s,fingerprint:l,cells:i,events:[...r.all]};return u.clientSummaries=O(u,t.highVarianceCv),u}finally{await r.dispose();}}function Wt(e){let t=new Map;for(let o of e){let n=`${o.model.runtimeId}/${o.model.benchModelId}/${o.model.providerModelId}`,r=t.get(n);r||(r={key:n,model:o.model,workloads:[],skipped:[]},t.set(n,r)),o.skipReason?r.skipped.push({workload:o.workload,reason:o.skipReason}):r.workloads.push(o.workload);}return [...t.values()]}async function Ot(e,t){let{model:o}=e,n=o.task==="embedding",r=n?t.embedAdapters.get(o.runtimeId):t.llmAdapters.get(o.runtimeId),a=[],i=p=>({cellId:`${o.runtimeId}/${o.benchModelId}/${p.id}`,runtimeId:o.runtimeId,runtimeVersion:r?.runtimeVersion,model:o,workloadId:p.id,workloadKind:p.kind,resolvedBackend:"unknown",load:null,iterations:[],status:"skipped"});for(let{workload:p,reason:h}of e.skipped){let b=i(p);b.invalidReasons=[h],a.push(N(b,t));}if(e.workloads.length===0)return a;if(!r){for(let p of e.workloads){let h=i(p);h.invalidReasons=[`no adapter registered for ${o.runtimeId}`],a.push(N(h,t));}return a}let s=await r.isAvailable();if(!s.ok){for(let p of e.workloads){let h=i(p);h.invalidReasons=[`runtime unavailable: ${s.reason??"unknown"}`],a.push(N(h,t));}return a}let l=null,c=null,d,m=null,f=[],u=`${o.runtimeId}/${o.benchModelId}/${e.workloads[0].id}`;for(let p=1;p<=Math.max(1,t.policy.maxAttempts);p++)try{P(t.abortSignal);let h=await r.isModelCached(o),b=[],S=w();t.hooks?.onActivity?.({cellId:u,phase:"load"}),l=await B((v,g)=>$e(r.load(o,{abortSignal:v,onProgress:k=>{g(),typeof k.pct=="number"&&(b.length===0||k.pct-b[b.length-1].pct>=2)&&b.push({t:w(),pct:Math.round(k.pct*100)/100}),t.hooks?.onLoadProgress?.(u,k.pct),t.hooks?.onActivity?.({cellId:u,phase:"load",pct:k.pct});}}),v),{parent:t.abortSignal,stallMs:t.policy.loadStallMs,timeoutMs:t.policy.loadTimeoutMs,what:"load"});let M=w();if(c={cached:h,startT:S,endT:M,progress:b.length>0?Ft(b,50):void 0,declaredBytes:o.sizeBytes},t.memApi!=="none"&&(m=await W()),t.policy.warmupRuns>0){let v=w();for(let g=0;g<t.policy.warmupRuns;g++){P(t.abortSignal);let k=l;t.hooks?.onActivity?.({cellId:u,phase:"warmup",iteration:g+1,total:t.policy.warmupRuns,chars:0,chunks:0}),await B((A,E)=>n?k.model.doEmbed({values:["warmup probe"],abortSignal:A}):Ge(k.model,{prompt:"Reply with the single word: ready",maxTokens:4,temperature:0},A,(Ze,et)=>{E(),t.hooks?.onActivity?.({cellId:u,phase:"warmup",chars:Ze,chunks:et});}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.iterationTimeoutMs,what:"warmup"}),t.hooks?.onActivity?.({cellId:u,phase:"warmup",iteration:g+1,total:t.policy.warmupRuns});}d=w()-v;}break}catch(h){if(t.abortSignal?.aborted)throw l&&await D(l),T();l&&await D(l),l=null;let b=Fe(h);if(b.name==="TimeoutError"&&t.trace.record("cell-timeout",`${u}: ${b.message}`),p<Math.max(1,t.policy.maxAttempts)){f.push({error:b,at:w()}),t.trace.record("cell-retry",`${u} attempt ${p+1}`),t.hooks?.onCellRetry?.(u,p+1,b);continue}let S=t.memApi!=="none"?await W(5e3):null;for(let M of e.workloads){let v=i(M);v.status="error",v.load=c,v.error=b,f.length>0&&(v.attempts=[...f]),t.memApi!=="none"&&(v.memory={api:t.memApi,baseline:t.baselineMemory??void 0,postLoad:m??void 0,atError:S??void 0}),a.push(N(v,t));}return a}if(!l)return a;let y=false;for(let p of e.workloads){P(t.abortSignal);let h=i(p);h.resolvedBackend=l.resolvedBackend,l.runtimeConfig&&(h.runtimeConfig={...l.runtimeConfig}),y||(h.load=c,h.warmupMs=d,f.length>0&&(h.attempts=[...f]),y=true);let b=t.cellIndexRef();t.hooks?.onCellStart?.(h.cellId,b,t.totalCells);let S=l,M=async()=>{switch(p.kind){case "llm-generate":await Nt(h,S,p,t);break;case "embed-single":case "embed-batch":await Dt(h,S,p,t);break;case "quality-mmlu":{let g=p.items;h.quality=await B((k,A)=>pe(S.model,g,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onIteration?.(h.cellId,E,g),t.hooks?.onActivity?.({cellId:h.cellId,phase:"quality",iteration:E,total:g});},promptSuffix:o.qualityPromptSuffix}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.qualityTimeoutMs,what:"quality lane"}),h.status="ok";break}case "quality-sts":{let g=p.items;h.quality=await B((k,A)=>fe(S.model,g,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onIteration?.(h.cellId,E,g),t.hooks?.onActivity?.({cellId:h.cellId,phase:"quality",iteration:E,total:g});}}),{parent:t.abortSignal,stallMs:t.policy.chunkStallMs,timeoutMs:t.policy.qualityTimeoutMs,what:"quality lane"}),h.status="ok";break}}},v=Math.max(1,t.policy.maxAttempts);for(let g=1;g<=v;g++)try{await M();break}catch(k){if(t.abortSignal?.aborted)throw await D(l),T();let A=Fe(k);if(A.name==="TimeoutError"&&t.trace.record("cell-timeout",`${h.cellId}: ${A.message}`),g<v){h.attempts=[...h.attempts??[],{error:A,at:w()}],h.iterations=[],h.quality=void 0,h.invalidReasons=void 0,h.status="skipped",t.trace.record("cell-retry",`${h.cellId} attempt ${g+1}`),t.hooks?.onCellRetry?.(h.cellId,g+1,A);continue}h.status="error",h.error=A;}if(t.memApi!=="none"&&(h.status==="ok"||h.status==="error")){let g=await W(5e3);h.memory={api:t.memApi,baseline:t.baselineMemory??void 0,postLoad:m??void 0,...h.status==="ok"?{postRun:g??void 0}:{atError:g??void 0}};}a.push(N(h,t));}if(await D(l),t.policy.measureWarmReload&&c?.cached===false)try{P(t.abortSignal);let p=`${o.runtimeId}/${o.benchModelId}/warm-reload`;t.hooks?.onActivity?.({cellId:p,phase:"reload"});let h=w(),b=await B((k,A)=>$e(r.load(o,{abortSignal:k,onProgress:E=>{A(),t.hooks?.onActivity?.({cellId:p,phase:"reload",pct:E.pct});}}),k),{parent:t.abortSignal,stallMs:t.policy.loadStallMs,timeoutMs:t.policy.loadTimeoutMs,what:"load"}),S=w(),M=b.resolvedBackend,v=b.runtimeConfig?{...b.runtimeConfig}:void 0;await D(b);let g={cellId:`${o.runtimeId}/${o.benchModelId}/warm-reload`,runtimeId:o.runtimeId,runtimeVersion:r.runtimeVersion,model:o,workloadId:"warm-reload",workloadKind:"llm-generate",resolvedBackend:M,...v?{runtimeConfig:v}:{},load:{cached:!0,startT:h,endT:S,declaredBytes:o.sizeBytes},iterations:[],status:"ok"};a.push(N(g,t));}catch{if(t.abortSignal?.aborted)throw T()}return a}async function Nt(e,t,o,n){let r=[],a=[];for(let i=0;i<n.policy.timedRuns;i++){let s=i+1;for(let l=0;;l++){P(n.abortSignal),n.hooks?.onIteration?.(e.cellId,s,n.policy.timedRuns),await ze(n,e.cellId);let c=[];n.trace.isHidden&&c.push("started-hidden");let d=ne(n.trace);n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:s,total:n.policy.timedRuns,chars:0,chunks:0});let m=w(),{chunks:f,text:u,providerUsage:y,finishReason:p}=await B((S,M)=>Ge(t.model,{prompt:o.prompt,systemPrompt:o.systemPrompt,maxTokens:o.maxTokens,temperature:o.temperature},S,(v,g)=>{M(),n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:s,total:n.policy.timedRuns,chars:v,chunks:g});}),{parent:n.abortSignal,stallMs:n.policy.chunkStallMs,timeoutMs:n.policy.iterationTimeoutMs,what:`iteration ${s}`}),h=w();ne(n.trace)>d&&c.push("hidden-during-run"),u.length<L&&c.push("degenerate-output");let b={startT:m,chunks:f,endT:h,text:u,providerUsage:y?{...y,fidelity:se[e.runtimeId]}:void 0,finishReason:p,gates:c};if(Ve(c)&&l<Math.max(1,n.policy.maxAttempts)){a.push(b),n.trace.record("iteration-redo",`${e.cellId} iteration ${s}: ${c.join(",")}`);continue}r.push(b);break}}e.iterations=r,a.length>0&&(e.discardedIterations=a),He(e,r);}function Ve(e){return e.includes("started-hidden")||e.includes("hidden-during-run")}function He(e,t){let o=t.some(n=>n.gates.length>0);if(e.status=o?"invalid":"ok",o){let n=new Set;for(let r of t)for(let a of r.gates)n.add(a==="degenerate-output"?`degenerate output: fewer than ${L} generated chars in a timed iteration`:a==="started-hidden"||a==="hidden-during-run"?"tab hidden during a timed iteration (the tab did not return in time for a repeat)":"validity gate fired during a timed region");e.invalidReasons=[...n];}}async function ze(e,t){if(!e.trace.isHidden)return true;e.hooks?.onActivity?.({cellId:t,phase:"waiting-visible"});let o=w()+e.policy.visibilityWaitMs;for(;e.trace.isHidden&&w()<o;)P(e.abortSignal),await F(Math.min(250,e.policy.visibilityWaitMs),e.abortSignal);return !e.trace.isHidden}async function Dt(e,t,o,n){let r=[],a=[];for(let i=0;i<n.policy.timedRuns;i++)for(let s=0;;s++){P(n.abortSignal),n.hooks?.onIteration?.(e.cellId,i+1,n.policy.timedRuns),await ze(n,e.cellId);let l=[];n.trace.isHidden&&l.push("started-hidden");let c=ne(n.trace),d=w(),{embeddings:m}=await B(y=>t.model.doEmbed({values:o.texts,abortSignal:y}),{parent:n.abortSignal,stallMs:n.policy.iterationTimeoutMs,timeoutMs:n.policy.iterationTimeoutMs,what:`iteration ${i+1}`}),f=w();ne(n.trace)>c&&l.push("hidden-during-run"),n.hooks?.onActivity?.({cellId:e.cellId,phase:"iteration",iteration:i+1,total:n.policy.timedRuns});let u={startT:d,endT:f,count:o.texts.length,dimensions:m[0]?.length??t.model.dimensions,gates:l};if(Ve(l)&&s<Math.max(1,n.policy.maxAttempts)){a.push(u),n.trace.record("iteration-redo",`${e.cellId} iteration ${i+1}: ${l.join(",")}`);continue}r.push(u);break}e.iterations=r,a.length>0&&(e.discardedIterations=a),He(e,r);}async function Ge(e,t,o,n){let r=[],a="",i,s;if(e.doStream)for await(let l of e.doStream({...t,abortSignal:o})){let c=w();l.text.length>0&&(r.push({t:c,c:l.text.length}),a+=l.text,n?.(a.length,r.length)),l.done&&(i=l.usage,s=l.finishReason);}else {let l=await e.doGenerate({...t,abortSignal:o}),c=w();a=l.text,r.push({t:c,c:l.text.length}),n?.(a.length,r.length),i=l.usage,s=l.finishReason;}return {chunks:r,text:a,providerUsage:i,finishReason:s}}var te=class extends Error{constructor(t){super(t),this.name="TimeoutError";}};async function B(e,t){let o=new AbortController,n=()=>{},r=new Promise((u,y)=>{n=y;}),a=()=>{o.abort(t.parent?.reason),n(T());};t.parent?.aborted&&a(),t.parent?.addEventListener("abort",a,{once:true});let i=w(),s=i,l=false,c=u=>{l||(l=true,o.abort(new te(u)),n(new te(u)));},m=setInterval(()=>{if(l)return;let u=w();if(u-s>=t.stallMs){c(t.what==="load"?`load made no progress for ${t.stallMs} ms`:`${t.what}: no stream progress for ${t.stallMs} ms`);return}u-i>=t.timeoutMs&&c(`${t.what}: exceeded the ${t.timeoutMs} ms budget`);},Math.max(10,Math.min(t.stallMs,t.timeoutMs)/4)),f=()=>{s=w();};try{let u=e(o.signal,f);return u.catch(()=>{}),await Promise.race([u,r])}finally{l=true,clearInterval(m),t.parent?.removeEventListener("abort",a);}}function ne(e){let t=0;for(let o of e.all)o.type==="visibility-hidden"&&t++;return t}async function Ut(e,t,o){let n=w()+t;for(;w()<n;){let r=e.pressureState;if(r===void 0||r==="nominal"||r==="fair")return;await F(1e3,o);}}function Ft(e,t){if(e.length<=t)return e;let o=[],n=(e.length-1)/(t-1);for(let r=0;r<t;r++)o.push(e[Math.round(r*n)]);return o}var $t=4e3;function Fe(e){let t=e,o=t?.cause,n=o instanceof Error?o.message:typeof o=="string"?o:void 0,r=o instanceof Error&&o.name&&o.name!=="Error"?o.name:void 0,a=o instanceof Error&&typeof o.stack=="string"&&o.stack.length>0?o.stack.slice(0,$t):void 0;return {name:t?.name??"Error",message:t?.message??String(e),...n!==void 0?{cause:n}:{},...r?{causeName:r}:{},...a?{causeStack:a}:{}}}function N(e,t){return t.hooks?.onCellFinish?.(e),e}async function $e(e,t){let o=await e;if(t.aborted)throw await D(o),T();return o}async function D(e){try{await e.dispose();}catch{}}function P(e){if(e?.aborted)throw T()}function Vt(){let e=globalThis.crypto;if(e?.randomUUID)return e.randomUUID();let t=new Uint8Array(16);if(e?.getRandomValues)e.getRandomValues(t);else for(let n=0;n<16;n++)t[n]=Math.floor(Math.random()*256);t[6]=t[6]&15|64,t[8]=t[8]&63|128;let o=[...t].map(n=>n.toString(16).padStart(2,"0")).join("");return `${o.slice(0,8)}-${o.slice(8,12)}-${o.slice(12,16)}-${o.slice(16,20)}-${o.slice(20)}`}function be(e){return ge(e)}function ge(e){if(e===null||typeof e=="number"||typeof e=="boolean"||typeof e=="string")return JSON.stringify(e);if(typeof e>"u")return "null";if(Array.isArray(e))return `[${e.map(t=>ge(t)).join(",")}]`;if(typeof e=="object")return `{${Object.entries(e).filter(([,o])=>o!==void 0).sort(([o],[n])=>o<n?-1:o>n?1:0).map(([o,n])=>`${JSON.stringify(o)}:${ge(n)}`).join(",")}}`;throw new TypeError(`canonicalJson: unsupported value type ${typeof e}`)}async function ye(e){let t=globalThis.crypto?.subtle;if(!t)throw new Error("Web Crypto (crypto.subtle) is not available in this environment");let o=new TextEncoder().encode(e),n=await t.digest("SHA-256",o);return [...new Uint8Array(n)].map(r=>r.toString(16).padStart(2,"0")).join("")}async function Ke(e){let{digest:t,nonce:o,...n}=e;return ye(be(n))}async function Ye(e){let{digest:t,...o}=e;return ye(be(o))}async function Ht(e){return e.digest?await Ke(e)===e.digest?true:e.nonce!==void 0?await Ye(e)===e.digest:false:false}function zt(e){let t=[],o={...e,environment:{...e.environment}};"nonce"in o&&(delete o.nonce,t.push("nonce"));let n=o.environment;if(n.locale&&typeof n.locale=="object"){let a={...n.locale};for(let i of ["timeZone","timeZoneOffsetMinutes","calendar"])i in a&&(delete a[i],t.push(`environment.locale.${i}`));n.locale=a;}if("languages"in n&&(delete n.languages,t.push("environment.languages")),n.power&&typeof n.power=="object"){let a={...n.power};for(let i of ["chargingTimeSec","dischargingTimeSec"])i in a&&(delete a[i],t.push(`environment.power.${i}`));if(typeof a.level=="number"){let i=Y(a.level);i!==a.level&&(a.level=i,t.push("environment.power.level (coarsened)"));}n.power=a;}if(n.display&&typeof n.display=="object"){let a={...n.display};for(let i of ["prefersReducedMotion","prefersColorScheme"])i in a&&(delete a[i],t.push(`environment.display.${i}`));n.display=a;}let r=t.some(a=>a!=="nonce");return r&&o.schemaVersion<3&&(o.schemaVersion=3,t.push("schemaVersion (raised)")),{run:o,changed:r,removed:t}}function oe(e){let t=e.environment,o=t.gpu.available?[t.gpu.vendor??"gpu",t.gpu.architecture].filter(Boolean).join("-"):"no-webgpu";return [t.os.platform.toLowerCase().replace(/\s+/g,"-"),o].join("/")}function ve(e,t){let o=t?.trim();if(!o||e.endsWith("/no-webgpu")||/or similar/i.test(o)||!/\d/.test(o))return e;let n=o.toLowerCase().replace(/\((tm|r|c)\)/g,"").replace(/^mesa\s+/,"").replace(/[^a-z0-9]+/g,"-").replace(/^-+|-+$/g,"");return n?`${e.split("/")[0]}/${n}`:e}function Xe(e){return ve(oe(e),e.environment.gpuModel)}var Je=3;function Gt(e,t=Je){let o=new Map;for(let r of e){let a=r.protocol,i=oe(r),s=ve(i,r.environment.gpuModel),l=r.clientSummaries??O(r),c=new Map(l.map(d=>[d.cellId,d]));for(let d of r.cells){if(d.status!=="ok")continue;let m=c.get(d.cellId);if(!m)continue;let f=[a,s,d.runtimeId,d.model.benchModelId,d.workloadId].join("|"),u=o.get(f);u||(u={row:{protocol:a,deviceClass:i,deviceSubclass:s,runtimeId:d.runtimeId,benchModelId:d.model.benchModelId,modelName:d.model.displayName,workloadId:d.workloadId,resolvedBackends:[],browsers:[],highVariance:false},ttft:[],decode:[],overall:[],single:[],batch:[],loadCold:[],loadWarm:[],quality:[],parseRate:[],runIds:new Set},o.set(f,u)),u.runIds.add(r.runId),Qe(u.row.resolvedBackends,d.resolvedBackend),Qe(u.row.browsers,r.environment.browser.name),u.row.highVariance||=m.highVariance,m.ttftMs&&u.ttft.push(m.ttftMs.median),m.decodeCharsPerSec&&u.decode.push(m.decodeCharsPerSec.median),m.overallCharsPerSec&&u.overall.push(m.overallCharsPerSec.median),m.singleLatencyMs&&u.single.push(m.singleLatencyMs.median),m.batchTextsPerSec&&u.batch.push(m.batchTextsPerSec.median),m.loadMs!==void 0&&(m.loadCached?u.loadWarm:u.loadCold).push(m.loadMs),m.qualityScore!==void 0&&u.quality.push(m.qualityScore),m.qualityParseRate!==void 0&&u.parseRate.push(m.qualityParseRate);}}let n=[];for(let r of o.values())n.push({...r.row,submissions:r.runIds.size,ttftMs:C(r.ttft),decodeCharsPerSec:C(r.decode),overallCharsPerSec:C(r.overall),singleLatencyMs:C(r.single),batchTextsPerSec:C(r.batch),loadColdMs:C(r.loadCold),loadWarmMs:C(r.loadWarm),qualityScore:C(r.quality),qualityParseRate:C(r.parseRate),provisional:r.runIds.size<t});return n.sort((r,a)=>a.protocol.localeCompare(r.protocol,void 0,{numeric:true})||r.deviceClass.localeCompare(a.deviceClass)||r.deviceSubclass.localeCompare(a.deviceSubclass)||r.benchModelId.localeCompare(a.benchModelId)||r.runtimeId.localeCompare(a.runtimeId)||r.workloadId.localeCompare(a.workloadId)),n}function Qe(e,t){t&&!e.includes(t)&&e.push(t);}function C(e){return e.length>0?q(X(e)):void 0}function q(e){return Math.round(e*100)/100}function we(e){let t=e==null?"":String(e);return /[",\n]/.test(t)?`"${t.replace(/"/g,'""')}"`:t}function Kt(e){let o=[["protocol","deviceClass","deviceSubclass","runtimeId","benchModelId","modelName","workloadId","submissions","ttftMs","decodeCharsPerSec","overallCharsPerSec","singleLatencyMs","batchTextsPerSec","loadColdMs","loadWarmMs","qualityScore","qualityParseRate","resolvedBackends","browsers","highVariance","provisional"].join(",")];for(let n of e)o.push([n.protocol,n.deviceClass,n.deviceSubclass,n.runtimeId,n.benchModelId,n.modelName,n.workloadId,n.submissions,n.ttftMs,n.decodeCharsPerSec,n.overallCharsPerSec,n.singleLatencyMs,n.batchTextsPerSec,n.loadColdMs,n.loadWarmMs,n.qualityScore,n.qualityParseRate,n.resolvedBackends.join(";"),n.browsers.join(";"),n.highVariance,n.provisional].map(we).join(","));return o.join(`
|
|
35
35
|
`)+`
|
|
36
36
|
`}function Yt(e){let o=[["runId","createdAt","protocol","suite","deviceClass","deviceSubclass","browser","browserVersion","os","gpuVendor","gpuArchitecture","cores","deviceMemoryGB","crossOriginIsolated","fingerprintMflops","runtimeId","runtimeVersion","benchModelId","providerModelId","quantization","sizeBytes","workloadId","resolvedBackend","iteration","ttftMs","decodeCharsPerSec","generatedChars","overallCharsPerSec","streamIncremental","durationMs","loadMs","loadCached","status"].join(",")];for(let n of e){let r=n.environment,a=[n.runId,n.createdAt,n.protocol,n.suite,oe(n),Xe(n),r.browser.name,r.browser.version,r.os.platform,r.gpu.vendor??"",r.gpu.architecture??"",r.hardware.cores??"",r.hardware.deviceMemoryGB??"",r.flags.crossOriginIsolated,n.fingerprint?q(n.fingerprint.mflops):""];for(let i of n.cells){let s=i.load?q(i.load.endT-i.load.startT):"",l=i.load?String(i.load.cached):"",c=[i.runtimeId,i.runtimeVersion??"",i.model.benchModelId,i.model.providerModelId,i.model.quantization??"",i.model.sizeBytes??"",i.workloadId,i.resolvedBackend];if(i.iterations.length===0){o.push([...a,...c,"","","","","","","",s,l,i.status].map(we).join(","));continue}i.iterations.forEach((d,m)=>{let f="",u="",y="",p="",h="";if("chunks"in d){let b=d.chunks.reduce((g,k)=>g+k.c,0);y=String(b);let S=d.endT-d.startT;S>0&&b>0&&(p=String(q(b/S*1e3)));let M=ee(d);h=String(M);let v=d.chunks.find(g=>g.c>0);if(M&&v){f=String(q(v.t-d.startT));let g=d.chunks[d.chunks.length-1],k=b-v.c,A=g.t-v.t;A>0&&(u=String(q(k/A*1e3)));}}o.push([...a,...c,m+1,f,u,y,p,h,q(d.endT-d.startT),s,l,i.status].map(we).join(","));});}}return o.join(`
|
|
37
37
|
`)+`
|
|
38
|
-
`}exports.ACCEPTED_SCHEMA_VERSIONS=z;exports.BENCH_PROTOCOL_VERSION=j;exports.BENCH_SCHEMA_VERSION=U;exports.EMBED_WORKLOADS=Ae;exports.GENERATION_BUDGET=ae;exports.HEADLINE_MIN_SUBMISSIONS=Je;exports.LEADERBOARD_PROTOCOL_VERSIONS=tt;exports.LLM_WORKLOADS=Se;exports.MIN_GENERATED_CHARS=L;exports.MMLU_MAX_TOKENS=qe;exports.MMLU_OUTPUT_CAP=je;exports.PLAUSIBILITY_RULES_VERSION=Et;exports.QUALITY_WORKLOADS=Ie;exports.RUNTIME_EXECUTION_ORDER=G;exports.RUN_POLICIES=it;exports.STREAM_COHERENCE_MIN_SPAN_RATIO=
|
|
38
|
+
`}exports.ACCEPTED_SCHEMA_VERSIONS=z;exports.BENCH_PROTOCOL_VERSION=j;exports.BENCH_SCHEMA_VERSION=U;exports.EMBED_WORKLOADS=Ae;exports.GENERATION_BUDGET=ae;exports.HEADLINE_MIN_SUBMISSIONS=Je;exports.LEADERBOARD_PROTOCOL_VERSIONS=tt;exports.LLM_WORKLOADS=Se;exports.MIN_GENERATED_CHARS=L;exports.MMLU_MAX_TOKENS=qe;exports.MMLU_OUTPUT_CAP=je;exports.PLAUSIBILITY_RULES_VERSION=Et;exports.QUALITY_WORKLOADS=Ie;exports.RUNTIME_EXECUTION_ORDER=G;exports.RUN_POLICIES=it;exports.STREAM_COHERENCE_MIN_SPAN_RATIO=Oe;exports.STSB_SUBSET=me;exports.TINY_MMLU=H;exports.TraceRecorder=$;exports.USAGE_FIDELITY=se;exports.WORKLOADS_BY_ID=at;exports.aggregateRuns=Gt;exports.canonicalJson=be;exports.captureEnvironment=ce;exports.checkPlausibility=Ue;exports.coarseBatteryLevel=Y;exports.computeLegacyRunDigest=Ye;exports.computeRunDigest=Ke;exports.deriveDeviceType=Te;exports.detectEngine=Re;exports.detectWasmFeatures=Ee;exports.deviceClassOf=oe;exports.deviceSubclassOf=Xe;exports.formatMMLUPrompt=We;exports.geomean=Rt;exports.hrNow=w;exports.inferTimerResolutionUs=le;exports.isIncrementalStream=ee;exports.mean=V;exports.median=X;exports.memoryApiAvailable=K;exports.orderCells=ie;exports.parseGpuModel=_e;exports.parseMMLUAnswer=J;exports.parseUserAgent=Me;exports.quantile=Q;exports.refineDeviceClass=ve;exports.resolveGpuModel=xe;exports.rowsToCSV=Kt;exports.runBenchmarkSuite=jt;exports.runFingerprint=de;exports.runMMLUFidelity=pe;exports.runSTSQuality=fe;exports.runsToLongCSV=Yt;exports.sampleMemoryBytes=W;exports.scrubRunForPublication=zt;exports.sha256Hex=ye;exports.sleep=F;exports.spearman=ue;exports.stddev=Be;exports.summarize=_;exports.summarizeCell=Z;exports.summarizeRun=O;exports.validateRunShape=Ne;exports.validateSubmission=qt;exports.verifyRunDigest=Ht;//# sourceMappingURL=index.cjs.map
|
|
39
39
|
//# sourceMappingURL=index.cjs.map
|