@slexisvn/mlfw 0.1.4 → 0.1.5

package/dist/index.browser.js

@@ -100,7 +100,7 @@ fn rnn(@builtin(local_invocation_id) lid : vec3<u32>) {
 `)}printFunction(e,t=null){let n=t===null;n&&(t=[]),this.valueNames.clear(),this._nextValueId=0;let s=e.args.map((i,a)=>`${this._nameValue(i)}: ${Rn(i.type)}`),o=e.outputTypes.map(i=>Rn(i)).join(", ");t.push(`${this._indentPrefix()}func @${e.name}(${s.join(", ")}) -> (${o}) {`),this.indent++;for(let i of e.body)this.printBlock(i,t,i===e.entryBlock);return this.indent--,t.push(`${this._indentPrefix()}}`),n?t.join(`
 `):void 0}printBlock(e,t,n=!1){if(!n&&e.arguments.length>0){let s=e.arguments.map(o=>`${this._nameValue(o)}: ${Rn(o.type)}`);t.push(`${this._indentPrefix()}^bb(${s.join(", ")}):`)}for(let s of e)this.printOperation(s,t)}printOperation(e,t=null){let n=t===null;n&&(t=[]);let s=this._indentPrefix(),o=s;if(e.numResults>0){let i=[];for(let a=0;a<e.numResults;a++)i.push(this._nameValue(e.getResult(a)));o+=i.join(", ")+" = "}if(o+=e.opName,e.numOperands>0){let i=[];for(let a=0;a<e.numOperands;a++)i.push(this._valueName(e.getOperand(a)));o+="("+i.join(", ")+")"}else o+="()";if(e.attributes.size>0){let i=[];for(let[a,u]of CP(e.attributes))i.push(`${a} = ${wg(u)}`);o+=" {"+i.join(", ")+"}"}if(e.numResults>0){let i=[];for(let a=0;a<e.numResults;a++)i.push(Rn(e.getResult(a).type));o+=" : "+i.join(", ")}t.push(o);for(let i=0;i<e.regions.length;i++){let a=e.regions[i];t.push(`${s}{`),this.indent++;for(let u of a){if(u.arguments.length>0){let p=u.arguments.map(c=>`${this._nameValue(c)}: ${Rn(c.type)}`);t.push(`${this._indentPrefix()}^bb(${p.join(", ")}):`)}for(let p of u)this.printOperation(p,t)}this.indent--,t.push(`${s}}`)}return n?t.join(`
 `):void 0}_nameValue(e){if(this.valueNames.has(e))return this.valueNames.get(e);let t=`%${this._nextValueId++}`;return this.valueNames.set(e,t),t}_valueName(e){return this.valueNames.has(e)?this.valueNames.get(e):"%?"}_indentPrefix(){let e="";for(let t=0;t<this.indent;t++)e+=this.indentStr;return e}};l(CP,"sortedEntries");l(wg,"formatAttrValue");l(DP,"printModule")});var t$,Mg,vO=M(()=>{"use strict";Oe();Vs();t$={eq:l((r,e)=>r===e,"eq"),ne:l((r,e)=>r!==e,"ne"),gt:l((r,e)=>r>e,"gt"),ge:l((r,e)=>r>=e,"ge"),lt:l((r,e)=>r<e,"lt"),le:l((r,e)=>r<=e,"le")},Mg=class{static{l(this,"ShapeEnv")}_symbols;_guards;_bindings;_nextId;constructor(){this._symbols=new Map,this._guards=[],this._bindings=new Map,this._nextId=0}allocate(e,t,n){let s=`s${this._nextId++}`;return this._symbols.set(s,{hint:n,inputIdx:e,dimIdx:t}),s}produceShapeSpec(e,t,n){let s=new Array(t.length),o=new Array(t.length);for(let i=0;i<t.length;i++)if(n&&n.has(i)){let a=this.allocate(e,i,t[i]);s[i]=X,o[i]=a}else{let a=this.allocate(e,i,t[i]);this.guardRelation(a,"eq",t[i]),s[i]=t[i],o[i]=t[i]}return{irShape:s,symShape:o}}guardRelation(e,t,n){this._guards.push({lhs:e,op:t,rhs:n})}guardDivisible(e,t){this._guards.push({type:"divisible",sym:e,divisor:t})}bindInputShapes(e){this._bindings.clear();for(let[t,n]of this._symbols)this._bindings.set(t,e[n.inputIdx].shape[n.dimIdx])}evaluateGuards(){for(let e=0;e<this._guards.length;e++){let t=this._guards[e];if("type"in t){if(this._resolve(t.sym)%t.divisor!==0)return{passed:!1,failedGuard:t};continue}let n=this._resolve(t.lhs),s=this._resolve(t.rhs);if(!t$[t.op](n,s))return{passed:!1,failedGuard:t}}return{passed:!0,failedGuard:null}}resolveSymbolicShape(e){let t=new Array(e.length);for(let n=0;n<e.length;n++)t[n]=this._resolve(e[n]);return t}_resolve(e){return typeof e=="number"?e:typeof e=="string"?this._bindings.get(e):e instanceof Ce?Ce.evaluate(e,this._bindings):e}get symbols(){return this._symbols}get guards(){return this._guards}get bindings(){return this._bindings}}});function ss(){return Ig}var Ig,Eg,ya=M(()=>{"use strict";es();Yo();xs();Oe();At();ga();vO();l0();Ig=null;l(ss,"getActiveTracer");Eg=class{static{l(this,"Tracer")}_name;_shapeEnv;_inputTypes;_inputSymShapes;_outputTypes;_outputSymShapes;_inputs;_func;_builder;_module;_capturedParams;_capturedParamOrder;constructor(e){this._name=e||"traced",this._shapeEnv=new Mg,this._inputTypes=[],this._inputSymShapes=[],this._outputTypes=[],this._outputSymShapes=[],this._inputs=[],this._func=null,this._builder=null,this._module=null,this._capturedParams=new Map,this._capturedParamOrder=[]}get shapeEnv(){return this._shapeEnv}createInput(e,t,n){let s=this._inputTypes.length,{irShape:o,symShape:i}=this._shapeEnv.produceShapeSpec(s,e,n);if(n)for(let u=0;u<i.length;u++)typeof i[u]=="string"&&this._shapeEnv.guardRelation(i[u],"gt",0);let a=new I(o,t);return this._inputTypes.push(a),this._inputSymShapes.push(i),{shape:o,dtype:t,tensorType:a}}_initGraph(){this._func=new un(this._name,this._inputTypes,[]),this._func.inputTypes=[...this._func.inputTypes],this._builder=new An(this._func),this._module=new wn(this._name);let e=[],t=this._func.args;for(let n=0;n<t.length;n++){let s=t[n],o=this._inputTypes[n];s.symbolicShape=this._inputSymShapes[n];let i=new ft(s,o.shape,o.dtype,this,this._inputSymShapes[n]);e.push(i)}return this._inputs=e,e}recordOp(e,t,n){let s=[];for(let u of t)u instanceof ft&&s.push(u.irValue);let o=this._requireBuilder(),i=bm(o,e,s,n),a=[];for(let u=0;u<i.numResults;u++){let p=i.getResult(u),c=p.type,h=this._propagateSymbolicShape(e,i,t,c,u);p.symbolicShape=h,a.push(new ft(p,c.shape,c.dtype,this,h))}return a.length===1?a[0]:a}_propagateSymbolicShape(e,t,n,s,o=0){let i=n.filter(c=>c instanceof ft),a=ne.get(t.opName||e);if(a&&a.propagateSymbolicShapes){let c=new Map;for(let f of i)c.set(f.irValue,f.symbolicShape);let h=a.propagateSymbolicShapes(t,c);if(h&&h[o])return h[o]}let u=s.shape,p=new Array(u.length);for(let c=0;c<u.length;c++){if(u[c]!==X){p[c]=u[c];continue}let h=null;for(let f of i){let d=f.symbolicShape;if(!d)continue;let m=u.length-d.length,g=c-m;if(g>=0&&g<d.length&&typeof d[g]=="string"){h=d[g];break}}p[c]=h!==null?h:X}return p}scan(e,t,n){let s=l(f=>f instanceof ft?f.irValue:this.captureConstant(f).irValue,"toIr"),o=e.map(s),i=t.map(s),u=this._requireBuilder().scanOp(o,i,(f,d,m)=>{let g=this._requireBuilder();this._builder=f;try{let y=l(w=>new ft(w,w.type.shape,w.type.dtype,this,[...w.type.shape]),"wrap"),[_,b]=n(m.map(y),d.map(y));return[_.map(w=>w.irValue),b.map(w=>w.irValue)]}finally{this._builder=g}}),p=t.length,c=[],h=[];for(let f=0;f<u.numResults;f++){let d=u.getResult(f),m=new ft(d,d.type.shape,d.type.dtype,this,[...d.type.shape]);f<p?c.push(m):h.push(m)}return[c,h]}captureConstant(e){let t=this._capturedParams.get(e);if(t)return t;if(e.shape.length===0&&e.data){let u=e.data[0],c=this._requireBuilder().scalarConstant(u,e.dtype).getResult(0),h=new ft(c,[],e.dtype,this,[]);return this._capturedParams.set(e,h),h}let n=new I(e.shape,e.dtype),s=this._requireFunc();s.inputTypes.push(n);let i=s.entryBlock.addArgument(n),a=new ft(i,e.shape,e.dtype,this,[...e.shape]);return this._capturedParams.set(e,a),this._capturedParamOrder.push(e),a}get capturedParams(){return this._capturedParamOrder}markOutput(e){e instanceof ft&&(this._requireBuilder().returnOp([e.irValue]),this._outputSymShapes=[e.symbolicShape]),this._outputTypes=[new I(e.shape,e.dtype)]}markOutputs(e){let t=e.map(n=>n.irValue);this._requireBuilder().returnOp(t),this._outputTypes=e.map(n=>new I(n.shape,n.dtype)),this._outputSymShapes=e.map(n=>n.symbolicShape)}get outputSymShapes(){return this._outputSymShapes}getGraphModule(){let e=this._requireFunc(),t=this._requireModule();return e.outputTypes=Object.freeze(this._outputTypes),Object.isFrozen(e.inputTypes)||(e.inputTypes=Object.freeze(e.inputTypes)),t.addFunction(e),t}activate(){Ig=this}deactivate(){Ig===this&&(Ig=null)}_requireBuilder(){if(!this._builder)throw new Error("Tracer graph has not been initialized");return this._builder}_requireFunc(){if(!this._func)throw new Error("Tracer graph has not been initialized");return this._func}_requireModule(){if(!this._module)throw new Error("Tracer graph has not been initialized");return this._module}}});function FO(){return[...$O]}function Vg(r,e,t,n=$O){return n.filter(s=>s.phase===r).sort((s,o)=>s.priority-o.priority).map(s=>s.factory(e,t)).filter(Boolean)}var $O,uT=M(()=>{"use strict";$O=[];l(FO,"snapshotGraphPasses");l(Vg,"graphPassesForPhase")});function zO(r){return r instanceof Map?r:r&&typeof r=="object"?new Map(Object.entries(r)):new Map}var Gg,VO=M(()=>{"use strict";uT();l(zO,"toMap");Gg=class{static{l(this,"CompilerContext")}constructor({loweringRules:e=null,codegenEntries:t=null,graphPasses:n=null}={}){this.loweringRules=zO(e),this.codegenEntries=zO(t),this.graphPasses=n||FO()}get hasOverrides(){return this.loweringRules.size>0||this.codegenEntries.size>0}getLoweringRule(e){return this.loweringRules.get(e)||null}getCodegenEntry(e){return this.codegenEntries.get(e)||null}passesForPhase(e,t,n){return Vg(e,t,n,this.graphPasses)}}});function jg(r){return new lT().print(r)}var lT,cT=M(()=>{"use strict";Vn();lT=class{static{l(this,"TensorIRPrinter")}constructor(){this.indent=0,this.out=[]}print(e){return this.out=[],this.visit(e),this.out.join("")}push(e){this.out.push(e)}newline(){this.out.push(`
-`+"  ".repeat(this.indent))}visit(e){if(!e)return;let t="visit"+e.type;this[t]?this[t](e):this.push(`[UnknownNode: ${e.type}]`)}visitPrimFunc(e){this.push(`prim_func ${e.name}(${e.params.map(t=>t.name).join(", ")}) {`),this.indent++,this.newline();for(let[t,n]of e.bufferMap)this.push(`${n.name} = buffer_map(${t.name}, shape=[${n.shape.join(",")}], dtype=${n.dtype})`),this.newline();this.visit(e.body),this.indent--,this.newline(),this.push("}")}visitSeqNode(e){for(let t=0;t<e.stmts.length;t++)this.visit(e.stmts[t]),t<e.stmts.length-1&&this.newline()}visitForNode(e){let t=e.kind==="serial"?"":`@${e.kind} `,n=e.threadTag?`[${e.threadTag}] `:"";this.push(`for ${e.loopVar.name} in 0..`),this.visit(e.extent),this.push(` ${t}${n}{`),this.indent++,this.newline(),this.visit(e.body),this.indent--,this.newline(),this.push("}")}visitBlockNode(e){this.push(`block ${e.name} {`),this.indent++,this.newline();for(let t of e.iterVars)this.push(`bind ${t.iterVar.name} = `),this.visit(t.binding),this.newline();e.reads.length>0&&(this.push("reads(["),this.push(e.reads.map(t=>`${t.buffer.name}[...]`).join(", ")),this.push("])"),this.newline()),e.writes.length>0&&(this.push("writes(["),this.push(e.writes.map(t=>`${t.buffer.name}[...]`).join(", ")),this.push("])"),this.newline()),e.initBody&&(this.push("init {"),this.indent++,this.newline(),this.visit(e.initBody),this.indent--,this.newline(),this.push("}"),this.newline()),this.visit(e.body),this.indent--,this.newline(),this.push("}")}visitBufferStoreNode(e){this.push(`${e.buffer.name}[`);for(let t=0;t<e.indices.length;t++)this.visit(e.indices[t]),t<e.indices.length-1&&this.push(", ");this.push("] = "),this.visit(e.value)}visitBufferLoadNode(e){this.push(`${e.buffer.name}[`);for(let t=0;t<e.indices.length;t++)this.visit(e.indices[t]),t<e.indices.length-1&&this.push(", ");this.push("]")}visitIfThenElseNode(e){this.push("if ("),this.visit(e.condition),this.push(") {"),this.indent++,this.newline(),this.visit(e.thenBody),this.indent--,this.newline(),this.push("}"),e.elseBody&&(this.push(" else {"),this.indent++,this.newline(),this.visit(e.elseBody),this.indent--,this.newline(),this.push("}"))}visitLetStmtNode(e){this.push(`let ${e.variable.name} = `),this.visit(e.value),this.newline(),this.visit(e.body)}visitAllocateNode(e){this.push(`allocate ${e.buffer.name}[${e.buffer.shape.join(", ")}] (${e.scope}) {`),this.indent++,this.newline(),this.visit(e.body),this.indent--,this.newline(),this.push("}")}visitEvaluateNode(e){this.push("evaluate "),this.visit(e.value)}visitMathOpNode(e){this.push("("),this.visit(e.a),e.b&&(this.push(` ${e.op} `),this.visit(e.b)),this.push(")")}visitCompareNode(e){this.push("("),this.visit(e.a),this.push(` ${Ka(e.direction)} `),this.visit(e.b),this.push(")")}visitCallExternNode(e){this.push(`${e.externName}(`);for(let t=0;t<e.args.length;t++)this.visit(e.args[t]),t<e.args.length-1&&this.push(", ");this.push(")")}visitVariableNode(e){this.push(e.name)}visitIntImmNode(e){this.push(e.value.toString())}visitFloatImmNode(e){this.push(e.value.toString())}visitCastNode(e){this.push(`cast<${e.toDtype}>(`),this.visit(e.expr),this.push(")")}};l(jg,"printTensorIR")});var qg,GO=M(()=>{"use strict";Kt();cT();qg=class{static{l(this,"TirPassManager")}constructor(){this.passes=[],this.trace=null,this.verifyHook=null}addPass(e){this.passes.push(e)}setTrace(e){this.trace=e}setVerifyHook(e){this.verifyHook=e}run(e,t){for(let n of this.passes)this._runPass(n,e,t);return e}_runPass(e,t,n){let s=n.trace;e.trace=s,s.phaseStart(e.phase);let o=performance.now();e.begin(n);for(let i=0;i<t.length;i++){let a=t[i];if(!n.failed.has(a.name))try{let u=e.run(a,n);u&&u!==a&&(t[i]=u)}catch(u){if(n.errors.push(new Dn(e.phase,a.name,u.message)),n.failed.add(a.name),s.errorEvent(e.phase,a.name,u.message),!n.resilient)break}}if(e.end(n),s.phaseEnd(e.phase,performance.now()-o),e.snapshotPoint&&s.shouldSnapshot(e.snapshotPoint))for(let i of t)n.failed.has(i.name)||s.irDump(e.snapshotPoint+":"+i.name,jg(i));this.verifyHook&&this._verifyFuncs(t,n),e.trace=null}_verifyFuncs(e,t){for(let n of e){if(t.failed.has(n.name))continue;let s=this.verifyHook(n);if(s&&s.length>0){let o=s.join("; ");if(t.resilient)t.errors.push(new Dn("verification",n.name,o)),t.failed.add(n.name);else throw new Error("TensorIR verification failed for "+n.name+": "+o)}}}}});function jO(r){let e=new Ua;return e.add(new Ja),e.add(new Za),e.add(new Qa(r)),e.add(new Xa),e.add(new eu(r)),e.add(new tu),e.add(new nu),e.add(new bh),e.add(new Ya),e.add(new vh),e.add(new Nh),e.add(new Ah),e.add(new Oh),r&&(e.add(new kh(r)),e.add(new xh(r)),e.add(new Sh(r))),e}var b$,w$,Ug,qO=M(()=>{"use strict";nn();ac();Cw();ru();l(jO,"buildAlgebraicPatterns");b$=jO(!1),w$=jO(!0),Ug=class extends Ge{static{l(this,"AlgebraicSimplificationPass")}constructor(e={}){super("algebraic_simplify"),this.preservedAnalyses=new Set,this.patterns=e.fastMath?w$:b$}run(e,t){return new Gu(this.patterns).applyPatterns(e,10,this.trace)}}});function UO(r){let e=ne.get(r);return!!(e&&e.isConstant)}function T$(r,e){return!mb(e)||typeof r!="number"?!0:Number.isInteger(r)&&Number.isSafeInteger(r)}function KO(r,e,t){if(t.has(r))return t.get(r);let n=x$(r,e,t);return t.set(r,n),n}function x$(r,e,t){let n=r.definingOp;if(!n)return;if(UO(n.opName))return n.getAttr("value");if(e.has(n))return;e.add(n);let s=ne.get(n.opName);if(!s||!s.fold||s.hasSideEffects||n.regions.length>0)return;let o=new Array(n.numOperands),i=new Array(n.numOperands);for(let a=0;a<n.numOperands;a++){let u=KO(n.getOperand(a),e,t);if(u===void 0)return;o[a]=u,i[a]=n.getOperand(a).definingOp}try{return s.fold(o,n.attributes,i)}catch{return}}var Kg,WO=M(()=>{"use strict";nn();es();At();Kt();Oe();l(UO,"isConstantProducer");l(T$,"isFoldResultRepresentable");l(KO,"resolveConstantValue");l(x$,"computeConstantValue");Kg=class extends Ge{static{l(this,"ConstantFoldPass")}constructor(){super("constant_fold")}run(e,t){let n=!1,s=0,o=new An(e),i=new Map;for(let a of[...e.opsRecursive()]){if(!a.parentBlock||UO(a.opName))continue;let u=ne.get(a.opName);if(!u||a.regions.length>0||u.hasSideEffects||u.getMemoryEffects&&u.getMemoryEffects(a).length>0||!u.fold||a.numOperands===0)continue;let p=new Array(a.numOperands),c=new Array(a.numOperands),h=!0;for(let f=0;f<a.numOperands;f++){let d=KO(a.getOperand(f),new Set,i);if(d===void 0){h=!1;break}p[f]=d,c[f]=a.getOperand(f).definingOp}if(h)try{let f=u.fold(p,a.attributes,c);if(f===void 0||!T$(f,a.getResult(0).type.dtype))continue;o.block=a.parentBlock,o.setInsertionPoint(a);let d=o.constant(f,a.getResult(0).type);a.replaceAllResultsWith([d.getResult(0)]),a.erase(),n=!0,s++}catch(f){this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,foldError:a.opName,message:f.message,level:ue.DEBUG})}}return this.trace&&this.trace.level>=ue.DEBUG&&s>0&&this.trace.emit({type:"pass_detail",passName:this.name,foldedCount:s,level:ue.DEBUG}),n?oe.CHANGED:oe.UNCHANGED}}});var Wg,HO=M(()=>{"use strict";nn();At();Kt();Wg=class extends Ge{static{l(this,"CSEPass")}constructor(){super("cse"),this.preservedAnalyses=new Set}run(e,t){let n=!1,s=0,o=typeof e.blocksRecursive=="function"?[...e.blocksRecursive()]:[...e.body];for(let i of o){let a=new Map;for(let u of[...i.ops()]){if(!u.parentBlock||u.regions&&u.regions.length>0)continue;let p=ne.get(u.opName);if(p&&p.hasSideEffects||p&&p.getMemoryEffects&&p.getMemoryEffects(u).length>0)continue;let c=u.structuralHash();if(!a.has(c)){a.set(c,[u]);continue}let h=a.get(c),f=!1;for(let d of h)if(d.parentBlock&&d.structuralEquals(u)){let m=[];for(let g=0;g<d.numResults;g++)m.push(d.getResult(g));u.replaceAllResultsWith(m),u.erase(),n=!0,s++,f=!0;break}f||h.push(u)}}return this.trace&&this.trace.level>=ue.DEBUG&&s>0&&this.trace.emit({type:"pass_detail",passName:this.name,eliminated:s,level:ue.DEBUG}),n?oe.CHANGED:oe.UNCHANGED}}});function fT(r,e){let t=pT[r],n=pT[e];return t===void 0||n===void 0?!1:r===je.REDUCTION?e===je.ELEMENTWISE:e===je.REDUCTION?t<=pT[je.INJECTIVE]:!0}function _l(r){let e=!1,t=!1,n=!1,s=!1;for(let o of r){let i=ne.get(o.opName);if(!i||i.isOpaque){n=!0;continue}i.isReduction?e=!0:i.isInjective?t=!0:i.isBroadcast?s=!0:i.isElementwise||(n=!0)}return n?je.OPAQUE:e?je.REDUCTION:t?je.INJECTIVE:s?je.BROADCAST:je.ELEMENTWISE}function Aa(r){let e=ne.get(r.opName);return!e||e.isOpaque?je.OPAQUE:e.isReduction?je.REDUCTION:e.isInjective?je.INJECTIVE:e.isBroadcast?je.BROADCAST:e.isElementwise?je.ELEMENTWISE:je.OPAQUE}function YO(r,e){if(r.length!==e.length)return!1;for(let t=0;t<r.length;t++)if(!(r[t]===X||e[t]===X)&&r[t]!==e[t])return!1;return!0}var je,pT,gl,yl,Oa=M(()=>{"use strict";Oe();At();Ui();Tr();je=Object.freeze({ELEMENTWISE:"kElementwise",BROADCAST:"kBroadcast",REDUCTION:"kReduction",INJECTIVE:"kInjective",OPAQUE:"kOpaque",HORIZONTAL:"kHorizontal"}),pT={[je.ELEMENTWISE]:0,[je.BROADCAST]:1,[je.INJECTIVE]:2,[je.REDUCTION]:3};l(fT,"canFusePatterns");l(_l,"classifyFusionKind");l(Aa,"classifyOpPattern");gl=Object.freeze({legal:!0}),yl=class{static{l(this,"FusionLegality")}constructor(e={}){this.maxFusionSize=e.maxFusionSize||512,this.maxSharedMemory=e.maxSharedMemory||49152,this.libraryOps=e.libraryOps||new Set,this.allowReductionFusion=e.allowReductionFusion!==!1,this._lowerableCache=new Map}isOpLowerable(e){let t=this._lowerableCache.get(e);return t===void 0&&(t=qt(e)||ko(e)||$b(e),this._lowerableCache.set(e,t)),t}canFuse(e,t){if(!e||!t)return{legal:!1,reason:"null op"};if(e===t)return{legal:!1,reason:"same op"};if(e.regions.length>0&&e.opName!=="fusion"&&e.opName!=="reduce")return{legal:!1,reason:"producer has control flow regions"};if(t.regions.length>0&&t.opName!=="fusion"&&t.opName!=="reduce")return{legal:!1,reason:"consumer has control flow regions"};let n=ne.get(e.opName),s=ne.get(t.opName);if(!n)return{legal:!1,reason:"unknown producer op"};if(!s)return{legal:!1,reason:"unknown consumer op"};if(!this.isOpLowerable(e.opName))return{legal:!1,reason:`producer op '${e.opName}' has no lowering rule`};if(!this.isOpLowerable(t.opName))return{legal:!1,reason:`consumer op '${t.opName}' has no lowering rule`};if(n.isOpaque)return{legal:!1,reason:"producer is opaque (use EpilogueFusionPass for dot/conv epilogues)"};if(s.isOpaque)return{legal:!1,reason:"consumer is opaque"};let o=Aa(e),i=Aa(t);return fT(o,i)?(o===je.REDUCTION||i===je.REDUCTION)&&!this.allowReductionFusion?{legal:!1,reason:"reduction fusion disabled by target"}:o===je.ELEMENTWISE&&i===je.ELEMENTWISE?this._checkElementwisePair(e,t):(o===je.BROADCAST||o===je.REDUCTION)&&i===je.ELEMENTWISE?gl:this._checkProducerConsumerShapes(e,t):{legal:!1,reason:`cannot fuse pattern ${o} -> ${i}`}}canMergeGroups(e,t){if(e.size+t.size>this.maxFusionSize)return{legal:!1,reason:"merged group exceeds max fusion size"};let n=0;for(let s of e.ops){let o=ne.get(s.opName);if(o&&o.isReduction&&n++,o&&o.isOpaque)return{legal:!1,reason:"opaque op in merge (use EpilogueFusionPass)"}}for(let s of t.ops){let o=ne.get(s.opName);if(o&&o.isReduction&&n++,o&&o.isOpaque)return{legal:!1,reason:"opaque op in merge (use EpilogueFusionPass)"}}return n>1?{legal:!1,reason:"merged group would contain multiple reductions"}:gl}_checkElementwisePair(e,t){let n=this._getOutputShape(e),s=this._getOutputShape(t);return n&&s&&!YO(n,s)?{legal:!1,reason:`elementwise shape mismatch: [${n}] vs [${s}]`}:gl}_checkProducerConsumerShapes(e,t){let n=this._getOutputShape(e);if(!n)return gl;for(let s=0;s<t.numOperands;s++)if(t.getOperand(s).definingOp===e){let o=t.getOperand(s).type;if(o instanceof I)return YO(n,o.shape)?gl:{legal:!1,reason:`shape mismatch on data edge: [${n}] vs [${o.shape}]`}}return gl}_getOutputShape(e){for(let t=0;t<e.numResults;t++){let n=e.getResult(t).type;if(n instanceof I)return n.shape}return null}};l(YO,"shapesCompatible")});function Ba(r){return Fh(r,"throw")}function tf(r,{labelOf:e,sameLabel:t=l((i,a)=>i===a,"sameLabel"),canMerge:n=l(()=>!0,"canMerge"),onAttach:s=l(()=>{},"onAttach"),sort:o=Ba}){let i=o(r),a=new Map,u=new Map,p=[],c=0,h=l((m,g)=>{if(m===g)return!0;let y=[g],_=new Set;for(;y.length>0;){let b=y.pop();if(b===m)return!0;if(_.has(b))continue;_.add(b);let w=u.get(b);if(w)for(let x of w)y.push(x)}return!1},"isUpstreamOf"),f=l(m=>{let g=new Set;for(let y=0;y<m.numOperands;y++){let _=m.getOperand(y).definingOp;if(!_)continue;let b=a.get(_);b&&g.add(b)}return g},"operandParts"),d=l((m,g)=>{for(let y of f(m)){if(y===g)continue;let _=u.get(g);_||(_=new Set,u.set(g,_)),_.add(y)}},"recordEdges");for(let m of i){let g=e(m);if(g==null)continue;let y=!1;for(let _=0;_<m.numOperands;_++){let b=m.getOperand(_).definingOp;if(!b)continue;let w=a.get(b);if(!w||!t(w.label,g)||!n(w,m,g))continue;let x=!1;for(let S of f(m))if(S!==w&&h(w,S)){x=!0;break}if(!x){w.ops.push(m),w.opSet.add(m),s(w,m),a.set(m,w),d(m,w),y=!0;break}}if(!y){let _={id:c++,label:g,ops:[m],opSet:new Set([m])};s(_,m),p.push(_),a.set(m,_),d(m,_)}}return{partitions:p,opToPart:a,preds:u}}function _i(r,e,{pullConstants:t=!1,isConstant:n=l(()=>!1,"isConstant")}={}){let s=[],o=new Set,i=[],a=new Set,u=[],p=new Set;for(let c of e){for(let h=0;h<c.numOperands;h++){let f=c.getOperand(h),d=f.definingOp;if(!(d&&r.has(d))){if(t&&d&&n(d)){p.has(d)||(p.add(d),u.push(d));continue}o.has(f)||(o.add(f),s.push(f))}}for(let h=0;h<c.numResults;h++){let f=c.getResult(h);if(a.has(f))continue;let d=!1;for(let m of f.uses())if(!r.has(m.user)){d=!0;break}d&&(a.add(f),i.push(f))}}return{inputs:s,outputs:i,constDefs:u}}function hT(r,e){let t=new Map,n=new Map;for(let a of r)t.set(a,0),n.set(a,[]);for(let a of r){let u=e.get(a);if(u)for(let p of u)n.has(p)&&(n.get(p).push(a),t.set(a,t.get(a)+1))}let s=[];for(let a of r)t.get(a)===0&&s.push(a);let o=[],i=0;for(;i<s.length;){let a=s[i++];o.push(a);for(let u of n.get(a)){let p=t.get(u)-1;t.set(u,p),p===0&&s.push(u)}}return o.length===r.length?o:null}var nf=M(()=>{"use strict";ji();l(Ba,"topoSortOps");l(tf,"buildPartitions");l(_i,"computePartitionIO");l(hT,"topoSortPartitions")});function S$(r){for(let e=0;e<r.numResults;e++){let t=r.getResult(e).type;if(t instanceof I)return t.shape.join(",")}return null}var Eo,Hg,Yg=M(()=>{"use strict";Oe();At();Oa();Ui();nf();Eo=class{static{l(this,"FusionGroup")}constructor(e){this.id=e,this.ops=[],this.opSet=new Set,this.kind=null,this._inputValues=null,this._outputValues=null,this.minTopoPos=1/0,this.maxTopoPos=-1/0}addOp(e,t){this.opSet.has(e)||(this.ops.push(e),this.opSet.add(e),this._inputValues=null,this._outputValues=null,t!==void 0&&(t<this.minTopoPos&&(this.minTopoPos=t),t>this.maxTopoPos&&(this.maxTopoPos=t)))}hasOp(e){return this.opSet.has(e)}merge(e){for(let t of e.ops)this.addOp(t);e.minTopoPos<this.minTopoPos&&(this.minTopoPos=e.minTopoPos),e.maxTopoPos>this.maxTopoPos&&(this.maxTopoPos=e.maxTopoPos)}computeIO(){if(this._inputValues&&this._outputValues)return;let{inputs:e,outputs:t}=_i(this.opSet,this.ops);this._inputValues=e,this._outputValues=t}getInputValues(){return this.computeIO(),this._inputValues}getOutputValues(){return this.computeIO(),this._outputValues}get size(){return this.ops.length}classifyKind(){return this.kind=_l(this.ops),this.kind}allOpsInlineFusable(){for(let e of this.ops){let t=ne.get(e.opName);if(!t)return!1;if(!(t.isReduction||t.isConstant)&&!ko(e.opName))return!1}return!0}};l(S$,"outputShapeKey");Hg=class{static{l(this,"FusionGroupBuilder")}constructor(e){this.legality=e,this._nextId=0,this._topoIndex=null}buildProducerConsumerGroups(e){this._topoIndex=new Map;let t=0;for(let i of e.ops())this._topoIndex.set(i,t++);let n=[],s=new Map;for(let i of e.ops()){let a=ne.get(i.opName);if(!a||a.isConstant||a.isTerminator||a.isReduction)continue;let u=this._topoIndex.get(i);for(let p=0;p<i.numOperands;p++){let c=i.getOperand(p).definingOp;if(!c)continue;let h=ne.get(c.opName);if(!h||h.isConstant||h.isReduction)continue;let f=s.get(i),d=s.get(c);if(f&&f===d||!this.legality.canFuse(c,i).legal)continue;let g=this._topoIndex.get(c);if(!f&&!d){let y=new Eo(this._nextId++);y.addOp(c,g),y.addOp(i,u),s.set(c,y),s.set(i,y)}else if(f&&!d)f.size<this.legality.maxFusionSize&&!this._wouldCreateCycle(f,c)&&(f.addOp(c,g),s.set(c,f));else if(!f&&d)d.size<this.legality.maxFusionSize&&!this._wouldCreateCycle(d,i)&&(d.addOp(i,u),s.set(i,d));else if(this.legality.canMergeGroups(f,d).legal&&!this._mergeWouldCreateCycle(f,d)){f.merge(d);for(let _ of d.ops)s.set(_,f)}}}let o=new Set;for(let i of s.values())o.has(i)||i.size<2||(o.add(i),i.classifyKind(),n.push(i));return n}_wouldCreateCycle(e,t){for(let n=0;n<t.numOperands;n++){let s=t.getOperand(n).definingOp;if(!(!s||e.hasOp(s)||this._topoIndex.get(s)<e.minTopoPos)&&this._dependsOnGroup(s,e))return!0}return!1}_dependsOnOps(e,t,n){let s=new Set,o=[e];for(s.add(e);o.length>0;){let i=o.pop();for(let a=0;a<i.numOperands;a++){let u=i.getOperand(a).definingOp;if(!u||s.has(u))continue;if(t.has(u))return!0;this._topoIndex.get(u)<n||(s.add(u),o.push(u))}}return!1}_dependsOnGroup(e,t){return this._dependsOnOps(e,t.opSet,t.minTopoPos)}_mergeWouldCreateCycle(e,t){let n=Math.min(e.minTopoPos,t.minTopoPos),s=Math.max(e.maxTopoPos,t.maxTopoPos),o=new Set([...e.opSet,...t.opSet]);for(let i of o)for(let a=0;a<i.numOperands;a++){let u=i.getOperand(a).definingOp;if(!u||o.has(u))continue;let p=this._topoIndex.get(u);if(!(p<n||p>s)&&this._dependsOnOps(u,o,n))return!0}return!1}_bucketable(e,t){return t&&!t.isConstant&&!t.isTerminator&&!t.isOpaque&&e.numResults>0&&S$(e)!==null}buildHorizontalGroups(e){let t=[...e.ops()];this._topoIndex=new Map;for(let u=0;u<t.length;u++)this._topoIndex.set(t[u],u);let n=[],s=new Map,o=this.legality.maxFusionSize||t.length,i=new Map,a=0;for(let u=0;u<t.length;u++){let p=t[u];if(s.has(p))continue;let c=ne.get(p.opName);if(!this._bucketable(p,c))continue;let h=p.getResult(0).type;a++,i.set(p,a);let f=new Eo(this._nextId++);f.addOp(p,u),s.set(p,f);let d=Math.min(t.length,u+1+o);for(let m=u+1;m<d;m++){let g=t[m],y=!1;for(let b=0;b<g.numOperands;b++){let w=g.getOperand(b).definingOp;if(w&&i.get(w)===a){y=!0;break}}if(y){i.set(g,a);continue}if(f.size>=o||s.has(g))continue;let _=ne.get(g.opName);this._bucketable(g,_)&&h.equals(g.getResult(0).type)&&(this._sharesInput(p,g)||c.isElementwise&&_.isElementwise)&&(f.addOp(g,m),s.set(g,f),i.set(g,a))}f.size>=2&&(f.kind=je.HORIZONTAL,n.push(f))}return n}buildAllGroups(e){let t=this.buildProducerConsumerGroups(e),n=this.buildHorizontalGroups(e),s=new Map;for(let c of t)for(let h of c.ops)s.set(h,c);let o=new Set(s.keys()),i=[];for(let c of n){let h=!1;for(let f of c.ops)if(o.has(f)){h=!0;break}h||i.push(c)}let a=[...e.ops()],u=[];for(let c of a)for(let h=0;h<c.numOperands;h++){let f=c.getOperand(h).definingOp;f&&u.push(c,f)}for(let c of i)for(let h of c.ops)s.set(h,c);if(!this._condensedHasCycle(a,u,s))return[...t,...i];for(let c of i)for(let h of c.ops)s.delete(h);let p=[...t];for(let c of i){for(let h of c.ops)s.set(h,c);if(this._condensedHasCycle(a,u,s)){for(let h of c.ops)s.delete(h);continue}p.push(c)}return p}_condensedHasCycle(e,t,n){let s=l(h=>n.get(h)||h,"repOf"),o=new Map,i=new Set;for(let h of e)i.add(s(h));for(let h=0;h<t.length;h+=2){let f=s(t[h]),d=s(t[h+1]);if(d===f)continue;i.add(d),i.add(f);let m=o.get(d);m||(m=new Set,o.set(d,m)),m.add(f)}let a=0,u=1,p=2,c=new Map;for(let h of i)c.set(h,a);for(let h of i){if(c.get(h)!==a)continue;let f=[h];for(;f.length>0;){let d=f[f.length-1],m=c.get(d);if(m===a){c.set(d,u);let g=o.get(d);if(g)for(let y of g){let _=c.get(y);if(_===u)return!0;_===a&&f.push(y)}}else m===u&&c.set(d,p),f.pop()}}return!1}_sharesInput(e,t){if(e.numOperands===0||t.numOperands===0)return!1;let n=new Set;for(let s=0;s<e.numOperands;s++)n.add(e.getOperand(s));for(let s=0;s<t.numOperands;s++)if(n.has(t.getOperand(s)))return!0;return!1}_hasDependency(e,t){for(let n=0;n<e.numResults;n++){let s=e.getResult(n);for(let o=0;o<t.numOperands;o++)if(t.getOperand(o)===s)return!0}return!1}}});var k$,bi,Jg=M(()=>{"use strict";Oe();At();k$=Object.freeze({memory:1,launch:1e3}),bi=class{static{l(this,"FusionCostModel")}constructor(e={}){this.memoryBandwidthGBs=e.memoryBandwidthGBs||900,this.computeTFLOPs=e.computeTFLOPs||15,this.launchOverheadUs=e.launchOverheadUs||5,this.minBenefitRatio=e.minBenefitRatio||1.05,this.maxRegistersPerThread=e.maxRegistersPerThread||255,this.maxSharedMemory=e.maxSharedMemory||49152,this.maxCodeSizeOps=e.maxCodeSizeOps||256,this.libraryOps=e.libraryOps||new Set,this.registerBytesPerOp=e.registerBytesPerOp||8,this.policy=e.policy||null,this.benefitWeights={...k$,...e.benefitWeights||{}}}edgeBenefit(e){let t=this.benefitWeights;return t.launch*this.launchOverheadUs+t.memory*e}estimateOpCost(e){let t=this.estimateFLOPs(e),n=this.estimateBytes(e);return{flops:t,bytes:n,arithmeticIntensity:n>0?t/n:0}}estimateFLOPs(e){let t=ne.get(e.opName);if(t&&t.getFlops)return t.getFlops(e);let n=1;for(let s=0;s<e.numResults;s++){let o=e.getResult(s).type;if(o instanceof I){let i=o.numel();i!==X&&(n=i);break}}if(t&&t.isReduction&&e.numOperands>0){let s=e.getOperand(0).type;if(s instanceof I){let o=s.numel();if(o!==X)return o}}return n}estimateBytes(e){let t=0;for(let n=0;n<e.numOperands;n++){let s=e.getOperand(n).type;if(s instanceof I){let o=s.sizeInBytes();o!==X&&(t+=o)}}for(let n=0;n<e.numResults;n++){let s=e.getResult(n).type;if(s instanceof I){let o=s.sizeInBytes();o!==X&&(t+=o)}}return t}estimateGroupCost(e){let t=e.opSet,n=new Map,s=0,o=0,i=0,a=0,u=!1,p=!1,c=0,h=0,f=0,d=0,m=new Set;for(let x of e.ops){let S=this.estimateFLOPs(x);n.set(x,S),s+=S,o+=this.estimateBytes(x),this.libraryOps.has(x.opName)&&a++;let T=ne.get(x.opName);if(T){let k=this._outputElements(x);T.isReduction&&(u=!0,c+=k),T.isElementwise&&(p=!0,h+=k)}for(let k=0;k<x.numResults;k++){let A=x.getResult(k);m.add(A);let N=0,v=!1;for(let O of A.uses())t.has(O.user)?N++:v=!0;N>1&&(i+=(N-1)*S),(N>0||v)&&f++}for(let k=0;k<x.numOperands;k++){let A=x.getOperand(k);if(!m.has(A))continue;let N=!1;for(let v of A.uses())if(t.has(v.user)&&v.user!==x){N=!0;break}N||f--}f>d&&(d=f)}let g=e.getInputValues(),y=e.getOutputValues(),_=0;for(let x of g)if(x.type instanceof I){let S=x.type.sizeInBytes();S!==X&&(_+=S)}for(let x of y)if(x.type instanceof I){let S=x.type.sizeInBytes();S!==X&&(_+=S)}let b=0;for(let x of e.ops)for(let S=0;S<x.numResults;S++){let T=x.getResult(S),k=0;for(let A of T.uses())if(t.has(A.user)&&(k++,k>1))break;if(k>1){let A=T.type;if(A instanceof I){let N=A.sizeInBytes();N!==X&&(b+=N)}}}let w=0;return u&&p&&h>0&&c>0&&(w=Math.abs(h-c)),{unfusedFLOPs:s,unfusedBytes:o,fusedFLOPs:s+i,fusedBytes:_,recomputeCost:i,memorySaved:o-_,launchSaved:(e.size-1)*this.launchOverheadUs,registerPressure:d*this.registerBytesPerOp,sharedMemoryUsage:b,parallelismLoss:w,libraryCallLoss:a}}shouldFuse(e){if(e.size<2)return{fuse:!1,reason:"group too small"};if(this.policy&&typeof this.policy.shouldFuse=="function"){let n=this.policy.shouldFuse(e,this);if(n)return n}let t=this.estimateGroupCost(e);if(t.libraryCallLoss>0)return{fuse:!1,reason:"fusion would lose library call opportunity",cost:t};if(t.registerPressure>this.maxRegistersPerThread)return{fuse:!1,reason:`register pressure ${t.registerPressure} exceeds limit ${this.maxRegistersPerThread}`,cost:t};if(t.sharedMemoryUsage>this.maxSharedMemory)return{fuse:!1,reason:`shared memory ${t.sharedMemoryUsage} exceeds limit ${this.maxSharedMemory}`,cost:t};if(e.size>this.maxCodeSizeOps)return{fuse:!1,reason:`code size ${e.size} exceeds limit ${this.maxCodeSizeOps}`,cost:t};if(t.memorySaved<=0&&t.launchSaved<=0)return{fuse:!1,reason:"no memory or launch benefit",cost:t};if(t.parallelismLoss>0){let n=t.memorySaved+t.launchSaved*1e3;if(t.parallelismLoss>n*this.minBenefitRatio)return{fuse:!1,reason:"parallelism loss outweighs fusion benefit",cost:t}}return{fuse:!0,reason:`saves ${t.memorySaved} bytes, ${t.launchSaved}us launch`,cost:t}}_outputElements(e){for(let t=0;t<e.numResults;t++){let n=e.getResult(t).type;if(n instanceof I){let s=n.numel();if(s!==X)return s}}return 0}}});function bl(r){let e=null;for(let t of r.ops())e=t;return e&&e.opName==="yield"?e:null}function wl(r){let e=0,t=r.regions[0]?.entryBlock;if(!t)return 0;for(let n of t.ops())n.opName!=="yield"&&e++;return e}function Tl(r){let e=0,t=r.regions[0]?.entryBlock;if(!t)return 0;for(let n of t.ops()){let s=ne.get(n.opName);s&&s.isReduction&&e++}return e}function xl(r){let e=r.regions[0]?.entryBlock;if(!e)return!1;for(let t of e.ops())if(t.opName!=="yield"&&!ko(t.opName))return!1;return!0}function v$(r){let e=new Map,t=0;for(let n=r.firstOp;n;n=n._next)e.set(n,t++);return e}function dT(r){let e=v$(r);return(t,n)=>{let s=e.get(t),o=e.get(n);return s!==void 0&&o!==void 0&&s<o}}function Sl(r,e){let t=wo(r.ops,g=>r.hasOp(g),"null");if(t===null||t.length===0)return null;r._inputValues=null,r._outputValues=null;let n=r.getInputValues(),s=r.getOutputValues(),o=n.map(g=>g.type),i=s.map(g=>g.type),a=new Ct,u=new yn(o);a.addBlock(u);let p=new Map;for(let g=0;g<n.length;g++)p.set(n[g],u.arguments[g]);for(let g of t)u.pushOp(g.clone(p));let c=s.map(g=>{let y=p.get(g);if(y===void 0)throw new Error("Fusion materialization: output value not found in valueMap");return y});u.pushOp(new Re("yield",c,[]));let h=new Re("fusion",n,i,{fusion_kind:r.kind||e},[a]),f=t[0].parentBlock;if(!f)return null;let d=dT(f),m=null;for(let g of n){let y=g.definingOp;!y||r.hasOp(y)||(!m||!d(y,m))&&(m=y)}m&&m.parentBlock===f?f.insertAfter(h,m):f.insertBefore(h,t[0]);for(let g=0;g<s.length;g++)s[g].replaceAllUsesWith(h.getResult(g));for(let g of t)g.dropAllOperands(),g.parentBlock&&g.parentBlock.removeOp(g);return h}var kl=M(()=>{"use strict";At();Ui();sr();zi();ji();l(bl,"getYieldOp");l(wl,"countInnerOps");l(Tl,"countReductions");l(xl,"allInnerOpsFusable");l(v$,"blockPositionIndex");l(dT,"makeComesBefore");l(Sl,"materializeFusionGroup")});var Zg,JO=M(()=>{"use strict";nn();Oa();Yg();Jg();kl();Kt();Zg=class extends Ge{static{l(this,"FusionPass")}constructor(e={}){super("FusionPass");let t=e.target||{};this.legality=new yl({maxFusionSize:t.maxFusionSize||e.maxFusionSize,maxSharedMemory:t.sharedMemoryBytes||e.maxSharedMemory,libraryOps:t.libraryOps||e.libraryOps,allowReductionFusion:e.allowReductionFusion}),this.costModel=new bi({memoryBandwidthGBs:t.memoryBandwidthGBs,computeTFLOPs:t.computeTFLOPs,maxRegistersPerThread:t.registersPerThread,maxSharedMemory:t.sharedMemoryBytes,libraryOps:t.libraryOps,policy:t.getAttr?t.getAttr("fusionPolicy"):null,...e.cost}),this.groupBuilder=new Hg(this.legality)}run(e,t){let n=this.groupBuilder.buildAllGroups(e),s=[];for(let i of n)this._createsCycle(e,i)?this._traceDecision(i,!1,"fusing would create a dependency cycle"):s.push(i);let o=[];for(let i of s){if(!i.allOpsInlineFusable()){this._traceDecision(i,!1,"group contains ops without inline fusion support");continue}let a=this.costModel.shouldFuse(i);this._traceDecision(i,a.fuse,a.reason),a.fuse&&o.push(i)}if(o.length===0)return oe.UNCHANGED;for(let i of o)Sl(i,je.ELEMENTWISE);return oe.CHANGED}_traceDecision(e,t,n){if(!this.trace||this.trace.level<ue.DEBUG)return;let s=e.ops.map(o=>o.opName);this.trace.emit({type:"fusion_decision",passName:this.name,groupSize:e.ops.length,ops:s,anchor:s[s.length-1]||null,fuse:t,reason:n||null,level:ue.DEBUG}),this.trace.explain("fusion",s.join("+"),t?"fused":"not-fused",n||null,{groupSize:s.length})}_createsCycle(e,t){let n=t.opSet,s=t.getOutputValues(),o=new Set,i=[];for(let u of s)for(let p of u.uses())n.has(p.user)||(i.push(p.user),o.add(p.user));let a=0;for(;a<i.length;){let u=i[a++];if(n.has(u))return!0;for(let p=0;p<u.numResults;p++)for(let c of u.getResult(p).uses())o.has(c.user)||(o.add(c.user),i.push(c.user))}return!1}}});function Qg(r){return lu(r.opName)||ln(r.opName)}function ZO(r){if(Qg(r))return!0;let e=ne.get(r.opName);return e!==null&&e.isElementwise}function A$(r,e){let t=r.getOperand(0).definingOp;return t&&e.has(t)&&!Qg(t)?r.getOperand(1).definingOp:t}function O$(r,e){if(!r.parentBlock||r.parentBlock!==e.parentBlock)return!1;let t=r.parentBlock.firstOp;for(;t;){if(t===r)return!0;if(t===e)return!1;t=t._next}return!1}function B$(r,e){for(let t of r)if(t!==e){for(let n=0;n<t.numResults;n++)for(let s of t.getResult(n).uses())if(!r.has(s.user))return!0}return!1}function R$(r,e){let t=ne.get(r.opName),n=t&&t.getAttr("epilogueTag");return n?n(r,e):"activation"}function M$(r){let e=[],t=new Set,n=new Set;function s(f){if(!(n.has(f)||f===r)){n.add(f);for(let d=0;d<f.numOperands;d++){let m=f.getOperand(d).definingOp;m&&m!==r&&!n.has(m)&&Qg(m)&&s(m)}e.push(f),t.add(f)}}l(s,"absorb");let o=[],i=r.getResult(0);for(let f of i.uses())ZO(f.user)&&o.push(f.user);for(;o.length>0;){let f=o.pop();if(!n.has(f)){s(f);for(let d=0;d<f.numResults;d++)for(let m of f.getResult(d).uses())!n.has(m.user)&&ZO(m.user)&&o.push(m.user)}}let a=[];for(let f of e)Qg(f)||a.push(R$(f,t));let u=e[e.length-1];for(let f=e.length-1;f>=0;f--){let d=e[f];for(let m=0;m<d.numResults;m++){for(let g of d.getResult(m).uses())if(!t.has(g.user)){u=d,f=-1;break}if(f<0)break}}let p=new Set;for(let f=0;f<r.numOperands;f++)p.add(r.getOperand(f));let c=[],h=new Set;for(let f of e)for(let d=0;d<f.numOperands;d++){let m=f.getOperand(d);if(h.has(m))continue;h.add(m);let g=m.definingOp;g!==r&&(g&&t.has(g)||p.has(m)||c.push(m))}return{chain:e,chainSet:t,tags:a,lastOp:u,extras:c}}function I$(r){let e=null,t=null,n=null,s=null,o=new Set;for(let i=0;i<2;i++){let a=r.getOperand(i),u=a.definingOp;if(!u||u.opName!=="convert")continue;let p=!1;for(let h of a.uses())if(h.user!==r){p=!0;break}if(p)continue;let c=u.getAttr("target_dtype")||u.getResult(0).type.dtype;i===0?(e=c,n=u.getOperand(0)):(t=c,s=u.getOperand(0)),o.add(u)}return{lhsCast:e,rhsCast:t,lhsInput:n,rhsInput:s,removed:o}}var N$,Xg,QO=M(()=>{"use strict";nn();sr();At();Tr();Kt();l(Qg,"isPassthrough");l(ZO,"isEpilogueCandidate");N$=new Map([["add",(r,e)=>e.has(A$(r,e))?"residual_add":"bias"],["sub",()=>"bias"],["mul",()=>"scale"],["maximum",()=>"relu"],["clamp",()=>"clamp"],["neg",()=>"neg"],["exp",()=>"exp"],["tanh",()=>"tanh"],["sqrt",()=>"sqrt"],["abs",()=>"abs"],["log",()=>"log"]]);l(A$,"resolveOtherOperand");for(let[r,e]of N$)ne.has(r)&&ne.registerOpAttr(r,"epilogueTag",e);l(O$,"comesBefore");l(B$,"hasEscapingUse");l(R$,"classifyTag");l(M$,"collectChainAndAnalyze");l(I$,"collectPrologue");Xg=class extends Ge{static{l(this,"EpilogueFusionPass")}constructor(e={}){super("EpilogueFusionPass"),this.maxEpilogueOps=e.maxEpilogueOps||16,this.target=e.target||null}run(e){if(this.target&&!this.target.enableEpilogueFusion)return oe.UNCHANGED;let t=!1,n=[];for(let s of e.ops()){let o=ne.get(s.opName);o&&o.isOutEWiseFusable&&n.push(s)}for(let s of n){let o=M$(s);if(o.chain.length>this.maxEpilogueOps)continue;let i=I$(s),a=o.chain.length>0,u=i.lhsCast!==null||i.rhsCast!==null;if(!a&&!u)continue;let{chain:p,tags:c,extras:h}=o,f=0;for(let k of c)k==="bias"||k==="residual_add"||k==="scale"?f++:k==="clamp"&&(f+=2);if(f!==h.length)continue;let d=i.lhsInput||s.getOperand(0),m=i.rhsInput||s.getOperand(1),g=[d,m,...h],y=a?o.lastOp:s,_=y.getResult(0).type,b=new Map(s.attributes);b.set("epilogue_ops",p.map(k=>k.opName)),b.set("epilogue_tags",c),b.set("num_dot_operands",2),b.set("num_extra_inputs",h.length),i.lhsCast&&b.set("lhs_prologue_cast",i.lhsCast),i.rhsCast&&b.set("rhs_prologue_cast",i.rhsCast);let w=new Re("fused_dot_epilogue",g,[_],b),x=s.parentBlock;if(!x)continue;let S=new Set(p);S.add(s);for(let k of i.removed)S.add(k);if(B$(S,y))continue;let T=null;for(let k of g){let A=k.definingOp;!A||S.has(A)||(!T||!O$(A,T))&&(T=A)}T&&T.parentBlock===x?x.insertAfter(w,T):x.insertBefore(w,s),y.getResult(0).replaceAllUsesWith(w.getResult(0));for(let k=p.length-1;k>=0;k--)p[k].dropAllOperands(),p[k].parentBlock&&p[k].parentBlock.removeOp(p[k]);s.dropAllOperands(),s.parentBlock&&s.parentBlock.removeOp(s);for(let k of i.removed)k.dropAllOperands(),k.parentBlock&&k.parentBlock.removeOp(k);t=!0}return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,dotsFound:n.length,changed:t,level:ue.DEBUG}),t?oe.CHANGED:oe.UNCHANGED}}});var ey,XO=M(()=>{"use strict";nn();sr();zi();Oe();At();Kt();Oa();kl();ey=class extends Ge{static{l(this,"FusionMergerPass")}constructor(e={}){super("FusionMergerPass"),this.maxFusionSize=e.maxFusionSize||512,this.maxReductions=e.maxReductions||1,this.launchOverheadUs=e.launchOverheadUs||5,this.minMemorySavings=e.minMemorySavings||0}run(e){let t=[];for(let u of e.ops())u.opName==="fusion"&&t.push(u);if(t.length<2)return oe.UNCHANGED;let n=new Set(t),s=this._buildProducerConsumerEdges(t,n);if(s.length===0)return oe.UNCHANGED;let o=!1,i=0,a=new Set;for(let{producer:u,consumer:p,sharedResults:c}of s)a.has(u)||a.has(p)||this._canMerge(u,p)&&this._shouldMerge(u,p,c)&&(this._mergeCreatesCycle(u,p,n)||(this._merge(u,p,c),a.add(u),a.add(p),o=!0,i++));return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,fusionOps:t.length,edges:s.length,mergeCount:i,level:ue.DEBUG}),o?oe.CHANGED:oe.UNCHANGED}_buildProducerConsumerEdges(e,t){let n=[],s=new Map;for(let o of e)for(let i=0;i<o.numResults;i++){let a=o.getResult(i);for(let u of a.uses()){if(!t.has(u.user))continue;let p=u.user;if(p===o)continue;let c=`${o.id}|${p.id}`,h=s.get(c);h||(h={producer:o,consumer:p,sharedResults:new Map},s.set(c,h),n.push(h)),h.sharedResults.set(i,u.operandIndex)}}return n}_canMerge(e,t){let n=wl(e),s=wl(t);if(n+s>this.maxFusionSize)return!1;let o=0;return o+=Tl(e),o+=Tl(t),!(o>this.maxReductions||!xl(e)||!xl(t))}_shouldMerge(e,t,n){let s=0;for(let[u]of n){let p=e.getResult(u);if(p.type instanceof I){let c=p.type.sizeInBytes();c!==X&&(s+=c)}}let o=!0;for(let u=0;u<e.numResults;u++){let p=e.getResult(u);for(let c of p.uses())if(c.user!==t){o=!1;break}if(!o)break}if(o)return!0;let i=this._estimateRecomputeCost(e);return s+this.launchOverheadUs*1e3>i}_mergeCreatesCycle(e,t,n){let s=new Set,o=[];for(let a=0;a<t.numOperands;a++){let p=t.getOperand(a).definingOp;p&&p!==e&&p.opName!=="constant"&&(s.has(p)||(s.add(p),o.push(p)))}let i=0;for(;i<o.length;){let a=o[i++];if(a===e)return!0;for(let u=0;u<a.numOperands;u++){let p=a.getOperand(u).definingOp;p&&!s.has(p)&&(s.add(p),o.push(p))}}return!1}_merge(e,t,n){let s=e.regions[0].entryBlock,o=t.regions[0].entryBlock,i=bl(s),a=bl(o);if(!i||!a)return;let u=new Map;for(let[N,v]of n)u.set(v,N);let p=[],c=new Map;for(let N=0;N<e.numOperands;N++)c.set(N,p.length),p.push(e.getOperand(N));let h=new Map;for(let N=0;N<t.numOperands;N++){if(u.has(N))continue;let v=t.getOperand(N),O=-1;for(let R=0;R<p.length;R++)if(p[R]===v){O=R;break}O>=0?h.set(N,O):(h.set(N,p.length),p.push(v))}let f=p.map(N=>N.type),d=new Ct,m=new yn(f);d.addBlock(m);let g=new Map;for(let N=0;N<s.arguments.length;N++)g.set(s.arguments[N],m.arguments[c.get(N)]);for(let N of s.ops())N.opName!=="yield"&&m.pushOp(N.clone(g));for(let N=0;N<i.numOperands;N++){let v=i.getOperand(N),O=g.get(v)||v;for(let[R,B]of u)B===N&&g.set(o.arguments[R],O)}for(let N=0;N<o.arguments.length;N++){if(u.has(N))continue;let v=h.get(N);v!==void 0&&g.set(o.arguments[N],m.arguments[v])}for(let N of o.ops())N.opName!=="yield"&&m.pushOp(N.clone(g));let y=[],_=[],b=[];for(let N=0;N<e.numResults;N++){let v=!1;for(let O of e.getResult(N).uses())if(O.user!==t){v=!0;break}if(v){let O=i.getOperand(N),R=g.get(O)||O;b.push({resultIdx:N,value:R}),y.push(R),_.push(e.getResult(N).type)}}for(let N=0;N<t.numResults;N++){let v=a.getOperand(N),O=g.get(v)||v;y.push(O),_.push(t.getResult(N).type)}let w=new Re("yield",y,[]);m.pushOp(w);let x=[];for(let N of m.ops())N.opName!=="yield"&&x.push(N);let S=_l(x),T=new Re("fusion",p,_,{fusion_kind:S},[d]),k=t;if(!k.parentBlock)return;k.parentBlock.insertBefore(T,k);let A=0;for(let{resultIdx:N}of b)e.getResult(N).replaceAllUsesWith(T.getResult(A)),A++;for(let N=0;N<t.numResults;N++)t.getResult(N).replaceAllUsesWith(T.getResult(A)),A++;e.dropAllOperands(),e.parentBlock&&e.parentBlock.removeOp(e),t.dropAllOperands(),t.parentBlock&&t.parentBlock.removeOp(t)}_estimateRecomputeCost(e){let t=0,n=e.regions[0]?.entryBlock;if(!n)return 0;for(let s of n.ops()){if(s.opName==="yield")continue;let o=ne.get(s.opName);if(o&&o.getFlops)t+=o.getFlops(s);else for(let i=0;i<s.numResults;i++){let a=s.getResult(i).type;if(a instanceof I){let u=a.numel();u!==X&&(t+=u);break}}}return t}}});function E$(r,e){let t=Math.min(r.id,e.id),n=Math.max(r.id,e.id);return`${t}|${n}`}var rf,eB=M(()=>{"use strict";nn();sr();zi();Oe();Oa();Kt();kl();rf=class extends Ge{static{l(this,"MultiOutputFusionPass")}constructor(e={}){super("MultiOutputFusionPass"),this.maxFusionSize=e.maxFusionSize||512,this.maxReductions=e.maxReductions||1,this.minSharedInputBytes=e.minSharedInputBytes||0,this.maxOutputs=e.maxOutputs||8,this.maxConsumersPerInput=e.maxConsumersPerInput||64}run(e){let t=[];for(let a of e.ops())a.opName==="fusion"&&t.push(a);if(t.length<2)return oe.UNCHANGED;let n=this._findCandidates(t);if(n.length===0)return oe.UNCHANGED;let s=!1,o=0,i=new Set;for(let{left:a,right:u,sharedInputs:p,sharedBytes:c}of n)i.has(a)||i.has(u)||this._canMerge(a,u)&&(this._mergeCreatesCycle(a,u)||(this._mergeMultiOutput(a,u,p),i.add(a),i.add(u),s=!0,o++));return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,fusionOps:t.length,candidates:n.length,mergeCount:o,level:ue.DEBUG}),s?oe.CHANGED:oe.UNCHANGED}_findCandidates(e){let t=new Map;for(let o of e)for(let i=0;i<o.numOperands;i++){let u=o.getOperand(i).id,p=t.get(u);p||(p=[],t.set(u,p)),p.push(o)}let n=new Map;for(let[o,i]of t){if(i.length<2)continue;let a=Math.min(i.length,this.maxConsumersPerInput);for(let u=0;u<a;u++)for(let p=u+1;p<a;p++){let c=i[u],h=i[p];if(c===h)continue;let f=E$(c,h),d=n.get(f);d||(d={left:c,right:h,sharedInputs:new Set,sharedBytes:0},n.set(f,d)),d.sharedInputs.add(o)}}let s=[];for(let o of n.values()){let i=0;for(let a of o.sharedInputs){let u=t.get(a);if(!u||u.length===0)continue;let p=u[0];for(let c=0;c<p.numOperands;c++){let h=p.getOperand(c);if(h.id===a&&h.type instanceof I){let f=h.type.sizeInBytes();f!==X&&(i+=f);break}}}o.sharedBytes=i,i>=this.minSharedInputBytes&&s.push(o)}return s.sort((o,i)=>i.sharedBytes-o.sharedBytes),s}_canMerge(e,t){if(this._hasProducerConsumerEdge(e,t)||this._hasProducerConsumerEdge(t,e))return!1;let n=wl(e),s=wl(t);if(n+s>this.maxFusionSize||e.numResults+t.numResults>this.maxOutputs)return!1;let i=0;return i+=Tl(e),i+=Tl(t),!(i>this.maxReductions||!xl(e)||!xl(t))}_mergeCreatesCycle(e,t){let n=new Set,s=[];for(let i=0;i<e.numResults;i++){let a=e.getResult(i);for(let u of a.uses())u.user!==t&&!n.has(u.user)&&(n.add(u.user),s.push(u.user))}let o=0;for(;o<s.length;){let i=s[o++];if(i===t)return!0;for(let a=0;a<i.numResults;a++)for(let u of i.getResult(a).uses())n.has(u.user)||(n.add(u.user),s.push(u.user))}n.clear(),s.length=0;for(let i=0;i<t.numResults;i++){let a=t.getResult(i);for(let u of a.uses())u.user!==e&&!n.has(u.user)&&(n.add(u.user),s.push(u.user))}for(o=0;o<s.length;){let i=s[o++];if(i===e)return!0;for(let a=0;a<i.numResults;a++)for(let u of i.getResult(a).uses())n.has(u.user)||(n.add(u.user),s.push(u.user))}return!1}_mergeMultiOutput(e,t,n){let s=e.regions[0].entryBlock,o=t.regions[0].entryBlock,i=bl(s),a=bl(o);if(!i||!a)return;let u=[],p=new Map,c=new Map;for(let v=0;v<e.numOperands;v++){let O=e.getOperand(v),R=O.id,B=p.get(R);B===void 0&&(B=u.length,p.set(R,B),u.push(O)),c.set(v,B)}let h=new Map;for(let v=0;v<t.numOperands;v++){let O=t.getOperand(v),R=O.id,B=p.get(R);B===void 0&&(B=u.length,p.set(R,B),u.push(O)),h.set(v,B)}let f=u.map(v=>v.type),d=new Ct,m=new yn(f);d.addBlock(m);let g=new Map;for(let v=0;v<s.arguments.length;v++)g.set(s.arguments[v],m.arguments[c.get(v)]);for(let v of s.ops())v.opName!=="yield"&&m.pushOp(v.clone(g));for(let v=0;v<o.arguments.length;v++)g.set(o.arguments[v],m.arguments[h.get(v)]);for(let v of o.ops())v.opName!=="yield"&&m.pushOp(v.clone(g));let y=[],_=[];for(let v=0;v<i.numOperands;v++){let O=i.getOperand(v);y.push(g.get(O)||O),_.push(e.getResult(v).type)}for(let v=0;v<a.numOperands;v++){let O=a.getOperand(v);y.push(g.get(O)||O),_.push(t.getResult(v).type)}let b=new Re("yield",y,[]);m.pushOp(b);let w=[];for(let v of m.ops())v.opName!=="yield"&&w.push(v);let x=_l(w),S=new Re("fusion",u,_,{fusion_kind:x},[d]),T=e.parentBlock;if(!T)return;let k=dT(T),A=null;for(let v of u){let O=v.definingOp;!O||O===e||O===t||(!A||!k(O,A))&&(A=O)}A&&A.parentBlock===T?T.insertAfter(S,A):T.insertBefore(S,e);for(let v=0;v<e.numResults;v++)e.getResult(v).replaceAllUsesWith(S.getResult(v));let N=e.numResults;for(let v=0;v<t.numResults;v++)t.getResult(v).replaceAllUsesWith(S.getResult(N+v));e.dropAllOperands(),e.parentBlock&&e.parentBlock.removeOp(e),t.dropAllOperands(),t.parentBlock&&t.parentBlock.removeOp(t)}_hasProducerConsumerEdge(e,t){for(let n=0;n<e.numResults;n++)for(let s of e.getResult(n).uses())if(s.user===t)return!0;return!1}};l(E$,"pairKey")});var mT,at,Es=M(()=>{"use strict";mT=class{static{l(this,"UseDefResult")}constructor(e,t,n,s,o){this.topologicalOrder=e,this.valueToOp=t,this.opUsers=n,this.depth=s,this.height=o}},at=class{static get name(){return"use_def"}static get depKey(){return"useDef"}static get dependencies(){return[]}static compute(e){let t=[],n=new Map,s=new Map,o=new Set,i=new Set,a=l(c=>{if(o.has(c))return;i.add(c);let h=[{op:c,i:0}];for(;h.length>0;){let f=h[h.length-1],d=f.op;if(f.i<d.numOperands){let m=d.getOperand(f.i).definingOp;if(f.i++,m&&!o.has(m)){if(i.has(m))throw new Error("Cycle detected in UseDefAnalysis");i.add(m),h.push({op:m,i:0})}continue}i.delete(d),o.add(d),t.push(d),h.pop()}},"visit");for(let c of e.ops()){for(let h=0;h<c.numResults;h++)n.set(c.getResult(h),c);s.set(c,new Set)}let u=new Map,p=new Map;for(let c of e.ops())a(c);for(let c of t){let h=0;for(let f=0;f<c.numOperands;f++){let d=c.getOperand(f).definingOp;if(d){s.get(d).add(c);let m=u.get(d)||0;m+1>h&&(h=m+1)}}u.set(c,h)}for(let c=t.length-1;c>=0;c--){let h=t[c],f=0,d=s.get(h)||new Set;for(let m of d){let g=p.get(m)||0;g+1>f&&(f=g+1)}p.set(h,f)}return new mT(t,n,s,u,p)}}});var gT,ty,tB=M(()=>{"use strict";Es();Tr();gT=class{static{l(this,"DominanceResult")}constructor(e){this.idom=e}postDominates(e,t){let n=t;for(;n;){if(n===e)return!0;n=this.idom.get(n)}return!1}immediatePDom(e){return this.idom.get(e)||null}pathToPDom(e){let t=[],n=this.idom.get(e);for(;n&&(t.push(n),n!==this.idom.get(n));)n=this.idom.get(n);return t}},ty=class{static get name(){return"post_dominance"}static get depKey(){return"postDominance"}static get dependencies(){return[at]}static compute(e,t={}){let s=(t.useDef||at.compute(e)).topologicalOrder,o=new Map;for(let m of s)o.set(m,[]);for(let m of s)for(let g=0;g<m.numResults;g++){let y=m.getResult(g);for(let _ of y.uses()){let b=_.user;o.has(b)&&o.get(m).push(b)}}let i=new Set;for(let m of s)qt(m.opName)&&i.add(m);let a={opName:"__pdom_root__"},u=Math.max(1,Math.ceil(Math.log2(s.length+2))+1),p=new Map,c=new Map,h=new Map;c.set(a,0),h.set(a,new Array(u).fill(a));let f=l((m,g)=>{p.set(m,g),c.set(m,c.get(g)+1);let y=new Array(u);y[0]=g;for(let _=1;_<u;_++)y[_]=h.get(y[_-1])[_-1];h.set(m,y)},"link"),d=l((m,g)=>{if(c.get(m)<c.get(g)){let _=m;m=g,g=_}let y=c.get(m)-c.get(g);for(let _=0;_<u;_++)y>>_&1&&(m=h.get(m)[_]);if(m===g)return m;for(let _=u-1;_>=0;_--)h.get(m)[_]!==h.get(g)[_]&&(m=h.get(m)[_],g=h.get(g)[_]);return h.get(m)[0]},"lca");for(let m=s.length-1;m>=0;m--){let g=s[m];if(i.has(g)){f(g,a);continue}let y=null;for(let _ of o.get(g))p.has(_)&&(y=y===null?_:d(y,_));y!==null&&f(g,y)}for(let[m,g]of p)g===a&&p.delete(m);return new gT(p,s)}}});function nB(r){return qt(r)||ln(r)}var ny,rB=M(()=>{"use strict";nn();At();Oa();Yg();Jg();tB();Kt();Es();kl();Tr();l(nB,"isSkipOp");ny=class extends Ge{static{l(this,"DominatorFusionPass")}constructor(e={}){super("DominatorFusionPass"),this.requiredAnalyses=[at];let t=e.target||{};this.maxFusionSize=t.maxFusionSize||e.maxFusionSize||512,this.maxReductions=e.maxReductions||1,this.libraryOps=t.libraryOps||e.libraryOps||new Set,this.costModel=new bi({memoryBandwidthGBs:t.memoryBandwidthGBs,computeTFLOPs:t.computeTFLOPs,maxRegistersPerThread:t.registersPerThread,maxSharedMemory:t.sharedMemoryBytes,libraryOps:t.libraryOps,...e.cost})}run(e,t){let n=t?t.getAnalysis(at,e):at.compute(e),s=ty.compute(e,{useDef:n}),o=n.topologicalOrder,i=this._buildGroups(o,s);if(i.length===0)return oe.UNCHANGED;let a=[];for(let u of i){if(!u.allOpsInlineFusable()){this._explain(u,!1,"group contains ops without inline fusion support");continue}let p=this.costModel.shouldFuse(u);this._explain(u,p.fuse,p.reason),p.fuse&&a.push(u)}if(a.length===0)return oe.UNCHANGED;for(let u of a)Sl(u,je.ELEMENTWISE);return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,groupsBuilt:i.length,groupsFused:a.length,level:ue.DEBUG}),oe.CHANGED}_explain(e,t,n){if(!this.trace||!this.trace.explainsEnabled)return;let s=e.ops.map(o=>o.opName);this.trace.explain("fusion",s.join("+"),t?"fused":"not-fused",n||null,{groupSize:s.length,strategy:"dominator"})}_buildGroups(e,t){let n=new Map,s=[],o=0;for(let p=e.length-1;p>=0;p--){let c=e[p];if(nB(c.opName))continue;let h=Aa(c);if(h===je.OPAQUE||this.libraryOps.has(c.opName))continue;let f=t.immediatePDom(c);if(!f||nB(f.opName))continue;let d=Aa(f);if(d===je.OPAQUE||this.libraryOps.has(f.opName)||!this._canFusePatterns(h,d)||!this._pathAllFusable(c,f,e,t))continue;let m=n.get(f);if(m){if(m.size+1>this.maxFusionSize||!this._checkReductionLimit(m,c))continue;m.addOp(c),n.set(c,m),this._absorbIntermediates(c,f,m,n,e,t)}else{let g=n.get(c);if(g){if(g.size+1>this.maxFusionSize||!this._checkReductionLimit(g,f))continue;g.addOp(f),n.set(f,g),this._absorbIntermediates(c,f,g,n,e,t)}else{let y=new Eo(o++);if(y.addOp(c),y.addOp(f),!this._checkGroupReductions(y))continue;n.set(c,y),n.set(f,y),s.push(y),this._absorbIntermediates(c,f,y,n,e,t)}}}let i=new Set,a=[];for(let p of s)i.has(p)||p.size<2||(i.add(p),p.classifyKind(),a.push(p));for(let p of n.values())i.has(p)||p.size<2||(i.add(p),p.classifyKind(),a.push(p));let u=[];for(let p of a){if(!this._checkGroupReductions(p)){this._explain(p,!1,`group exceeds the ${this.maxReductions}-reduction limit`);continue}if(this._createsCycle(p)){this._explain(p,!1,"fusing would create a dependency cycle");continue}u.push(p)}return u}_createsCycle(e){let t=new Set;for(let o of e.getInputValues())o.definingOp&&!e.hasOp(o.definingOp)&&t.add(o.definingOp);if(t.size===0)return!1;let n=new Set,s=[];for(let o of e.getOutputValues())for(let i of o.uses())e.hasOp(i.user)||s.push(i.user);for(;s.length>0;){let o=s.pop();if(!n.has(o)){if(n.add(o),t.has(o))return!0;for(let i=0;i<o.numResults;i++)for(let a of o.getResult(i).uses())e.hasOp(a.user)||s.push(a.user)}}return!1}_canFusePatterns(e,t){return fT(e,t)}_pathAllFusable(e,t,n,s){let o=new Set,i=[e];for(o.add(e),o.add(t);i.length>0;){let a=i.pop();for(let u=0;u<a.numResults;u++)for(let p of a.getResult(u).uses()){let c=p.user;if(c===t||o.has(c)||!s.postDominates(t,c))continue;if(o.add(c),Aa(c)===je.OPAQUE||this.libraryOps.has(c.opName))return!1;i.push(c)}}return!0}_absorbIntermediates(e,t,n,s,o,i){let a=new Set,u=[e];for(a.add(e),a.add(t);u.length>0;){let p=u.pop();for(let c=0;c<p.numResults;c++)for(let h of p.getResult(c).uses()){let f=h.user;f===t||a.has(f)||!i.postDominates(t,f)||(a.add(f),Aa(f)===je.OPAQUE||this.libraryOps.has(f.opName))||(n.size<this.maxFusionSize&&this._checkReductionLimit(n,f)&&(n.addOp(f),s.set(f,n)),u.push(f))}}}_checkGroupReductions(e){let t=0;for(let n of e.ops){let s=ne.get(n.opName);s&&s.isReduction&&t++}return t<=this.maxReductions}_checkReductionLimit(e,t){let n=ne.get(t.opName);if(!n||!n.isReduction)return!0;let s=0;for(let o of e.ops){let i=ne.get(o.opName);i&&i.isReduction&&s++}return s<this.maxReductions}}});var ry,sB=M(()=>{"use strict";ry=class{static{l(this,"GraphCycles")}constructor(e,t){this._n=e,this._parent=new Int32Array(e),this._rank=new Int32Array(e),this._nodeAtRank=new Int32Array(e),this._out=new Array(e),this._in=new Array(e);for(let n=0;n<e;n++)this._parent[n]=n,this._rank[n]=n,this._nodeAtRank[n]=n,this._out[n]=new Set,this._in[n]=new Set;if(t)for(let[n,s]of t)n!==s&&(this._out[n].add(s),this._in[s].add(n))}find(e){let t=this._parent,n=e;for(;t[n]!==n;)n=t[n];for(;t[e]!==n;){let s=t[e];t[e]=n,e=s}return n}wouldCreateCycle(e,t){let n=this.find(e),s=this.find(t);if(n===s)return!1;let o=this._rank[n]<this._rank[s]?n:s,i=o===n?s:n;return this._hasIntermediatePath(o,i)}_hasIntermediatePath(e,t){let n=this._rank[t],s=new Set([e]),o=[e];for(;o.length>0;){let i=o.pop();for(let a of this._out[i]){let u=this.find(a);if(!(u===i||u===e)){if(u===t){if(i!==e)return!0;continue}this._rank[u]>=n||s.has(u)||(s.add(u),o.push(u))}}}return!1}merge(e,t){let n=this.find(e),s=this.find(t);if(n===s)return n;let o=this._rank[n]<this._rank[s]?n:s,i=o===n?s:n,a=this._rank[o],u=this._rank[i],p=this._out[n].size+this._in[n].size,c=this._out[s].size+this._in[s].size,h=p>=c?n:s,f=h===n?s:n,d=this._out[h],m=this._in[h];for(let g of this._in[f]){let y=this.find(g);y===h||y===f||(this._out[y].delete(f),this._out[y].add(h),m.add(y))}for(let g of this._out[f]){let y=this.find(g);y===h||y===f||(this._in[y].delete(f),this._in[y].add(h),d.add(y))}return d.delete(f),m.delete(f),d.delete(h),m.delete(h),this._parent[f]=h,this._nodeAtRank[this._rank[f]]=-1,this._reorder(a,u),h}_reorder(e,t){let n=[],s=[];for(let c=e;c<=t;c++){let h=this._nodeAtRank[c];if(!(h<0)){if(this.find(h)!==h){this._nodeAtRank[c]=-1;continue}n.push(c),s.push(h)}}if(s.length<=1){for(let c=0;c<s.length;c++)this._rank[s[c]]=n[c],this._nodeAtRank[n[c]]=s[c];return}let o=new Set(s),i=new Map;for(let c of s)i.set(c,0);for(let c of s)for(let h of this._out[c]){let f=this.find(h);f!==c&&o.has(f)&&i.set(f,i.get(f)+1)}let a=[];for(let c of s)i.get(c)===0&&a.push(c);let u=[],p=0;for(;p<a.length;){let c=a[p++];u.push(c);for(let h of this._out[c]){let f=this.find(h);if(f!==c&&o.has(f)){let d=i.get(f)-1;i.set(f,d),d===0&&a.push(f)}}}for(let c=0;c<u.length;c++){let h=u[c],f=n[c];this._rank[h]=f,this._nodeAtRank[f]=h}}}});var sy,oB=M(()=>{"use strict";sy=class{static{l(this,"MaxHeap")}constructor(){this._items=[]}get size(){return this._items.length}isEmpty(){return this._items.length===0}push(e,t){let n=this._items;n.push({priority:e,value:t});let s=n.length-1;for(;s>0;){let o=s-1>>1;if(n[o].priority>=n[s].priority)break;let i=n[o];n[o]=n[s],n[s]=i,s=o}}pop(){let e=this._items,t=e.length;if(t===0)return;let n=e[0],s=e.pop();if(t>1){e[0]=s;let o=e.length,i=0;for(;;){let a=i*2+1,u=a+1,p=i;if(a<o&&e[a].priority>e[p].priority&&(p=a),u<o&&e[u].priority>e[p].priority&&(p=u),p===i)break;let c=e[p];e[p]=e[i],e[i]=c,i=p}}return n.value}}});function L$(r){let e=ne.get(r.opName);return!e||e.isConstant||e.isTerminator||e.isOpaque?!1:e.isReduction?!0:ko(r.opName)}var oy,iB=M(()=>{"use strict";nn();At();Oe();Oa();Yg();Jg();Es();sB();oB();kl();Ui();Kt();l(L$,"isFusibleOp");oy=class extends Ge{static{l(this,"PriorityFusionPass")}constructor(e={}){super("PriorityFusionPass"),this.requiredAnalyses=[at];let t=e.target||{};this.maxReductions=e.maxReductions??1,this.legality=new yl({maxFusionSize:t.maxFusionSize||e.maxFusionSize,maxSharedMemory:t.sharedMemoryBytes||e.maxSharedMemory,libraryOps:t.libraryOps||e.libraryOps,allowReductionFusion:e.allowReductionFusion});let n=t.getAttr&&t.getAttr("fusionBenefitWeights")||e.benefitWeights;this.costModel=new bi({memoryBandwidthGBs:t.memoryBandwidthGBs,computeTFLOPs:t.computeTFLOPs,maxRegistersPerThread:t.registersPerThread,maxSharedMemory:t.sharedMemoryBytes,libraryOps:t.libraryOps,policy:t.getAttr?t.getAttr("fusionPolicy"):null,benefitWeights:n,...e.cost})}run(e,t){let s=(t?t.getAnalysis(at,e):at.compute(e)).topologicalOrder,o=s.length;if(o===0)return oe.UNCHANGED;let i=new Map;for(let T=0;T<o;T++)i.set(s[T],T);let a=[];for(let T=0;T<o;T++){let k=s[T];for(let A=0;A<k.numOperands;A++){let N=k.getOperand(A).definingOp;N&&i.has(N)&&a.push([i.get(N),T])}}let u=new ry(o,a),p=new Int32Array(o),c=new Map;for(let T=0;T<o;T++)if(L$(s[T])){let k=new Eo(T);k.addOp(s[T]),c.set(T,k)}let h=new sy,f=l((T,k)=>{let A=new Eo(-1);for(let N of T.ops)A.addOp(N);for(let N of k.ops)A.addOp(N);return A},"mergedGroup"),d=l(T=>{if(T.size<2||!T.allOpsInlineFusable())return!1;let k=0;for(let A of T.ops){let N=ne.get(A.opName);N&&N.isReduction&&k++}return k>this.maxReductions||(T.classifyKind(),T.kind===je.OPAQUE)?!1:this.costModel.shouldFuse(T).fuse===!0},"legalMerge"),m=l((T,k)=>{let A=0;for(let N=0;N<k.numOperands;N++){let v=k.getOperand(N);if(v.definingOp===T&&v.type instanceof I){let O=v.type.sizeInBytes();O!==X&&(A+=O)}}return A},"edgeBytes"),g=l((T,k)=>{let A=u.find(i.get(T)),N=u.find(i.get(k));if(A===N||!c.has(A)||!c.has(N)||!this.legality.canFuse(T,k).legal)return;let v=this.costModel.edgeBenefit(m(T,k));h.push(v,{a:A,b:N,va:p[A],vb:p[N],prodOp:T,consOp:k})},"pushCandidate"),y=l((T,k)=>{let A=new Set;for(let N of T.ops){for(let v=0;v<N.numResults;v++)for(let O of N.getResult(v).uses()){let R=i.get(O.user);if(R===void 0)continue;let B=u.find(R);if(B===k||!c.has(B))continue;let E=k*o+B;A.has(E)||(A.add(E),g(N,O.user))}for(let v=0;v<N.numOperands;v++){let O=N.getOperand(v).definingOp;if(!O)continue;let R=i.get(O);if(R===void 0)continue;let B=u.find(R);if(B===k||!c.has(B))continue;let E=B*o+k;A.has(E)||(A.add(E),g(O,N))}}},"reEval"),_=new Set;for(let T=0;T<o;T++){if(!c.has(T))continue;let k=s[T];for(let A=0;A<k.numOperands;A++){let N=k.getOperand(A).definingOp;if(!N)continue;let v=i.get(N);if(v===void 0||!c.has(v))continue;let O=v*o+T;_.has(O)||(_.add(O),g(N,k))}}let b=0;for(;!h.isEmpty();){let T=h.pop(),k=u.find(T.a),A=u.find(T.b);if(k===A||p[k]!==T.va||p[A]!==T.vb)continue;let N=c.get(k),v=c.get(A);if(!N||!v||!this.legality.canFuse(T.prodOp,T.consOp).legal)continue;let O=f(N,v);if(!d(O)||u.wouldCreateCycle(k,A))continue;let R=u.merge(k,A),B=N.size>=v.size?N:v,E=B===N?v:N;B.merge(E),c.delete(k),c.delete(A),c.set(R,B),p[R]=(p[k]>p[A]?p[k]:p[A])+1,b++,y(B,R)}if(b===0)return oe.UNCHANGED;let w=[],x=new Set;for(let T of c.values())x.has(T)||T.size<2||(x.add(T),w.push(T));let S=0;for(let T of w)T.classifyKind(),this._explain(T),Sl(T,je.ELEMENTWISE)&&S++;return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,ops:o,merges:b,fusions:S,level:ue.DEBUG}),S>0?oe.CHANGED:oe.UNCHANGED}_explain(e){if(!this.trace||!this.trace.explainsEnabled)return;let t=e.ops.map(n=>n.opName);this.trace.explain("fusion",t.join("+"),"fused",null,{groupSize:t.length,strategy:"priority"})}}});function C$(r,e){return r===e?!0:!r||!e?!1:typeof r.equals=="function"?r.equals(e):!1}var wi,iy,aB=M(()=>{"use strict";Oe();wi=class{static{l(this,"LayoutPreference")}constructor(e,t,n=0){this.inputs=e,this.outputs=t,this.cost=n}},iy=class{static{l(this,"LayoutPolicy")}constructor(e){this.target=e,this._rules=new Map,this._initDefaultRules()}registerRule(e,t){this._rules.set(e,t)}getPreference(e){let t=this._rules.get(e.opName);return t?t(e,this.target):null}estimateConversionCost(e,t,n){if(!(n instanceof I)||C$(e,t))return 0;let s=n.numel();return s<0?1024:s*2}estimateBenefit(e,t,n){if(!(t instanceof I))return 0;let s=t.numel();if(s<0)return 0;let o=e.opName;if(o==="dot"||o==="conv"||o==="matmul")return s*4*n;if(o==="reduce")return s*2*n;let i=this.target.cacheLineBytes||64;return s*4<=i*4?0:Math.floor(s*.5)}_initDefaultRules(){this._rules.set("conv",(e,t)=>{let s=e.getOperand(0).type?.rank||4;if(t.preferredConvLayout)return new wi([t.preferredConvLayout,null],[t.preferredConvLayout]);if(t.isGPU()&&s===4){let o=new Et([0,2,3,1]);return new wi([o,null],[o])}if(t.isCPU()&&s===4){let o=new Et([0,2,3,1]);return new wi([o,null],[o])}return null}),this._rules.set("dot",(e,t)=>{let n=e.getOperand(0).type,s=e.getOperand(1).type;if(!n||!s)return null;let o=Et.rowMajor(n.rank);if(t.isCPU()&&s.rank===2){let a=Et.columnMajor(s.rank);return new wi([o,a],[o])}let i=Et.rowMajor(s.rank);return new wi([o,i],[o])}),this._rules.set("reduce",(e,t)=>{let n=e.getResult(0).type;return n?new wi([null],[Et.rowMajor(n.rank)]):null})}};l(C$,"layoutEquals")});function uB(r,e){let t=new Map,n=null,s=0;for(let i=0;i<r.numOperands;i++){let a=r.getOperand(i),u=e.get(a);if(!u)continue;let p=u.hash?u.hash():0,c=(t.get(p)||0)+1;t.set(p,c),c>s&&(s=c,n=u)}if(n)return n;let o=r.getResult(0);return o&&o.type instanceof I?Et.rowMajor(o.type.rank):Et.rowMajor(1)}function D$(r,e){return r===e?!0:!r||!e?!1:typeof r.equals=="function"?r.equals(e):!1}function lB(r){if(r instanceof Et)return r;if(r&&typeof r.toLayout=="function")try{return r.toLayout()}catch{}return r&&r.baseOrder&&!r.isBlocked?.()?new Et(r.baseOrder):r&&r.order?new Et(r.order):null}var yT,ay,cB=M(()=>{"use strict";Es();Oe();At();_n();Tr();yT=class{static{l(this,"LayoutAnalysisResult")}constructor(e,t,n){this.assignments=e,this.conversions=t,this.totalCost=n}},ay=class{static get name(){return"layout"}static get depKey(){return"layout"}static get dependencies(){return[at]}static compute(e,t,n){let s=t.useDef,o=new Map,i=s.topologicalOrder;for(let p of e.args)p.type instanceof I&&o.set(p,p.type.layout||Et.rowMajor(p.type.rank));for(let p=0;p<i.length;p++){let c=i[p];if(qt(c.opName))continue;let h=n?n.getPreference(c):null;if(h&&h.outputs.length>0){for(let m=0;m<c.numResults;m++){let g=c.getResult(m);if(!(g.type instanceof I))continue;let y=h.outputs[m]||null;y?o.set(g,y):o.set(g,uB(c,o))}continue}let f=ne.get(c.opName),d=f&&f.hasTrait(H.ELEMENTWISE);for(let m=0;m<c.numResults;m++){let g=c.getResult(m);g.type instanceof I&&(d?o.set(g,uB(c,o)):o.set(g,Et.rowMajor(g.type.rank)))}}let a=[],u=0;for(let p=0;p<i.length;p++){let c=i[p],h=n?n.getPreference(c):null;if(h)for(let f=0;f<c.numOperands;f++){let d=c.getOperand(f);if(!(d.type instanceof I))continue;let m=o.get(d);if(!m)continue;let g=h.inputs[f];if(!g||D$(m,g))continue;let y=lB(m),_=lB(g);!y||!_||y.equals(_)||(a.push({value:d,consumer:c,operandIdx:f,from:y,to:_}),n&&(u+=n.estimateConversionCost(y,_,d.type)))}}return new yT(o,a,u)}};l(uB,"resolveFromInputs");l(D$,"layoutEquals");l(lB,"toPlainLayout")});function P$(r,e,t){let n=r.id,s=e.hash?e.hash():0,o=t.hash?t.hash():0;return`${n}:${s}:${o}`}var uy,pB=M(()=>{"use strict";nn();sr();Oe();aB();cB();Es();Kt();uy=class extends Ge{static{l(this,"LayoutTransformPass")}constructor(e={}){super("LayoutTransformPass"),this.target=e.target||null,this._policy=null}run(e,t){if(!this.target)return oe.UNCHANGED;this._policy||(this._policy=new iy(this.target));let n=t?t.getAnalysis(at,e):at.compute(e),s=ay.compute(e,{useDef:n},this._policy);if(s.conversions.length===0)return oe.UNCHANGED;let o=new Map;for(let p of s.conversions){let{value:c,consumer:h,operandIdx:f,from:d,to:m}=p,g=P$(c,d,m),y=o.get(g);y||(y={value:c,from:d,to:m,consumers:[],cost:this._policy.estimateConversionCost(d,m,c.type),benefit:0},o.set(g,y)),y.consumers.push({consumer:h,operandIdx:f});let _=this.target.layoutAwareOps&&this.target.layoutAwareOps.has(h.opName);y.benefit+=_?this._policy.estimateBenefit(h,c.type,1):0}let i=0,a=0,u=[];for(let p of o.values())p.benefit<p.cost||(u.push(p),i+=p.cost,a+=p.benefit);if(u.length===0||i>a)return oe.UNCHANGED;for(let p of u){let c=p.from instanceof Et?p.from.order:Array.from({length:p.value.type.rank},(y,_)=>_),h=p.to instanceof Et?p.to.order:Array.from({length:p.value.type.rank},(y,_)=>_),f=new I(p.value.type.shape,p.value.type.dtype,p.to),d=new Re("layout_transform",[p.value],[f],{src_layout:[...c],dst_layout:[...h]}),m=p.value.definingOp;m&&m.parentBlock?m.parentBlock.insertAfter(d,m):p.consumers[0].consumer.parentBlock&&p.consumers[0].consumer.parentBlock.insertBefore(d,p.consumers[0].consumer);let g=d.getResult(0);for(let y of p.consumers)y.consumer.replaceOperand(y.operandIdx,g)}return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,conversions:s.conversions.length,uniqueTransforms:u.length,level:ue.DEBUG}),oe.CHANGED}};l(P$,"valueLayoutKey")});function fB(r,e){if(r===e)return!0;if(typeof r=="number"&&typeof e=="number")return r===e;if(typeof r!=typeof e||r.length!==e.length)return!1;for(let t=0;t<r.length;t++)if(r[t]!==e[t])return!1;return!0}function hB(r){let e=0;for(let t=0;t<r.length;t++)e=(e<<5)-e+r.charCodeAt(t)&2147483647;return e}function z$(r){if(typeof r=="number"){let t=new Float64Array([r]),n=new Uint32Array(t.buffer);return(n[0]^n[1])&2147483647}let e=r.length;for(let t=0;t<Math.min(r.length,8);t++){let n=new Float64Array([r[t]]),s=new Uint32Array(n.buffer);e=(e^(s[0]^s[1]))*16777619&2147483647}return e}var kr,$$,F$,hr,_T=M(()=>{"use strict";Oe();kr=Object.freeze({PER_TENSOR_SYMMETRIC:"per_tensor_symmetric",PER_TENSOR_ASYMMETRIC:"per_tensor_asymmetric",PER_CHANNEL:"per_channel",PER_GROUP:"per_group"}),$$=new Set(Object.values(kr)),F$=new Set(["i8","ui8"]),hr=class r{static{l(this,"QuantizationParams")}constructor(e){this.scheme=e.scheme,this.scale=e.scale,this.zeroPoint=e.zeroPoint,this.axis=e.axis??null,this.groupSize=e.groupSize??null,this.dtype=e.dtype||"i8",this.numBits=e.numBits||Gs(this.dtype)*8,this._hash=null}clampRange(){if(this.isSymmetric()){let n=2**(this.numBits-1)-1;return[-n,n]}if(this.dtype==="ui8")return[0,2**this.numBits-1];let e=-(2**(this.numBits-1)),t=2**(this.numBits-1)-1;return[e,t]}quantize(e){let[t,n]=this.clampRange(),s=Math.round(e/this.getScalarScale()+this.getScalarZeroPoint());return Math.max(t,Math.min(n,s))}dequantize(e){return(e-this.getScalarZeroPoint())*this.getScalarScale()}quantizeArray(e){let[t,n]=this.clampRange(),s=new Array(e.length);if(this.isPerChannel())throw new Error("Use quantizeArrayPerChannel(floatArr, shape) for per-channel quantization");let o=this.getScalarScale(),i=this.getScalarZeroPoint();for(let a=0;a<e.length;a++)s[a]=Math.max(t,Math.min(n,Math.round(e[a]/o+i)));return s}dequantizeArray(e){let t=new Array(e.length);if(this.isPerChannel())throw new Error("Use dequantizeArrayPerChannel(intArr, shape) for per-channel dequantization");let n=this.getScalarScale(),s=this.getScalarZeroPoint();for(let o=0;o<e.length;o++)t[o]=(e[o]-s)*n;return t}_channelStride(e){let t=1;for(let n=this.axis+1;n<e.length;n++)t*=e[n];return t}quantizeArrayPerChannel(e,t){let[n,s]=this.clampRange(),o=this._channelStride(t),i=t[this.axis],a=new Array(e.length);for(let u=0;u<e.length;u++){let p=Math.floor(u/o)%i,c=this.getScaleForChannel(p),h=this.getZeroPointForChannel(p);a[u]=Math.max(n,Math.min(s,Math.round(e[u]/c+h)))}return a}dequantizeArrayPerChannel(e,t){let n=this._channelStride(t),s=t[this.axis],o=new Array(e.length);for(let i=0;i<e.length;i++){let a=Math.floor(i/n)%s;o[i]=(e[i]-this.getZeroPointForChannel(a))*this.getScaleForChannel(a)}return o}quantizeArrayPerGroup(e){let[t,n]=this.clampRange(),s=new Array(e.length);for(let o=0;o<e.length;o++){let i=Math.floor(o/this.groupSize),a=this.scale[i],u=this.zeroPoint[i];s[o]=Math.max(t,Math.min(n,Math.round(e[o]/a+u)))}return s}dequantizeArrayPerGroup(e){let t=new Array(e.length);for(let n=0;n<e.length;n++){let s=Math.floor(n/this.groupSize);t[n]=(e[n]-this.zeroPoint[s])*this.scale[s]}return t}getScaleForGroup(e){return this.scale[e]}getZeroPointForGroup(e){return this.zeroPoint[e]}getScaleForChannel(e){return this.isPerChannel()?this.scale[e]:this.getScalarScale()}getZeroPointForChannel(e){return this.isPerChannel()?this.zeroPoint[e]:this.getScalarZeroPoint()}getScalarScale(){return typeof this.scale=="number"?this.scale:this.scale[0]}getScalarZeroPoint(){return typeof this.zeroPoint=="number"?this.zeroPoint:this.zeroPoint[0]}numChannels(){return this.isPerChannel()?typeof this.scale=="number"?1:this.scale.length:1}isPerChannel(){return this.scheme===kr.PER_CHANNEL}isPerGroup(){return this.scheme===kr.PER_GROUP}isSymmetric(){return this.scheme===kr.PER_TENSOR_SYMMETRIC}equals(e){return this===e?!0:!(!(e instanceof r)||this.scheme!==e.scheme||this.dtype!==e.dtype||this.numBits!==e.numBits||this.axis!==e.axis||this.groupSize!==e.groupSize||!fB(this.scale,e.scale)||!fB(this.zeroPoint,e.zeroPoint))}hash(){if(this._hash!==null)return this._hash;let e=2166136261;return e=(e^hB(this.scheme))*16777619&2147483647,e=(e^hB(this.dtype))*16777619&2147483647,e=(e^this.numBits)*16777619&2147483647,e=(e^z$(this.scale))*16777619&2147483647,this._hash=e,e}serialize(){let e=this.isPerChannel()||this.isPerGroup();return{scheme:this.scheme,scale:e?[...this.scale]:this.scale,zeroPoint:e?[...this.zeroPoint]:this.zeroPoint,axis:this.axis,groupSize:this.groupSize,dtype:this.dtype,numBits:this.numBits}}static deserialize(e){return new r(e)}static fromRange(e,t,n,s="i8",o=8){if(n===kr.PER_TENSOR_SYMMETRIC){let f=Math.max(Math.abs(e),Math.abs(t)),d=2**(o-1)-1,m=f/d||1e-10;return new r({scheme:n,scale:m,zeroPoint:0,dtype:s,numBits:o})}let[i,a]=s==="ui8"?[0,2**o-1]:[-(2**(o-1)),2**(o-1)-1],p=(t-e||1e-10)/(a-i),c=Math.round(i-e/p),h=Math.max(i,Math.min(a,c));return new r({scheme:n,scale:p,zeroPoint:h,dtype:s,numBits:o})}static fromRangePerChannel(e,t,n,s="i8",o=8){let i=e.length,a=new Float64Array(i),u=new Int32Array(i),p=2**(o-1)-1;for(let c=0;c<i;c++){let h=Math.max(Math.abs(e[c]),Math.abs(t[c]));a[c]=h/p||1e-10,u[c]=0}return new r({scheme:kr.PER_CHANNEL,scale:a,zeroPoint:u,axis:n,dtype:s,numBits:o})}static defaultForActivation(e,t="i8",n=8){return r.fromRange(-6,6,e,t,n)}static fromConstantArray(e,t,n="i8",s=8){let o=1/0,i=-1/0;for(let a=0;a<e.length;a++)e[a]<o&&(o=e[a]),e[a]>i&&(i=e[a]);return isFinite(o)||(o=-1),isFinite(i)||(i=1),o===i&&(o-=.5,i+=.5),r.fromRange(o,i,t,n,s)}static fromConstantArrayPerChannel(e,t,n,s="i8",o=8){let i=t[n],a=1;for(let c=n+1;c<t.length;c++)a*=t[c];let u=new Array(i).fill(1/0),p=new Array(i).fill(-1/0);for(let c=0;c<e.length;c++){let h=Math.floor(c/a)%i,f=e[c];f<u[h]&&(u[h]=f),f>p[h]&&(p[h]=f)}for(let c=0;c<i;c++)isFinite(u[c])||(u[c]=-1),isFinite(p[c])||(p[c]=1),u[c]===p[c]&&(u[c]-=.5,p[c]+=.5);return r.fromRangePerChannel(u,p,n,s,o)}static fromConstantArrayPerGroup(e,t,n="i8",s=4){let o=Math.ceil(e.length/t),i=new Float64Array(o),a=new Int32Array(o),u=2**(s-1)-1;for(let p=0;p<o;p++){let c=p*t,h=Math.min(c+t,e.length),f=0;for(let d=c;d<h;d++){let m=Math.abs(e[d]);m>f&&(f=m)}i[p]=f/u||1e-10,a[p]=0}return new r({scheme:kr.PER_GROUP,scale:i,zeroPoint:a,groupSize:t,dtype:n,numBits:s})}static isQuantizableDtype(e){return F$.has(e)}static isValidScheme(e){return $$.has(e)}};l(fB,"scaleEquals");l(hB,"hashStr");l(z$,"hashScaleValue")});function q$(r){for(let e=0;e<r.numOperands;e++){let t=r.getOperand(e).definingOp;if(t&&t.opName==="constant")return!0}return!1}function U$(r,e,t){for(let n=0;n<r.numOperands;n++){let s=r.getOperand(n);if(!e.has(s)&&(!(s.type instanceof I)||!Lt(s.type.dtype)))return!1}return r.numOperands>0}var V$,G$,j$,ly,cy,dB=M(()=>{"use strict";nn();sr();Oe();At();Es();_T();Kt();Tr();V$=new Set(["softmax","sqrt","div","rsqrt","log","exp","tanh"]),G$=new Set(["dot","conv","add","mul","sub"]),j$=new Map([["dot","quantized_dot"],["conv","quantized_conv"]]);for(let[r,e]of j$)ne.has(r)&&ne.registerOpAttr(r,"quantizedVariant",e);ly=class{static{l(this,"QuantizationConfig")}constructor(e={}){this.scheme=e.scheme||kr.PER_TENSOR_SYMMETRIC,this.calibration=e.calibration||null,this.targetDtype=e.targetDtype||"i8",this.excludeOps=e.excludeOps||V$,this.quantizableOps=e.quantizableOps||G$,this.sensitivityThreshold=e.sensitivityThreshold||0,this.sensitivityResult=e.sensitivityResult||null,this.weightOnly=e.weightOnly||!1,this.target=e.target||null}},cy=class extends Ge{static{l(this,"QuantizationPass")}constructor(e={}){super("QuantizationPass"),this.requiredAnalyses=[at],this.config=e instanceof ly?e:new ly(e)}run(e,t){let s=(t?t.getAnalysis(at,e):at.compute(e)).topologicalOrder,o=new Set;this._paramsByValue=new Map;let i=this.config,a=!1;if(i.target&&!i.target.supportsInt8)return oe.UNCHANGED;for(let p=0;p<s.length;p++){let c=s[p];if(qt(c.opName))continue;if(i.excludeOps.has(c.opName)||!i.quantizableOps.has(c.opName)){for(let d=0;d<c.numOperands;d++)o.has(c.getOperand(d))&&(a=this._insertDequantBefore(c,d,c.getOperand(d),i)||a);continue}if(i.sensitivityResult&&i.sensitivityThreshold>0&&i.sensitivityResult.isSensitive(c,i.sensitivityThreshold)||i.weightOnly&&!q$(c))continue;let h=ne.get(c.opName),f=h?h.getAttr("quantizedVariant"):null;if(f&&U$(c,o,i)){i.scheme===kr.PER_CHANNEL&&this._canPerChannelDot(c,o)?a=this._replacePerChannelDot(c,i)||a:a=this._replaceWithNativeQuantized(c,f,o,i)||a;continue}a=this._insertDequantQuantBoundary(c,o,i)||a}let u=e.getReturnOp();if(u)for(let p=0;p<u.numOperands;p++){let c=u.getOperand(p);o.has(c)&&(a=this._insertDequantBefore(u,p,c,i)||a)}return this.trace&&this.trace.level>=ue.DEBUG&&a&&this.trace.emit({type:"pass_detail",passName:this.name,opsProcessed:s.length,changed:a,level:ue.DEBUG}),a?oe.CHANGED:oe.UNCHANGED}_getQuantParams(e,t){let n=Gs(t.targetDtype)*8;if(t.calibration&&t.calibration.hasData(e))return t.calibration.getQuantParams(e,t.scheme,t.targetDtype);let s=e.definingOp;if(s&&s.opName==="constant"){let o=s.getAttr("value");if(typeof o=="number")return hr.fromRange(-Math.abs(o)||-1,Math.abs(o)||1,t.scheme,t.targetDtype,n);if(o&&typeof o.length=="number")return hr.fromConstantArray(o,t.scheme,t.targetDtype,n)}return e.type instanceof I&&Lt(e.type.dtype)?hr.defaultForActivation(t.scheme,t.targetDtype,n):null}_resolveQuantParams(e,t){let n=this._paramsByValue.get(e);if(n)return n;let s=e.definingOp;return s&&s.opName==="quantize"?new hr({scheme:s.getAttr("scheme")||t.scheme,scale:s.getAttr("scale"),zeroPoint:s.getAttr("zero_point")||0,dtype:s.getAttr("target_dtype")||t.targetDtype}):this._getQuantParams(e,t)}_insertQuantizeAfter(e,t,n){let s=e.getResult(t),o=this._getQuantParams(s,n);if(!o)return null;let i=new I(s.type.shape,n.targetDtype),a=new Re("quantize",[s],[i],{scale:o.getScalarScale(),zero_point:o.getScalarZeroPoint(),scheme:n.scheme,target_dtype:n.targetDtype,...o.axis!==null?{axis:o.axis}:{}});e.parentBlock&&e.parentBlock.insertAfter(a,e);let u=a.getResult(0);return this._paramsByValue.set(u,o),u}_insertDequantBefore(e,t,n,s){let o=this._resolveQuantParams(n,s);if(!o)return!1;let i="f32",a=new I(n.type.shape,i),u=new Re("dequantize",[n],[a],{scale:o.getScalarScale(),zero_point:o.getScalarZeroPoint(),scheme:s.scheme,target_dtype:i});return e.parentBlock&&(e.parentBlock.insertBefore(u,e),e.replaceOperand(t,u.getResult(0))),!0}_insertDequantQuantBoundary(e,t,n){let s=!1;for(let o=0;o<e.numOperands;o++){let i=e.getOperand(o);t.has(i)&&(s=this._insertDequantBefore(e,o,i,n)||s)}for(let o=0;o<e.numResults;o++){let i=e.getResult(o);if(!(i.type instanceof I)||!Lt(i.type.dtype))continue;let a=this._insertQuantizeAfter(e,o,n);if(a){let u=[...i.uses()];for(let p of u)p.user!==a.definingOp&&p.user.replaceOperand(p.operandIndex,a);t.add(a),s=!0}}return s}_replaceWithNativeQuantized(e,t,n,s){let o=[],i={};for(let[h,f]of e.attributes||[])i[h]=f;for(let h=0;h<e.numOperands;h++){let f=e.getOperand(h);if(n.has(f)){o.push(f);let _=this._resolveQuantParams(f,s);if(_){let b=h===0?"lhs":"rhs";if(t==="quantized_dot")i[`${b}_scale`]=_.getScalarScale(),i[`${b}_zero_point`]=_.getScalarZeroPoint();else{let w=h===0?"input":"kernel";i[`${w}_scale`]=_.getScalarScale(),i[`${w}_zero_point`]=_.getScalarZeroPoint()}}continue}let d=this._getQuantParams(f,s);if(!d)return!1;let m=new I(f.type.shape,s.targetDtype),g=new Re("quantize",[f],[m],{scale:d.getScalarScale(),zero_point:d.getScalarZeroPoint(),scheme:s.scheme,target_dtype:s.targetDtype});e.parentBlock&&e.parentBlock.insertBefore(g,e),o.push(g.getResult(0));let y=h===0?t==="quantized_dot"?"lhs":"input":t==="quantized_dot"?"rhs":"kernel";i[`${y}_scale`]=d.getScalarScale(),i[`${y}_zero_point`]=d.getScalarZeroPoint()}let a=i.lhs_scale||i.input_scale||1,u=i.rhs_scale||i.kernel_scale||1;i.output_scale=a*u,i.output_zero_point=0;let p=new I(e.getResult(0).type.shape,"i32"),c=new Re(t,o,[p],i);if(e.parentBlock){e.parentBlock.insertBefore(c,e);let h=new I(e.getResult(0).type.shape,e.getResult(0).type.dtype),f=i.output_scale||1,d=new Re("dequantize",[c.getResult(0)],[h],{scale:f,zero_point:i.output_zero_point||0,scheme:s.scheme,target_dtype:e.getResult(0).type.dtype});e.parentBlock.insertBefore(d,e),e.replaceAllResultsWith([d.getResult(0)]),e.erase()}return!0}_canPerChannelDot(e,t){if(e.opName!=="dot")return!1;let n=e.getOperand(0),s=e.getOperand(1);if(t.has(n)||t.has(s)||!(n.type instanceof I)||n.type.shape.length!==2||!(s.type instanceof I)||s.type.shape.length!==2)return!1;let o=s.definingOp;if(!o||o.opName!=="constant")return!1;let i=o.getAttr("value");if(!i||typeof i=="number"||typeof i.length!="number")return!1;let a=e.getAttr("rhs_contracting")||[],u=e.getAttr("lhs_contracting")||[];return!(a.length!==1||u.length!==1||(e.getAttr("rhs_batch")||[]).length!==0||(e.getAttr("lhs_batch")||[]).length!==0)}_activationParams(e,t){let n=Gs(t.targetDtype)*8,s=kr.PER_TENSOR_SYMMETRIC;return t.calibration&&t.calibration.hasData(e)?t.calibration.getQuantParams(e,s,t.targetDtype):hr.defaultForActivation(s,t.targetDtype,n)}_replacePerChannelDot(e,t){let n=e.getOperand(0),s=e.getOperand(1),o=s.type.shape,i=s.definingOp.getAttr("value"),a=Gs(t.targetDtype)*8,p=e.getAttr("rhs_contracting")[0]===0?1:0,c=hr.fromConstantArrayPerChannel([...i],o,p,t.targetDtype,a),h=c.quantizeArrayPerChannel([...i],o),f=this._activationParams(n,t),d=f.getScalarScale(),m=f.getScalarZeroPoint(),g=e.parentBlock;if(!g)return!1;let y=new I(o,t.targetDtype),_=new Re("constant",[],[y],{value:h,tensor_type:y});g.insertBefore(_,e);let b=new I(n.type.shape,t.targetDtype),w=new Re("quantize",[n],[b],{scale:d,zero_point:m,scheme:kr.PER_TENSOR_SYMMETRIC,target_dtype:t.targetDtype});g.insertBefore(w,e);let x=e.getResult(0).type.shape,S={};for(let[U,P]of e.attributes||[])S[U]=P;S.lhs_scale=d,S.lhs_zero_point=m,S.rhs_scale=1,S.rhs_zero_point=0,S.output_scale=1,S.output_zero_point=0;let T=new I(x,"i32"),k=new Re("quantized_dot",[w.getResult(0),_.getResult(0)],[T],S);g.insertBefore(k,e);let A=new I(x,"f32"),N=new Re("convert",[k.getResult(0)],[A],{target_dtype:"f32"});g.insertBefore(N,e);let v=o[p],O=new Array(v);for(let U=0;U<v;U++)O[U]=d*c.getScaleForChannel(U);let R=new Set(e.getAttr("lhs_contracting")||[]),B=0;for(let U=0;U<n.type.shape.length;U++)R.has(U)||B++;let E=B,z=new I([v],"f32"),D=new Re("constant",[],[z],{value:O,tensor_type:z});g.insertBefore(D,e);let J=new Re("broadcast_in_dim",[D.getResult(0)],[A],{broadcast_dimensions:[E],result_shape:x});g.insertBefore(J,e);let j=new Re("mul",[N.getResult(0),J.getResult(0)],[A],{});return g.insertBefore(j,e),e.replaceAllResultsWith([j.getResult(0)]),e.erase(),!0}};l(q$,"hasConstantOperand");l(U$,"allOperandsCanQuantize")});var bT,py,mB=M(()=>{"use strict";Es();Oe();bT=class{static{l(this,"LivenessResult")}constructor(e,t,n,s,o,i,a){this.liveIn=e,this.liveOut=t,this.intervals=n,this.opIndex=s,this.peakPressure=o,this.peakOp=i,this.pressureAtOp=a}interfere(e,t){if(e===t)return!0;let n=this.intervals.get(e),s=this.intervals.get(t);return n&&s?n.start<=s.end&&s.start<=n.end:!1}liveAtOp(e){return this.liveIn.get(e)||new Set}intervalOf(e){return this.intervals.get(e)||null}},py=class r{static get name(){return"liveness"}static get depKey(){return"liveness"}static get dependencies(){return[at]}static buildIntervals(e,t){let n=new Map;for(let o=0;o<t.length;o++)n.set(t[o],o);let s=new Map;for(let o of e.args)s.set(o,{start:-1,end:-1});for(let o=0;o<t.length;o++){let i=t[o];for(let a=0;a<i.numResults;a++)s.set(i.getResult(a),{start:o,end:o})}for(let o=0;o<t.length;o++){let i=t[o];for(let a=0;a<i.numOperands;a++){let u=i.getOperand(a),p=s.get(u);p&&p.end<o&&(p.end=o)}}return{intervals:s,opIndex:n}}static compute(e,t={}){let n=t.useDef||at.compute(e),s=n.topologicalOrder,{intervals:o,opIndex:i}=r.buildIntervals(e,s),a=new Map,u=new Map;for(let f of s)a.set(f,new Set),u.set(f,new Set);for(let f=s.length-1;f>=0;f--){let d=s[f],m=u.get(d),g=n.opUsers.get(d);if(g)for(let _ of g){let b=a.get(_);if(b)for(let w of b)m.add(w)}let y=a.get(d);for(let _ of m)y.add(_);for(let _=0;_<d.numResults;_++)y.delete(d.getResult(_));for(let _=0;_<d.numOperands;_++)y.add(d.getOperand(_))}for(let[f,d]of i){let m=a.get(f);for(let g of m){let y=o.get(g);y&&y.end<d&&(y.end=d)}}let p=0,c=null,h=new Map;for(let f=0;f<s.length;f++){let d=s[f],m=a.get(d),g=0;for(let y of m)if(y.type instanceof I){let _=y.type.sizeInBytes();_!==X&&(g+=_)}h.set(d,g),g>p&&(p=g,c=d)}return new bT(a,u,o,i,p,c,h)}}});var fy,hy,gB=M(()=>{"use strict";nn();sr();Oe();At();Kt();Es();mB();Tr();fy=class{static{l(this,"RematerializationConfig")}constructor(e={}){this.memoryBudget=e.memoryBudget||1/0,this.maxIterations=e.maxIterations||100,this.maxRecomputeCost=e.maxRecomputeCost||1/0,this.excludeOps=e.excludeOps||new Set}},hy=class extends Ge{static{l(this,"RematerializationPass")}constructor(e={}){super("RematerializationPass"),this.config=e instanceof fy?e:new fy(e)}run(e,t){if(this.config.memoryBudget===1/0)return oe.UNCHANGED;let n=!1,s=0,o=0;for(;s<this.config.maxIterations;){let i=at.compute(e),{peakPressure:a,candidates:u,opIndex:p}=this._analyzeIntervalPressure(e,i);if(o=a,a<=this.config.memoryBudget||u.length===0)break;u.sort((c,h)=>h.score-c.score),this._rematerialize(e,u[0],p),n=!0,s++}return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,iterations:s,peakPressure:o,budget:this.config.memoryBudget,changed:n,level:ue.DEBUG}),n?oe.CHANGED:oe.UNCHANGED}_analyzeIntervalPressure(e,t){let n=t.topologicalOrder,{intervals:s,opIndex:o}=py.buildIntervals(e,n),i=[];for(let[m,g]of s){if(!(m.type instanceof I))continue;let y=m.type.sizeInBytes();y===X||y<=0||(i.push({idx:g.start,delta:y,value:m}),i.push({idx:g.end+1,delta:-y,value:m}))}i.sort((m,g)=>m.idx-g.idx||m.delta-g.delta);let a=0,u=0,p=0,c=new Set,h=new Set,f=0;for(let m=-1;m<=n.length;m++){for(;f<i.length&&i[f].idx<=m;)a+=i[f].delta,i[f].delta>0&&i[f].value&&h.add(i[f].value),i[f].delta<0&&i[f].value&&h.delete(i[f].value),f++;if(a>u){u=a,p=m,c.clear();for(let g of h)c.add(g)}}let d=[];for(let m of c){if(!this._canRematerialize(m))continue;let g=m.definingOp,y=this._computeMemorySaved(m);if(y<=0)continue;let _=this._estimateRecomputeCost(g);_>=this.config.maxRecomputeCost||_!==0&&d.push({value:m,definingOp:g,memorySaved:y,recomputeCost:_,score:y/_})}return{peakPressure:u,peakIdx:p,candidates:d,opIndex:o}}_canRematerialize(e){if(e.isBlockArgument())return!1;let t=e.definingOp;if(!t||qt(t.opName)||ln(t.opName)||this.config.excludeOps.has(t.opName)||t.regions.length>0||t.hasSideEffects()||e.useCount<=1)return!1;for(let n=0;n<t.numOperands;n++){let s=t.getOperand(n);if(s.definingOp&&s.definingOp.hasSideEffects())return!1}return!0}_computeMemorySaved(e){if(!(e.type instanceof I))return 0;let t=e.type.sizeInBytes();return t===X?0:t}_operandExtensionCost(e,t,n){let s=0;for(let o=0;o<e.numOperands;o++){let i=e.getOperand(o),a=0;for(let u of i.uses()){let p=n.get(u.user)||0;p>a&&(a=p)}if(t>a&&i.type instanceof I){let u=i.type.sizeInBytes();u!==X&&(s+=u)}}return s}_estimateRecomputeCost(e){let t=ne.get(e.opName);if(!t||t.isOpaque)return 1/0;if(t.getFlops){let n=t.getFlops(e);if(n>0)return n}if(t.isElementwise||t.isBroadcast||t.isView){let n=0;for(let s=0;s<e.numResults;s++){let o=e.getResult(s).type;if(o instanceof I){let i=o.numel();if(i!==X){n=i;break}}}return n||1}if(t.isReduction)for(let n=0;n<e.numOperands;n++){let s=e.getOperand(n).type;if(s instanceof I){let o=s.numel();if(o!==X)return o}}return 1/0}_rematerialize(e,t,n){let{value:s,definingOp:o}=t,i=[];for(let c of s.uses())i.push({user:c.user,operandIndex:c.operandIndex});if(i.length<=1)return;i.sort((c,h)=>(n.get(c.user)||0)-(n.get(h.user)||0));let a=n.get(i[i.length-1].user)||0,u=this._operandExtensionCost(o,a,n),p=this._computeMemorySaved(s);if(!(u>=p))for(let c=1;c<i.length;c++){let h=i[c],f=h.user;if(!f.parentBlock)continue;let d=new Re(o.opName,[...o.operands],o.results.map(m=>m.type),new Map(o.attributes));f.parentBlock.insertBefore(d,f),f.replaceOperand(h.operandIndex,d.getResult(s.resultIndex))}}}});function oo(r){return ln(r.opName)}function yB(r){let e=0;for(let t=0;t<r.numResults;t++){let n=r.getResult(t).type;if(!n||!n.isFullyStatic)continue;let s=n.sizeInBytes();s>e&&(e=s)}return e}function W$(r){let e=r.definingOp;if(e&&e.opName==="broadcast"){let t=e.getOperand(0);e=t&&t.definingOp}if(e&&oo(e)){let t=e.getAttr("value");if(typeof t=="number")return t}return 0}function wT(r){if(r.opName!=="dot"&&r.opName!=="cublas_gemm")return null;let e=r.getOperand(0).type,t=r.getOperand(1).type,n=r.getResult(0).type;if(!e||!t||!n||e.dtype!=="f32"||t.dtype!=="f32"||n.dtype!=="f32")return null;let s=r.getAttr("lhs_batch")||[],o=r.getAttr("rhs_batch")||[];if(s.length>0||o.length>0)return null;let i=r.getAttr("lhs_contracting")||[],a=r.getAttr("rhs_contracting")||[];if(i.length!==1||a.length!==1||t.rank!==2||i[0]!==e.rank-1||a[0]!==0&&a[0]!==1||!e.isFullyStatic||!t.isFullyStatic||!n.isFullyStatic)return null;let u=r.getOperand(0).definingOp,p=r.getOperand(1).definingOp;if(u&&oo(u)||p&&oo(p))return null;let c=a[0]===1,h=e.shape[e.rank-1];if((c?t.shape[1]:t.shape[0])!==h)return null;let d=1;for(let g=0;g<e.rank-1;g++)d*=e.shape[g];let m=c?t.shape[0]:t.shape[1];return d<=0||m<=0||h<=0?null:{M:d,N:m,K:h,transB:c}}function _B(r){return{labelOf:l(e=>r.get(e),"labelOf"),canMerge:l((e,t)=>Math.max(e.maxBuf||0,yB(t))<=K$,"canMerge"),onAttach:l((e,t)=>{e.maxBuf=Math.max(e.maxBuf||0,yB(t))},"onAttach")}}function dy(r,e,t){let n=r.opSet,s=Ba(r.ops),{inputs:o,outputs:i,constDefs:a}=_i(n,s,{pullConstants:!0,isConstant:oo});for(let f of o)if(!f.type||!f.type.isFullyStatic)return null;for(let f of i)if(!f.type||!f.type.isFullyStatic)return null;let u=new un(e,o.map(f=>f.type),i.map(f=>f.type)),p=new Map;for(let f=0;f<o.length;f++)p.set(o[f],u.args[f]);for(let f of a)u.entryBlock.pushOp(f.clone(p));for(let f of s)u.entryBlock.pushOp(f.clone(p));let c=i.map(f=>p.get(f));if(c.some(f=>f===void 0))return null;u.entryBlock.pushOp(new Re("return",c,[]));let h=r.ops.length===1&&t.has(r.ops[0])?r.ops[0]:null;return{part:r,subFunc:u,inputs:o,outputs:i,dotOp:h}}function sf(r){if(H$.has(r.opName))return!0;if(r.regions)for(let e of r.regions){let t=e.entryBlock;if(t){for(let n of t.ops())if(sf(n))return!0}}return!1}function Y$(r){let e=0;for(let t=0;t<r.numResults;t++){let n=r.getResult(t).type;if(!n||!n.isFullyStatic)continue;let s=1;for(let o of n.shape)s*=o;s>e&&(e=s)}return e}function bB(r,e=256){if(r.functionCount!==1)return!1;let t=r.functions().next().value,n=new Map,s=l(o=>{if(!o)return 0;let i=n.get(o);if(i!==void 0)return i;n.set(o,0);let a=sf(o)?Y$(o):0;for(let u=0;u<o.numOperands;u++){let p=s(o.getOperand(u).definingOp);p>a&&(a=p)}return n.set(o,a),a},"maxBoundaryInSubtree");for(let o of t.ops())if(!(qt(o.opName)||!sf(o))){for(let i=0;i<o.numOperands;i++)if(s(o.getOperand(i).definingOp)>e)return!0}return!1}function wB(r,e,t){let n=new Map,s=0,o=l(d=>{let m=n.get(d);return m===void 0&&(m=s++,n.set(d,m)),m},"getSlot");for(let d of r.args)o(d);for(let d of t)for(let m of d.outputs)o(m);let i=[];for(let d of r.args)i.push(o(d));let a=[],u=new Set;for(let d=0;d<e.numOperands;d++){let m=e.getOperand(d),g=i.length,y=m.isBlockArgument&&m.isBlockArgument(),_=m.definingOp&&oo(m.definingOp);if(!y&&!_&&n.has(m)){let b=n.get(m);if(!u.has(b)){u.add(b),i.push(b);continue}i.push(s++),a.push({pos:g,kind:"copy",srcSlot:b});continue}if(y){i.push(s++),a.push({pos:g,kind:"copy",srcSlot:o(m)});continue}if(_){i.push(s++),a.push({pos:g,kind:"const",value:W$(m)});continue}return null}let p=[];for(let d of t){let m=[];for(let y of d.inputs){let _=n.get(y);if(_===void 0)return null;m.push(_)}let g=d.outputs.map(y=>n.get(y));p.push({name:d.subFunc.name,inputSlots:m,outputSlots:g})}let c=new Set(i),h=[],f=new Set;for(let[d,m]of n)if(!(c.has(m)||f.has(m))){if(f.add(m),!d.type||!d.type.isFullyStatic)return null;h.push({slot:m,shape:[...d.type.shape],dtype:d.type.dtype})}return{plan:{numSlots:s,argSlots:i,intermediates:h,steps:p,returnFixups:a}}}function Ti(r,e=2){if(r.functionCount!==1)return null;let t=r.functions().next().value,n=t.getReturnOp();if(!n)return null;let s=[],o=new Map,i=0;for(let g of t.ops())qt(g.opName)||oo(g)||(sf(g)?o.set(g,"boundary#"+i++):o.set(g,"native"),s.push(g));if(i<e||s.length===0)return null;let{partitions:a,preds:u}=tf(s,_B(o));if(a.length<2)return null;let p=hT(a,u);if(!p)return null;let c=t.name,h=[],f=new Map,d=0;for(let g of p){let y=dy(g,c+"_p"+d++,f);if(!y)return null;h.push(y)}let m=wB(t,n,h);if(!m)return null;r.removeFunction(t.name);for(let g of h)r.addFunction(g.subFunc);return{plan:m.plan}}function TB(r){if(r.functionCount!==1)return null;let e=r.functions().next().value,t=e.getReturnOp();if(!t)return null;let n=[],s=new Map,o=new Map,i=0,a=0;for(let y of e.ops()){if(qt(y.opName)||oo(y))continue;let _=wT(y);_?(s.set(y,"cublas#"+i),o.set(y,_),i++):sf(y)?s.set(y,"boundary#"+a++):s.set(y,"native"),n.push(y)}if(i+a===0||n.length===0)return null;let{partitions:u,preds:p}=tf(n,_B(s));if(u.length<2)return null;let c=hT(u,p);if(!c)return null;let h=e.name,f=[],d=0;for(let y of c){let _=dy(y,h+"_p"+d++,o);if(!_)return null;f.push(_)}let m=wB(e,t,f);if(!m)return null;let g=new Map;for(let y of f){if(!y.dotOp)continue;let _=o.get(y.dotOp),b=y.inputs.indexOf(y.dotOp.getOperand(0)),w=y.inputs.indexOf(y.dotOp.getOperand(1)),x=y.outputs.indexOf(y.dotOp.getResult(0));b<0||w<0||x<0||g.set(y.subFunc.name,{M:_.M,N:_.N,K:_.K,transB:_.transB,aIdx:b,bIdx:w,cIdx:y.inputs.length+x})}if(g.size===0)return null;r.removeFunction(e.name);for(let y of f)r.addFunction(y.subFunc);return{plan:m.plan,cublasInfos:g}}var K$,H$,my=M(()=>{"use strict";xs();sr();nf();Tr();l(oo,"isConstantOp");K$=32*1024;l(yB,"maxResultBytes");l(W$,"constScalarOf");l(wT,"cublasDotInfo");l(_B,"bufferLimitedConfig");l(dy,"materializePartition");H$=new Set(["dot","fused_dot_epilogue","cublas_gemm","reduce","conv","quantized_conv","scaled_dot_product_attention"]);l(sf,"containsBoundaryOp");l(Y$,"maxBoundaryResultBytes");l(bB,"hasDependentBoundaries");l(wB,"buildExecutionPlan");l(Ti,"splitGraphForNative");l(TB,"splitGraphForCublas")});function SB(r){return xB.push(r),r}var xB,gy,kB=M(()=>{"use strict";nn();xB=[];l(SB,"registerBackendOpRewrite");gy=class extends Ge{static{l(this,"BackendOpRewritePass")}constructor(e={}){super(e.name||"BackendOpRewritePass"),this.config=e,this.rewrites=e.rewrites||xB}run(e){let t=!1;for(let n of[...e.ops()]){let s=n.parentBlock;if(s){for(let o of this.rewrites)if(o.match(n,this.config)){o.build(n,s,this.config),t=!0;break}}}return t?oe.CHANGED:oe.UNCHANGED}}});var J$,yy,vB=M(()=>{"use strict";sr();my();kB();J$=SB({name:"dot->cublas_gemm",match:l(r=>r.opName==="dot"&&r.numOperands===2&&wT(r)!==null,"match"),build:l((r,e)=>{let t=r.results.map(s=>s.type),n=new Re("cublas_gemm",[r.getOperand(0),r.getOperand(1)],t,new Map(r.attributes),null);e.insertBefore(n,r),r.replaceAllResultsWith(n.results),r.erase()},"build")}),yy=class extends gy{static{l(this,"CublasRewritePass")}constructor(e={}){super({...e,name:"CublasRewritePass",rewrites:[J$]})}}});function NB(r,e,{cudaMatmulChain:t=!1,context:n=null}={}){let s=n?a=>n.passesForPhase(a,r,e):a=>Vg(a,r,e),o=[];for(let a of s("pre"))o.push(a);if(o.push(new Vu(e)),o.push(new zu("canonicalize",[new ia,new Ug({fastMath:r.optimization.fastMath}),new Kg,new Wg,new no],r.optimization.maxSimplifyIterations)),r.optimization.layout&&e&&(o.push(new uy({target:e})),o.push(new no)),r.quantization.enabled&&(o.push(new cy({...r.quantization,target:e})),o.push(new ia),o.push(new no)),r.matmulBackend!=="cublas"&&(r.fusion.epilogue!==void 0?r.fusion.epilogue:e&&e.enableEpilogueFusion)&&(o.push(new Xg({target:e})),o.push(new no)),r.fusion.enabled){let a=r.fusion,u=a.launchOverheadUs??Z$;a.strategy==="dominator"?o.push(new ny({target:e,...a})):a.strategy==="priority"?(o.push(new oy({target:e,cost:{launchOverheadUs:u},...a})),o.push(new rf({maxFusionSize:e?.maxFusionSize,...a}))):(o.push(new Zg({target:e,cost:{launchOverheadUs:u},...a})),o.push(new ey({maxFusionSize:e?.maxFusionSize,...a})),o.push(new rf({maxFusionSize:e?.maxFusionSize,...a}))),o.push(new no)}if(r.matmulBackend==="cublas"&&o.push(new yy),r.optimization.rematerialization){let a={...r.optimization.rematConfig};a.memoryBudget===void 0&&e&&e.memoryBudgetBytes>0&&(a.memoryBudget=e.memoryBudgetBytes),o.push(new hy(a))}for(let a of s("post"))o.push(a);return o}var Z$,AB=M(()=>{"use strict";am();Dw();qO();WO();HO();$w();JO();QO();XO();eB();rB();iB();pB();dB();Lw();gB();vB();uT();Z$=5;l(NB,"buildGraphPipeline")});var Wn,xi=M(()=>{"use strict";Wn=class{static{l(this,"PrimFuncPass")}constructor(e,t=null){this.name=e,this.phase=t||e,this.snapshotPoint=null,this.trace=null}begin(e){}run(e,t){throw new Error("PrimFuncPass.run not implemented")}end(e){}}});var _y,TT,Ra,xT=M(()=>{"use strict";De();_y=["iterCount","depth","parallelLoops","vectorizedLoops","unrolledLoops","threadBoundLoops","serialLoops","threadBlockSize","gridSize","underReduction","numMathOps","numExternCalls","numReads","numWrites","stride1Accesses","stridedAccesses","reuseCount","touchedBytes","arithmeticIntensity","vectorized","parallelized","innermostExtent"],TT=class{static{l(this,"ScheduleFeatures")}constructor(e){this.numLoops=e.numLoops||0,this.numBlocks=e.numBlocks||0,this.totalIterations=e.totalIterations||0,this.maxLoopDepth=e.maxLoopDepth||0,this.numParallelLoops=e.numParallelLoops||0,this.numVectorizedLoops=e.numVectorizedLoops||0,this.numUnrolledLoops=e.numUnrolledLoops||0,this.numThreadBound=e.numThreadBound||0,this.numSerialLoops=e.numSerialLoops||0,this.totalBufferBytes=e.totalBufferBytes||0,this.numBufferReads=e.numBufferReads||0,this.numBufferWrites=e.numBufferWrites||0,this.numMathOps=e.numMathOps||0,this.numExternCalls=e.numExternCalls||0,this.arithmeticIntensity=e.arithmeticIntensity||0,this.innermostExtent=e.innermostExtent||0,this.outermostExtent=e.outermostExtent||0,this.hasReduction=e.hasReduction||!1,this.reductionDepth=e.reductionDepth||0,this.threadBlockSize=e.threadBlockSize||0,this.gridSize=e.gridSize||0,this.strideOneAccesses=e.strideOneAccesses||0,this.nonStrideOneAccesses=e.nonStrideOneAccesses||0}},Ra=class r{static{l(this,"FeatureExtractor")}static extract(e){let t={numLoops:0,numBlocks:0,totalIterations:1,maxLoopDepth:0,currentDepth:0,numParallelLoops:0,numVectorizedLoops:0,numUnrolledLoops:0,numThreadBound:0,numSerialLoops:0,totalBufferBytes:0,buffersSeen:new Set,numBufferReads:0,numBufferWrites:0,numMathOps:0,numExternCalls:0,hasReduction:!1,reductionDepth:0,threadBlockSize:1,gridSize:1,innermostExtent:0,outermostExtent:0,loopExtents:[],strideOneAccesses:0,nonStrideOneAccesses:0};r._visitIterative(e.body,t);let n=t.totalBufferBytes,s=t.numMathOps+t.numExternCalls;return t.arithmeticIntensity=n>0?s/n:0,new TT(t)}static extractStatements(e){let t=[],n=[],s=0,o=[{node:e.body,action:"enter"}];for(;o.length>0;){let{node:i,action:a}=o.pop();if(i){if(a==="leaveFor"){n.pop();continue}if(a==="leaveBlock"){s--;continue}switch(i.type){case"ForNode":n.push(i),o.push({node:null,action:"leaveFor"}),o.push({node:i.body,action:"enter"});break;case"BlockNode":i.initBody&&(s++,o.push({node:null,action:"leaveBlock"}),o.push({node:i.initBody,action:"enter"})),o.push({node:i.body,action:"enter"});break;case"SeqNode":for(let u=i.stmts.length-1;u>=0;u--)o.push({node:i.stmts[u],action:"enter"});break;case"AllocateNode":case"LetStmtNode":o.push({node:i.body,action:"enter"});break;case"IfThenElseNode":i.elseBody&&o.push({node:i.elseBody,action:"enter"}),o.push({node:i.thenBody,action:"enter"});break;case"BufferStoreNode":t.push(r._statementVector(i,n,s));break}}}return t}static _statementVector(e,t,n){let s=1,o=0,i=0,a=0,u=0,p=0,c=1,h=1;for(let T of t){let k=T.extent&&T.extent.type==="IntImmNode"?T.extent.value:1;switch(s*=k,T.kind){case F.PARALLEL:o++;break;case F.VECTORIZED:i++;break;case F.UNROLLED:a++;break;case F.THREAD_BINDING:u++,T.threadTag&&T.threadTag.startsWith("threadIdx")?c*=k:T.threadTag&&T.threadTag.startsWith("blockIdx")&&(h*=k);break;default:p++;break}}let f={math:0,extern:0};r._countExpr(e.value,f);let d=t.map(T=>T.loopVar.name),m=[{buffer:e.buffer,indices:e.indices}];r._collectLoads(e.value,m);let g=0,y=0,_=0,b=0;for(let T of m){let k=T.buffer&&T.buffer.sizeInBytes?T.buffer.sizeInBytes():0;k>0&&(b+=k);let A=T.indices&&T.indices.length>0?T.indices[T.indices.length-1]:null;A&&A.type==="VariableNode"?g++:y++;let N=new Set;if(T.indices)for(let v of T.indices)r._collectVars(v,N);for(let v of d)N.has(v)||_++}let w=t.length>0?t[t.length-1]:null,x=w&&w.extent&&w.extent.type==="IntImmNode"?w.extent.value:0,S={iterCount:s,depth:t.length,parallelLoops:o,vectorizedLoops:i,unrolledLoops:a,threadBoundLoops:u,serialLoops:p,threadBlockSize:c,gridSize:h,underReduction:n>0?1:0,numMathOps:f.math,numExternCalls:f.extern,numReads:m.length-1,numWrites:1,stride1Accesses:g,stridedAccesses:y,reuseCount:_,touchedBytes:b,arithmeticIntensity:b>0?(f.math+f.extern)/b:0,vectorized:i>0?1:0,parallelized:o+u>0?1:0,innermostExtent:x};return _y.map(T=>S[T]||0)}static _countExpr(e,t){if(!(!e||typeof e!="object"))switch(e.type){case"MathOpNode":t.math++,r._countExpr(e.a,t),e.b&&r._countExpr(e.b,t);break;case"CompareNode":t.math++,r._countExpr(e.a,t),r._countExpr(e.b,t);break;case"CallExternNode":t.extern++;for(let n of e.args)r._countExpr(n,t);break;default:break}}static _collectLoads(e,t){if(!(!e||typeof e!="object")){if(e.type==="BufferLoadNode"){t.push({buffer:e.buffer,indices:e.indices});return}if(e.a&&r._collectLoads(e.a,t),e.b&&r._collectLoads(e.b,t),e.args)for(let n of e.args)r._collectLoads(n,t)}}static _collectVars(e,t){if(!(!e||typeof e!="object")){if(e.type==="VariableNode"){t.add(e.name);return}if(e.a&&r._collectVars(e.a,t),e.b&&r._collectVars(e.b,t),e.args)for(let n of e.args)r._collectVars(n,t);if(e.indices)for(let n of e.indices)r._collectVars(n,t)}}static _visitIterative(e,t){let n=[{node:e,action:"enter"}];for(;n.length>0;){let{node:s,action:o}=n.pop();if(s){if(o==="leave_for"){t.currentDepth--;continue}switch(s.type){case"ForNode":{t.numLoops++,t.currentDepth++,t.currentDepth>t.maxLoopDepth&&(t.maxLoopDepth=t.currentDepth);let i=s.extent.type==="IntImmNode"?s.extent.value:1;switch(t.loopExtents.push(i),t.numLoops===1&&(t.outermostExtent=i),t.innermostExtent=i,s.kind){case F.PARALLEL:t.numParallelLoops++;break;case F.VECTORIZED:t.numVectorizedLoops++;break;case F.UNROLLED:t.numUnrolledLoops++;break;case F.THREAD_BINDING:t.numThreadBound++,s.threadTag&&s.threadTag.startsWith("threadIdx")?t.threadBlockSize*=i:s.threadTag&&s.threadTag.startsWith("blockIdx")&&(t.gridSize*=i);break;default:t.numSerialLoops++;break}t.totalIterations*=i,n.push({node:null,action:"leave_for"}),n.push({node:s.body,action:"enter"});break}case"BlockNode":t.numBlocks++,s.initBody&&(t.hasReduction=!0,t.reductionDepth=t.currentDepth);for(let i of s.reads)r._visitBuffer(i.buffer,t);for(let i of s.writes)r._visitBuffer(i.buffer,t);n.push({node:s.body,action:"enter"}),s.initBody&&n.push({node:s.initBody,action:"enter"});break;case"SeqNode":for(let i=s.stmts.length-1;i>=0;i--)n.push({node:s.stmts[i],action:"enter"});break;case"AllocateNode":r._visitBuffer(s.buffer,t),n.push({node:s.body,action:"enter"});break;case"IfThenElseNode":s.elseBody&&n.push({node:s.elseBody,action:"enter"}),n.push({node:s.thenBody,action:"enter"});break;case"LetStmtNode":n.push({node:s.body,action:"enter"});break;case"BufferStoreNode":t.numBufferWrites++,r._visitBuffer(s.buffer,t),r._checkStride(s.buffer,s.indices,t),r._visitExpr(s.value,t);break}}}}static _visitBuffer(e,t){if(!e||t.buffersSeen.has(e))return;t.buffersSeen.add(e);let n=e.sizeInBytes();n>0&&(t.totalBufferBytes+=n)}static _visitExpr(e,t){if(e)switch(e.type){case"MathOpNode":t.numMathOps++,r._visitExpr(e.a,t),e.b&&r._visitExpr(e.b,t);break;case"CallExternNode":t.numExternCalls++;for(let n of e.args)r._visitExpr(n,t);break;case"BufferLoadNode":t.numBufferReads++,r._checkStride(e.buffer,e.indices,t);break;case"CompareNode":t.numMathOps++,r._visitExpr(e.a,t),r._visitExpr(e.b,t);break;default:break}}static _checkStride(e,t,n){if(!e||!t||t.length===0)return;let s=t[t.length-1];for(;s&&s.type==="MathOpNode"&&s.b&&s.b.type==="IntImmNode";)if(s.op==="+"&&s.b.value===0)s=s.a;else if(s.op==="*"&&s.b.value===1)s=s.a;else break;s&&s.type==="VariableNode"?n.strideOneAccesses++:n.nonStrideOneAccesses++}}});function OB(r,e){if(r.length===0)return 0;let t=0;for(let n of r)t+=e[n];return t/r.length}function ST(r,e,t,n,s,o,i){let a=t.length;if(n>=s||a<o*2)return{leaf:OB(t,e)};let u=r[0].length,p=0,c=0,h=new Set(t);for(let g of t){let y=e[g];p+=y,c+=y*y}let f=null,d=null;for(let g=0;g<u;g++){let y=i[g].filter(w=>h.has(w)),_=0,b=0;for(let w=0;w<y.length-1;w++){let x=e[y[w]];_+=x,b+=x*x;let S=r[y[w]][g],T=r[y[w+1]][g];if(S===T)continue;let k=w+1,A=a-k;if(k<o||A<o)continue;let N=p-_,v=c-b,O=b-_*_/k+(v-N*N/A);(!f||O<f.cost)&&(f={f:g,thr:(S+T)/2,sIdx:w,cost:O},d=y)}}if(!f)return{leaf:OB(t,e)};let m=d;return{f:f.f,thr:f.thr,left:ST(r,e,m.slice(0,f.sIdx+1),n+1,s,o,i),right:ST(r,e,m.slice(f.sIdx+1),n+1,s,o,i)}}function Q$(r){let e=r.length,t=e>0?r[0].length:0,n=new Array(t),s=[];for(let o=0;o<e;o++)s.push(o);for(let o=0;o<t;o++)n[o]=s.slice().sort((i,a)=>r[i][o]-r[a][o]);return n}function BB(r,e){for(;r&&r.leaf===void 0;)r=e[r.f]<=r.thr?r.left:r.right;return r?r.leaf:0}var of,RB=M(()=>{"use strict";l(OB,"leafValue");l(ST,"buildTree");l(Q$,"buildFeatureOrder");l(BB,"predictTree");of=class r{static{l(this,"GradientBoostedTrees")}constructor(e={}){this.numTrees=e.numTrees??60,this.maxDepth=e.maxDepth??3,this.lr=e.lr??.1,this.minSamples=e.minSamples??1,this.trees=null,this.base=0}fit(e,t){let n=e.length;if(n===0)return;let s=0;for(let p of t)s+=p;s/=n;let o=new Array(n).fill(s),i=[];for(let p=0;p<n;p++)i.push(p);let a=Q$(e),u=[];for(let p=0;p<this.numTrees;p++){let c=new Array(n);for(let f=0;f<n;f++)c[f]=t[f]-o[f];let h=ST(e,c,i,0,this.maxDepth,this.minSamples,a);for(let f=0;f<n;f++)o[f]+=this.lr*BB(h,e[f]);u.push(h)}this.trees=u,this.base=s}predict(e){if(!this.trees)return 0;let t=this.base;for(let n of this.trees)t+=this.lr*BB(n,e);return t}serialize(){return{trees:this.trees,base:this.base,lr:this.lr}}static deserialize(e){let t=new r({lr:e.lr});return t.trees=e.trees,t.base=e.base,t}}});function MB(r){let e=r[0].length,t=new Array(e+1).fill(0);for(let n of r)for(let s=0;s<e;s++){let o=n[s]||0;tF.has(s)?o>t[s]&&(t[s]=o):t[s]+=o}for(let n of nF)n<e&&(t[n]/=r.length);return t[e]=r.length,t}var X$,eF,tF,nF,kT,rF,by,vl,Nl,vT=M(()=>{"use strict";xT();RB();X$=new Set(["depth","threadBlockSize","gridSize","underReduction","vectorized","parallelized","innermostExtent"]),eF=new Set(["arithmeticIntensity"]),tF=new Set(_y.map((r,e)=>X$.has(r)?e:-1).filter(r=>r>=0)),nF=new Set(_y.map((r,e)=>eF.has(r)?e:-1).filter(r=>r>=0));l(MB,"aggregateStatements");kT=class{static{l(this,"CostEstimate")}constructor(e,t){this.score=e,this.breakdown=t}},rF={parallelism:2,vectorization:1.5,memoryCoalescing:2,occupancy:1,arithmeticIntensity:1,loopOverhead:-.5,codeSize:-.3},by=class{static{l(this,"AnalyticalCostModel")}constructor(e,t={}){this.target=e,this._weights={...rF,...e&&e.costModelWeights?e.costModelWeights:{},...t.weights||{}}}estimate(e){let t=Ra.extract(e);return this.estimateFromFeatures(t)}score(e){return this.estimate(e).score}estimateFromFeatures(e){let t=this._scoreParallelism(e),n=this._scoreVectorization(e),s=this._scoreMemoryAccess(e),o=this._scoreOccupancy(e),i=this._scoreIntensity(e),a=this._scoreOverhead(e),u=this._scoreCodeSize(e),p=this._weights,c=p.parallelism*t+p.vectorization*n+p.memoryCoalescing*s+p.occupancy*o+p.arithmeticIntensity*i+p.loopOverhead*a+p.codeSize*u;return new kT(c,{parallelism:t,vectorization:n,memoryCoalescing:s,occupancy:o,arithmeticIntensity:i,loopOverhead:a,codeSize:u})}_scoreParallelism(e){if(this.target.isGPU()){let n=e.threadBlockSize*e.gridSize,s=this.target.maxParallelism();return Math.min(1,n/Math.max(s*.1,1))}return e.numParallelLoops/Math.max(e.numLoops,1)}_scoreVectorization(e){return e.numLoops===0?0:this.target.isGPU()?e.strideOneAccesses/Math.max(e.strideOneAccesses+e.nonStrideOneAccesses,1):e.numVectorizedLoops>0?Math.min(1,e.innermostExtent/this.target.vectorWidth):0}_scoreMemoryAccess(e){let t=e.strideOneAccesses+e.nonStrideOneAccesses;return t===0?1:e.strideOneAccesses/t}_scoreOccupancy(e){if(!this.target.isGPU())return 1;if(e.threadBlockSize===0)return 0;let t=this.target.warpSize,n=Math.ceil(e.threadBlockSize/t),s=Math.floor(this.target.maxThreadsPerBlock/t);return Math.min(1,n/s)}_scoreIntensity(e){let t=this.target.supportsFloat16?1.5:1;return Math.min(1,e.arithmeticIntensity*10*t)}_scoreOverhead(e){return e.numSerialLoops/Math.max(e.numLoops,1)}_scoreCodeSize(e){return Math.min(1,(e.numMathOps+e.numExternCalls)/256)}compare(e,t){return this.estimate(e).score-this.estimate(t).score}},vl=class r{static{l(this,"LearnedCostModel")}constructor(e=null,t={}){this.opts={numTrees:t.numTrees??60,maxDepth:t.maxDepth??3,lr:t.lr??.1,minSamples:t.minSamples??1},this._gbt=e?of.deserialize(e):null,this._X=[],this._Y=[]}addSample(e,t){!e||e.length===0||Number.isFinite(t)&&(this._X.push(MB(e)),this._Y.push(t))}train(){if(this._X.length===0)return;let e=new of(this.opts);e.fit(this._X,this._Y),this._gbt=e}predict(e){return!this._gbt||!e||e.length===0?0:this._gbt.predict(MB(e))}get trained(){return this._gbt!==null}get sampleCount(){return this._X.length}serialize(){return{gbt:this._gbt?this._gbt.serialize():null,numSamples:this._X.length}}static deserialize(e){return new r(e&&e.gbt?e.gbt:null)}},Nl=class{static{l(this,"GuidedCostModel")}constructor(e,t,n={}){this.analytical=e,this.learned=t,this.confidenceSamples=n.confidenceSamples??8}_learnedConfident(){return this.learned&&this.learned.trained&&this.learned.sampleCount>=this.confidenceSamples}score(e){return this._learnedConfident()?this.learned.predict(Ra.extractStatements(e)):this.analytical.score(e)}}});function*IB(r){let e=[r];for(;e.length>0;){let t=e.pop();if(t){if(yield t,t.body&&e.push(t.body),t.stmts)for(let n of t.stmts)e.push(n);t.thenBody&&e.push(t.thenBody),t.elseBody&&e.push(t.elseBody),t.initBody&&e.push(t.initBody)}}}function af(r,e){for(let t of IB(r))if(t.type==="BlockNode"&&t.name===e)return t;return null}function io(r){let e=[];for(let t of IB(r))t.type==="BlockNode"&&e.push(t.name);return e}function uf(r,e){let t=dt(r,e);if(!t)return{spatial:0,reduction:0,reads:0,hasReduction:!1};let n=0,s=0;for(let o of t.loops)t.reductionLoopVars.has(o.loopVar.name)?s++:n++;return{spatial:n,reduction:s,reads:t.readBuffers.length,hasReduction:t.hasReduction}}var Al=M(()=>{"use strict";ro();l(IB,"walkStmts");l(af,"findBlock");l(io,"collectAllBlockNames");l(uf,"analyzeBlockStructure")});function LB(r){return r.isGPU()?oF:sF}function NT(r){let e=0,t=0;for(let[n,s]of r.order)n==="S"?e=Math.max(e,s+1):t=Math.max(t,s+1);return{spatialLevels:e,reductionLevels:t}}var sF,oF,EB,AT=M(()=>{"use strict";sF={name:"mlt_cpu",order:[["S",0],["S",1],["S",2],["S",3],["R",0]],roles:{S0:"parallelize",S3:"vectorize"}},oF={name:"mlt_gpu",order:[["S",0],["S",1],["S",2],["R",0]],roles:{S0:"blockIdx",S1:"threadIdx",S2:"unroll"}},EB={name:"ssrsrs_cpu",order:[["S",0],["S",1],["R",0],["S",2],["R",1],["S",3]],roles:{S0:"parallelize",S3:"vectorize",R1:"unroll"}};l(LB,"getTileStructure");l(NT,"levelCounts")});var os,vr,wy=M(()=>{"use strict";os=class{static{l(this,"SearchVariable")}constructor(e,t){this.name=e,this.candidates=t}sample(e){return this.candidates[e(this.candidates.length)]}},vr=class{static{l(this,"ScheduleSketch")}constructor(e,t,n){this.name=e,this.variables=t,this._apply=n}instantiate(e){return(t,n,s)=>{this._apply(t,n,s,e)}}sampleParams(e){let t={};for(let n of this.variables)t[n.name]=n.sample(e);return t}}});function iF(r){let e=[];for(let t=1;t*t<=r;t++){if(r%t!==0)continue;e.push(t);let n=r/t;n!==t&&e.push(n)}return e.sort((t,n)=>t-n),e}function aF(r,e){let t=Math.min(r.length,e.length);for(let n=0;n<t;n++)if(r[n]!==e[n])return r[n]-e[n];return r.length-e.length}function uF(r,e){let t=new Set,n=[];r.sort(aF);for(let i of r){let a=i.join(",");t.has(a)||(t.add(a),n.push(i))}if(n.length<=e)return n;let s=new Set,o=[];for(let i=0;i<e;i++){let a=Math.floor(i*(n.length-1)/(e-1));s.has(a)||(s.add(a),o.push(n[a]))}return o}function OT(r,e,t=48){if(!Number.isFinite(r)||r<1||e<=1)return[[r]];let n=[],s=t*8,o=l((i,a,u)=>{if(!(n.length>=s)){if(a===e-1){n.push([...u,i]);return}for(let p of iF(i))if(u.push(p),o(i/p,a+1,u),u.pop(),n.length>=s)return}},"rec");return o(r,0,[]),uF(n,t)}var CB=M(()=>{"use strict";l(iF,"divisorsOf");l(aF,"cmpTuple");l(uF,"selectDiverse");l(OT,"enumerateFactorizations")});function Ty(r){return r.extent&&r.extent.type==="IntImmNode"?r.extent.value:null}function PB(r,e,t,n){let s=[],o=t;for(let i=0;i<n.length-1;i++){let a=r.getLoops(e).find(h=>h.loopVar.name===o);if(!a)return null;let u=1;for(let h=i+1;h<n.length;h++)u*=n[h];let[p,c]=r.split(a,u);s.push(p.loopVar.name),o=c.loopVar.name}return s.push(o),s}function lF(r,e,t,n,s){let o=l(i=>r.getLoops(e).find(a=>a.loopVar.name===i),"find");for(let[i,a]of t.order){let u=t.roles[i+a];if(!u)continue;let p=i==="S"?n:s;if(u==="parallelize"){let c=p[0]&&a<p[0].length?o(p[0][a]):null;c&&r.parallelize(c)}else if(u==="vectorize"){let c=p[p.length-1],h=c&&a<c.length?o(c[a]):null;h&&r.vectorize(h)}else if(u==="unroll")for(let c of p){let h=a<c.length?o(c[a]):null;h&&r.unroll(h)}else(u==="blockIdx"||u==="threadIdx")&&p.forEach((c,h)=>{if(h>=DB.length)return;let f=a<c.length?o(c[a]):null;f&&r.bindThread(f,`${u}.${DB[h]}`)})}}function $B(r,e,t,n,s,o){let i=[];for(let h=0;h<n.length;h++){let f=PB(r,e,n[h],o[`s${h}`]);if(!f)return;i.push(f)}let a=[];for(let h=0;h<s.length;h++){let f=PB(r,e,s[h],o[`r${h}`]);if(!f)return;a.push(f)}let u=[];for(let[h,f]of t.order){let d=h==="S"?i:a;for(let m of d)f<m.length&&u.push(m[f])}let p=new Map(r.getLoops(e).map(h=>[h.loopVar.name,h])),c=u.map(h=>p.get(h)).filter(Boolean);c.length>=2&&r.reorder(...c),lF(r,e,t,i,a)}function FB(r){let e=r.loops.filter(n=>!r.reductionLoopVars.has(n.loopVar.name)),t=r.loops.filter(n=>r.reductionLoopVars.has(n.loopVar.name));return{spatialLoops:e,reductionLoops:t}}function zB(r,e,t,n){let s=[];return r.forEach((o,i)=>s.push(new os(`s${i}`,OT(Ty(o),t)))),e.forEach((o,i)=>s.push(new os(`r${i}`,OT(Ty(o),n)))),s}function VB(r,e){let{spatialLevels:t,reductionLevels:n}=NT(e),{spatialLoops:s,reductionLoops:o}=FB(r);if(s.length===0)return null;for(let p of[...s,...o])if(Ty(p)===null)return null;let i=zB(s,o,t,n),a=s.map(p=>p.loopVar.name),u=o.map(p=>p.loopVar.name);return new vr(e.name,i,(p,c,h,f)=>{$B(p,c,e,a,u,f)})}function GB(r,e){let{spatialLevels:t,reductionLevels:n}=NT(e),{spatialLoops:s,reductionLoops:o}=FB(r);if(s.length===0||o.length===0)return null;for(let p of[...s,...o])if(Ty(p)===null)return null;let i=zB(s,o,t,n),a=s.map(p=>p.loopVar.name),u=o.map(p=>p.loopVar.name);return new vr(e.name,i,(p,c,h,f)=>{p.decomposeReduction(c),$B(p,`${c}_upd`,e,a,u,f)})}var DB,jB=M(()=>{"use strict";wy();CB();AT();DB=["x","y","z"];l(Ty,"staticExtent");l(PB,"multiLevelSplit");l(lF,"applyRoles");l($B,"tileBlock");l(FB,"splitLoops");l(zB,"tilingVariables");l(VB,"createMultiLevelTilingSketch");l(GB,"createSSRSRSTilingSketch")});function UB(r){return Math.min(r&&r.maxThreadsPerBlock||256,256)}function pF(r){let e=new Set;for(let t=2;t*t<=r;t++){if(r%t!==0)continue;e.add(t);let n=r/t;n>1&&n<r&&e.add(n)}return[...e].sort((t,n)=>t-n)}function KB(r){return new vr("fused",[],(e,t,n,s)=>{e.fuseConsumer(t,r);let o=e.getLoops(t);o.length>0&&e.parallelize(o[0])})}function WB(r){let e=r.loops.filter(i=>r.reductionLoopVars.has(i.loopVar.name));if(e.length!==1)return null;let t=e[0],n=t.extent&&t.extent.type==="IntImmNode"?t.extent.value:null;if(n===null)return null;let s=pF(n);if(s.length===0)return null;let o=t.loopVar.name;return new vr("rfactor",[new os("rf_factor",s)],(i,a,u,p)=>{i.rfactor(a,o,p.rf_factor);let c=i.getLoops(`${a}_rf_p`);c.length>0&&i.parallelize(c[0])})}function HB(){return new vr("elementwise_cpu",[new os("vector_width",cF)],(r,e,t,n)=>{let s=r.getLoops(e);if(s.length===0)return;if(s.length===1){let a=s[0].extent;if(a.type==="IntImmNode"&&a.value>=n.vector_width*2){let[u,p]=r.split(s[0],n.vector_width);r.parallelize(u),r.vectorize(p)}else r.parallelize(s[0]);return}r.parallelize(s[0]);let o=s[s.length-1],i=o.extent;if(i.type==="IntImmNode"&&i.value>=n.vector_width){let[,a]=r.split(o,n.vector_width);r.vectorize(a)}})}function YB(){return new vr("elementwise_gpu",[new os("block_size",qB)],(r,e,t,n)=>{let s=r.getLoops(e);if(s.length===0)return;let o=s[0];for(let c=1;c<s.length;c++){let f=r.getLoops(e).find(d=>d.loopVar.name===s[c].loopVar.name);f&&o.body===f&&(o=r.fuseLoops(o,f))}let i=o.extent;if(i.type!=="IntImmNode"){r.bindThread(o,"threadIdx.x");return}let a=i.value,u=Math.min(t.maxThreadsPerBlock,1024);if(o0(r.func)&&a<=u){r.bindThread(o,"threadIdx.x");return}let p=Math.min(n.block_size,UB(t));if(a>p){let[c,h]=r.split(o,p);r.bindThread(c,"blockIdx.x"),r.bindThread(h,"threadIdx.x")}else r.bindThread(o,"threadIdx.x")})}function JB(){return new vr("reduction_cpu",[],(r,e,t,n)=>{let s=r.getLoops(e);s.length>0&&r.parallelize(s[0])})}function ZB(){return new vr("reduction_gpu",[new os("block_size",qB)],(r,e,t,n)=>{let s=r.getLoops(e);if(s.length===0)return;let o=dt(r.func,e),i=s.filter(c=>!o||!Ju(c,o));if(i.length===0)return;let a=i[0];for(let c=1;c<i.length;c++){let f=r.getLoops(e).find(d=>d.loopVar.name===i[c].loopVar.name);f&&a.body===f&&(a=r.fuseLoops(a,f))}let u=a.extent;if(u.type!=="IntImmNode"){r.bindThread(a,"threadIdx.x");return}let p=Math.min(n.block_size,UB(t));if(u.value>p){let[c,h]=r.split(a,p);r.bindThread(c,"blockIdx.x"),r.bindThread(h,"threadIdx.x")}else r.bindThread(a,"threadIdx.x")})}var qB,cF,QB=M(()=>{"use strict";ro();wy();qB=[32,64,128,256,512,1024],cF=[1,2,4,8,16];l(UB,"gpuThreadCap");l(pF,"reductionDivisors");l(KB,"createFusedTilingSketch");l(WB,"createRfactorSketch");l(HB,"createElementwiseCPUSketch");l(YB,"createElementwiseGPUSketch");l(JB,"createReductionCPUSketch");l(ZB,"createReductionGPUSketch")});function xy(r){return gm(r,xy,(e,t,n)=>{switch(e.type){case"PrimFunc":return t.name=e.name,t.params=e.params,t.body=n(e.body),t.bufferMap=new Map(e.bufferMap),t.shapeParams=e.shapeParams,t.shapeParamMap=e.shapeParamMap instanceof Map?new Map(e.shapeParamMap):e.shapeParamMap,t._setChild("body",t.body),t;case"AllocateNode":return t.buffer=e.buffer,t.scope=e.scope,t.body=n(e.body),t._setChild("body",t.body),t;case"LetStmtNode":return t.variable=e.variable,t.value=n(e.value),t.body=n(e.body),t._setChild("body",t.body),t;case"WhileNode":return t.condVar=e.condVar,t.condBody=n(e.condBody),t.loopBody=n(e.loopBody),t._setChild("condBody",t.condBody),t._setChild("loopBody",t.loopBody),t;default:for(let s of Object.keys(e)){if(s==="_parent"||s==="_parentKey"||s==="_parentIdx")continue;let o=e[s];o instanceof Map?t[s]=new Map(o):Array.isArray(o)?t[s]=o.map(n):typeof o=="object"&&o!==null&&o.type?t[s]=n(o):t[s]=o}return t}})}function Lo(r){return xy(r)}function Ma(r){return xy(r)}function BT(r){return xy(r)}function XB(r,e,t){let n=t.get(e);if(!n)return null;let s=[],o=n._parent;for(;o&&o!==r;)o.type==="ForNode"&&s.push(o),o=o._parent;s.reverse();let i=BT(n);for(let p=s.length-1;p>=0;p--){let c=s[p],h=new de(c.loopVar,BT(c.min),BT(c.extent),c.kind,i,c.threadTag);h._setChild("body",i),i=h}let a=new Map;for(let p of n.reads)a.set(p.buffer.name,p.buffer);for(let p of n.writes)a.set(p.buffer.name,p.buffer);let u=[];for(let p of r.params)a.has(p.name)&&u.push(p);return new Jo("__tune_"+e,u,i,a,[])}var lf=M(()=>{"use strict";De();Ww();l(xy,"deepCloneIR");l(Lo,"clonePrimFunc");l(Ma,"cloneTensorIR");l(BT,"cloneBlockSubtree");l(XB,"extractBlockMini")});function ET(r,e){let t=[];for(let n=1;n<=e;n*=2)n>=r&&t.push(n);return t}function ff(r){if(!r)return null;let e=[];for(let t of r){if(!t||t.type!=="VariableNode")return null;e.push(t.name)}return e}function fF(r,e){let t=[r];for(;t.length>0;){let n=t.pop();if(!(!n||typeof n!="object")){if(n.type==="BufferStoreNode"&&n.buffer&&n.buffer.name===e&&n.value&&n.value.type==="MathOpNode"&&n.value.op==="+")return n;if(n.body&&t.push(n.body),n.stmts)for(let s of n.stmts)t.push(s);n.thenBody&&t.push(n.thenBody),n.elseBody&&t.push(n.elseBody)}}return null}function pf(r){if(!r||r.broadcastDims||r.offset&&r.offset!==0)return!1;let e=r.shape,t=r.strides;if(!t||t.length!==e.length)return!1;let n=1;for(let s=e.length-1;s>=0;s--){if(typeof e[s]!="number"||e[s]<=0||t[s]!==n)return!1;n*=e[s]}return!0}function LT(r,e){if(!dt(r,e))return null;let n=af(r.body,e);if(!n||n.reads.length<2||n.writes.length<1)return null;let s=n.writes[0].buffer;if(!s||s.shape.length<2)return null;let o=s.shape.length,i=fF(n.body,s.name);if(!i)return null;let a=ff(i.indices);if(!a||a.length!==o)return null;let u=i.value,p=l(T=>T&&T.type==="BufferLoadNode"&&T.buffer&&T.buffer.name===s.name,"isCLoad"),c=p(u.a)?u.b:p(u.b)?u.a:null;if(!c||c.type!=="MathOpNode"||c.op!=="*")return null;let h=[c.a,c.b];if(!h.every(T=>T&&T.type==="BufferLoadNode"&&T.buffer))return null;let f=a[o-1],d=a.slice(0,o-1),m=null,g=null;for(let T of h){let k=ff(T.indices);if(!k||k.length!==o)continue;let A=!0;for(let N=0;N<o-1;N++)if(k[N]!==d[N]){A=!1;break}if(A){m=T,g=k[o-1];break}}if(!m||g==null)return null;let y=h[0]===m?h[1]:h[0],_=ff(y.indices);if(!_)return null;let b=m.buffer,w=y.buffer;if(b.shape.length!==o||b.dtype!=="f32"||w.dtype!=="f32"||s.dtype!=="f32")return null;for(let T=0;T<o-1;T++)if(b.shape[T]!==s.shape[T])return null;let x=s.shape[o-1],S=b.shape[o-1];if(![x,S].every(T=>typeof T=="number"&&T>0))return null;if(_.length===2&&w.shape.length===2){let T;if(_[0]===g&&_[1]===f)T=!1;else if(_[0]===f&&_[1]===g)T=!0;else return null;if(T){if(w.shape[0]!==x||w.shape[1]!==S)return null}else if(w.shape[0]!==S||w.shape[1]!==x)return null;let k=1;for(let A=0;A<o-1;A++){let N=s.shape[A];if(typeof N!="number"||N<=0)return null;k*=N}return o===2?{A:b,B:w,C:s,M:k,N:x,K:S,transB:T,batch:1}:!pf(b)||!pf(s)?null:{A:new me(b.name,[k,S],b.dtype,b.scope),B:w,C:new me(s.name,[k,x],s.dtype,s.scope),M:k,N:x,K:S,transB:T,batch:1}}if(o>=3&&_.length===o&&w.shape.length===o){let T=a.slice(0,o-2);for(let B=0;B<o-2;B++)if(_[B]!==T[B])return null;let k;if(_[o-2]===g&&_[o-1]===f)k=!1;else if(_[o-2]===f&&_[o-1]===g)k=!0;else return null;for(let B=0;B<o-2;B++)if(w.shape[B]!==s.shape[B])return null;if(k){if(w.shape[o-2]!==x||w.shape[o-1]!==S)return null}else if(w.shape[o-2]!==S||w.shape[o-1]!==x)return null;let A=s.shape[o-2];if(typeof A!="number"||A<=0)return null;let N=1;for(let B=0;B<o-2;B++){let E=s.shape[B];if(typeof E!="number"||E<=0)return null;N*=E}if(!pf(b)||!pf(w)||!pf(s))return null;let v=new me(b.name,[N,A,S],b.dtype,b.scope),O=new me(w.name,k?[N,x,S]:[N,S,x],w.dtype,w.scope),R=new me(s.name,[N,A,x],s.dtype,s.scope);return{A:v,B:O,C:R,M:A,N:x,K:S,transB:k,batch:N}}return null}function sR(r,e,t=32){let n=r.maxThreadsPerBlock||1024,s=r.warpSize||32,o=r.sharedMemoryBytes||49152,i=r.registersPerThread||255,a=4,{M:u,N:p,K:c}=e,h=ET(2,8),f=ET(2,s/2),d=ET(s/4,s/2),m=2*s,g=Math.min(n,8*s),y=4*s,_=new Set,b=[];for(let w of h)for(let x of h)if(!(w*x>y)&&!(w*x+w+x+s>i))for(let S of f)for(let T of f){let k=T*S;if(k<m||k>g||k%s!==0)continue;let A=w*S,N=x*T;if(!(A>u||N>p||A>y||N>y))for(let v of d){if(v>c||(A*v+v*N)*a>o)continue;let R=`${A}_${N}_${v}_${w}_${x}`;_.has(R)||(_.add(R),b.push({BM:A,BN:N,BK:v,TM:w,TN:x,threads:k}))}}return b.sort((w,x)=>nR(x,s)-nR(w,s)),b.slice(0,t)}function nR(r,e){let t=r.TM*r.TN,n=-Math.abs(r.TM-r.TN),s=-Math.abs(Math.log2(r.BM)-Math.log2(r.BN)),o=r.threads>=4*e&&r.threads<=8*e?1:0,i=r.BK===e/4?1:0;return o*100+t*4+n*6+s*4+i}function Sy(r,e){if(!e)return null;let t=sR(r,e,256);return t.find(s=>s.BM===64&&s.BN===64&&s.BK===8&&s.TM===4&&s.TN===4)||(t.length>0?t[0]:null)}function hF(r,e,t,n){let s=Ma(r.storeValue);return ti(s,o=>o.type==="BufferLoadNode"&&o.buffer?o.buffer.name===r.inputName?e:r.scalarConsts.has(o.buffer.name)?Ma(r.scalarConsts.get(o.buffer.name)):o:o.type==="VariableNode"?o.name===r.iv0?Ma(t):o.name===r.iv1?Ma(n):o:o)}function CT(r,e,t=null){let{A:n,B:s,C:o,M:i,N:a,K:u,transB:p}=r,c=r.batch||1,{BM:h,BN:f,BK:d,TM:m,TN:g}=e,y=f/g,_=h/m,b=y*_,w=Math.ceil(u/d),x=h*d,S=d*f,T=Math.ceil(x/b),k=Math.ceil(S/b),A=i%h!==0,N=a%f!==0,v=u%d!==0,O=x%b!==0,R=S%b!==0,B=new me("rb_As",[x],"f32","shared"),E=new me("rb_Bs",[S],"f32","shared"),z=new me("rb_acc",[m*g],"f32","local"),D=new me("rb_af",[m],"f32","local"),J=new me("rb_bf",[g],"f32","local"),j=Vt("rb_bx"),U=Vt("rb_by"),P=Vt("rb_tx"),V=Vt("rb_ty"),he=Vt("rb_tid"),le=Vt("rb_brow"),Ie=Vt("rb_bcol"),G=Vt("rb_k0"),K=Vt("rb_bz"),ie=c>1,ze=l((wt,Fs)=>ie?[K,wt,Fs]:[wt,Fs],"gIdx"),Se=l((wt,Fs)=>Pn(Nr(wt,Ze(g)),Fs),"accIdx"),Qe=Vt("rb_im"),It=Vt("rb_in"),Jt=Co(Qe,m,Co(It,g,new Y(z,[Se(Qe,It)],RT()))),Gt=Vt("rb_la"),tt=Vt("rb_aidx"),ye=Pn(le,eR(tt,Ze(d))),He=Pn(G,tR(tt,Ze(d))),gn=new q(n,ze(ye,He));if(A||v){let wt=A?Ls(ye,Ze(i)):null;v&&(wt=wt?MT(wt,Ls(He,Ze(u))):Ls(He,Ze(u))),gn=new xe(wt,gn,RT())}let jt=new Y(B,[tt],gn);O&&(jt=new xe(Ls(tt,Ze(x)),jt));let Zt=IT(Gt,T,new Ye(tt,Pn(he,Nr(Gt,Ze(b))),jt)),Qt=Vt("rb_lb"),on=Vt("rb_bidx"),Jn=Pn(G,eR(on,Ze(f))),Bn=Pn(Ie,tR(on,Ze(f))),yr=new q(s,p?ze(Bn,Jn):ze(Jn,Bn));if(v||N){let wt=v?Ls(Jn,Ze(u)):null;N&&(wt=wt?MT(wt,Ls(Bn,Ze(a))):Ls(Bn,Ze(a))),yr=new xe(wt,yr,RT())}let Zn=new Y(E,[on],yr);R&&(Zn=new xe(Ls(on,Ze(S)),Zn));let pt=IT(Qt,k,new Ye(on,Pn(he,Nr(Qt,Ze(b))),Zn)),Qn=Vt("rb_kk"),mt=Vt("rb_fi"),Or=Vt("rb_fj"),hs=Co(mt,m,new Y(D,[mt],new q(B,[Pn(Nr(Pn(Nr(V,Ze(m)),mt),Ze(d)),Qn)]))),$s=Co(Or,g,new Y(J,[Or],new q(E,[Pn(Nr(Qn,Ze(f)),Pn(Nr(P,Ze(g)),Or))]))),Br=Vt("rb_mi"),Xn=Vt("rb_ni"),_r=Co(Br,m,Co(Xn,g,new Y(z,[Se(Br,Xn)],Pn(new q(z,[Se(Br,Xn)]),Nr(new q(D,[Br]),new q(J,[Xn])))))),fo=Co(Qn,d,new te([hs,$s,_r])),Rr=Vt("rb_kt"),Mr=new Ye(G,Nr(Rr,Ze(d)),new te([Zt,pt,new On,fo,new On])),ds=IT(Rr,w,Mr),Qr=Vt("rb_wm"),Mi=Vt("rb_wn"),Vo=Pn(Pn(le,Nr(V,Ze(m))),Qr),Go=Pn(Pn(Ie,Nr(P,Ze(g))),Mi),Ii=new q(z,[Se(Qr,Mi)]),Xl=t?t.outBuffer:o,Fa=t?hF(t,Ii,Vo,Go):Ii,Ei=new Y(Xl,ze(Vo,Go),Fa);if(A||N){let wt=A?Ls(Vo,Ze(i)):null;N&&(wt=wt?MT(wt,Ls(Go,Ze(a))):Ls(Go,Ze(a))),Ei=new xe(wt,Ei)}let Li=Co(Qr,m,Co(Mi,g,Ei)),ho=new te([Jt,ds,Li]),Ci=new Ve(z,"local",new Ve(D,"local",new Ve(J,"local",ho))),jo=new Ye(he,Pn(Nr(V,Ze(y)),P),new Ye(le,Nr(U,Ze(h)),new Ye(Ie,Nr(j,Ze(f)),Ci))),mo=Math.ceil(a/f),qo=Math.ceil(i/h),Di=cf(U,"blockIdx.y",qo,cf(j,"blockIdx.x",mo,cf(V,"threadIdx.y",_,cf(P,"threadIdx.x",y,jo))));return ie&&(Di=cf(K,"blockIdx.z",c,Di)),new Ve(B,"shared",new Ve(E,"shared",Di))}function dF(r){let e=new os("config_index",r.map((n,s)=>s)),t=new vr("matmul_register_block_gpu",[e],(n,s,o,i)=>{let a=LT(n.func,s);if(!a)return;let u=r[i.config_index];if(!u)return;let p=CT(a,u);n.func.body=p,n.func._setChild&&n.func._setChild("body",p),n.func.gpuRegisterBlocked=!0});return t.configs=r,t.enumerate=()=>r.map((n,s)=>({config_index:s})),t}function mF(r){let e=io(r.body),t=null;for(let s of e){let o=uf(r,s);if(o.hasReduction&&o.spatial>=2&&o.reads>=2){if(t)return null;t=s}}if(!t)return null;let n=LT(r,t);if(!n)return null;for(let s of e){if(s===t)continue;let o=dt(r,s);if(!o||o.hasReduction||o.readBuffers.length>0)return null;for(let i of o.writeBuffers)if(i!==n.C.name)return null}return{reductionBlock:t,dims:n}}function gF(r){let e=new Map,t=new Map;Ss(r,n=>{if(n.type==="BufferStoreNode"&&n.buffer){let s=n.buffer.name;t.set(s,(t.get(s)||0)+1),(!n.indices||n.indices.length===0)&&n.value&&(n.value.type==="FloatImmNode"||n.value.type==="IntImmNode")&&e.set(s,n.value)}});for(let n of[...e.keys()])(t.get(n)||0)!==1&&e.delete(n);return e}function yF(r,e){let t=null;return Ss(r,n=>{if(n.type==="BufferStoreNode"&&n.buffer&&n.buffer.name===e)return t=n,id}),t}function _F(r,e){let t=null,n=0;return Ss(r,s=>{s.type==="BufferLoadNode"&&s.buffer&&s.buffer.name===e&&(n++,t=s)}),n===1?t:null}function oR(r){let e=io(r.body),t=null;for(let y of e){let _=uf(r,y);if(_.hasReduction&&_.spatial>=2&&_.reads>=2){if(t)return null;t=y}}if(!t)return null;let n=LT(r,t);if(!n)return null;let s=n.C.name,o=new Set;for(let y of e){let _=dt(r,y);if(!_)return null;for(let b of _.writeBuffers)o.add(b)}let i=[];for(let y of e){if(y===t)continue;let _=dt(r,y);if(_.hasReduction)return null;if(_.readBuffers.length===0){if(_.writeBuffers.every(b=>b===s))continue;return null}i.push({name:y,info:_})}if(i.length===0)return{reductionBlock:t,dims:n,epilogue:null};if(i.length!==1||(n.batch||1)!==1)return null;let a=i[0];if(a.info.writeBuffers.length!==1)return null;let u=a.info.writeBuffers[0],p=gF(r.body),c=0;for(let y of a.info.readBuffers){if(y===s){c++;continue}if(o.has(y)&&!p.has(y))return null}if(c!==1)return null;let h=af(r.body,a.name);if(!h)return null;let f=yF(h.body,u);if(!f)return null;let d=ff(f.indices);if(!d||d.length!==2)return null;let m=_F(f.value,s);if(!m)return null;let g=ff(m.indices);return!g||g.length!==d.length||g.some((y,_)=>y!==d[_])?null:{reductionBlock:t,dims:n,epilogue:{outBuffer:f.buffer,storeValue:f.value,inputName:s,iv0:d[0],iv1:d[1],scalarConsts:p}}}function iR(r,e,t){let n=mF(r);if(!n)return null;let s=rR.get(r);if(s===void 0){let o=sR(t,n.dims);s=o.length>0?dF(o):null,rR.set(r,s)}return s?e===n.reductionBlock?[s]:[]:null}var Ze,RT,Vt,Pn,Nr,eR,tR,Ls,MT,IT,Co,cf,rR,ky=M(()=>{"use strict";De();xo();ro();wy();Al();Gr();lf();Ze=l(r=>new L(r),"I"),RT=l(()=>new fe(0),"FZERO"),Vt=l(r=>new Ne(r,"i32"),"IV"),Pn=l((r,e)=>In("+",r,e),"ADD"),Nr=l((r,e)=>In("*",r,e),"MUL"),eR=l((r,e)=>In("//",r,e),"DIV"),tR=l((r,e)=>In("%",r,e),"MOD"),Ls=l((r,e)=>new qe("lt",r,e),"LT"),MT=l((r,e)=>new C("&&",r,e),"AND"),IT=l((r,e,t)=>new de(r,Ze(0),Ze(e),F.SERIAL,t),"forS"),Co=l((r,e,t)=>new de(r,Ze(0),Ze(e),F.UNROLLED,t),"forU"),cf=l((r,e,t,n)=>new de(r,Ze(0),Ze(t),F.THREAD_BINDING,n,e),"forT");l(ET,"pow2Range");l(ff,"plainVars");l(fF,"findAccStore");l(pf,"isContiguousRowMajor");l(LT,"matmulTileDims");l(sR,"enumerateRegisterBlockConfigs");l(nR,"goodness");l(Sy,"pickFixedConfig");l(hF,"foldEpilogue");l(CT,"buildRegisterBlockedMatmul");l(dF,"createMatmulRegisterBlockGPUSketch");l(mF,"analyzePureMatmul");l(gF,"collectScalarConstBuffers");l(yF,"findEpilogueStore");l(_F,"singleBufferLoad");l(oR,"analyzeMatmulEpilogue");rR=new WeakMap;l(iR,"richMatmulSketches")});function bF(r,e){let t=[r];for(;t.length;){let n=t.pop();if(!(!n||typeof n!="object")&&(n.type==="BlockNode"&&e(n),n.body&&t.push(n.body),n.initBody&&t.push(n.initBody),n.thenBody&&t.push(n.thenBody),n.elseBody&&t.push(n.elseBody),Array.isArray(n.stmts)))for(let s of n.stmts)t.push(s)}}function df(r){let e=[];bF(r.body,n=>{e.push({name:n.name,node:n,reads:(n.reads||[]).map(s=>s.buffer&&s.buffer.name).filter(Boolean),writes:(n.writes||[]).map(s=>s.buffer&&s.buffer.name).filter(Boolean)})});let t=new Map;for(let n of e)for(let s of n.reads)t.has(s)||t.set(s,[]),t.get(s).push(n.name);return{blocks:e,consumersByBuf:t,byName:new Map(e.map(n=>[n.name,n]))}}function DT(r){let e=[];for(let t of r){if(!t||t.type!=="VariableNode")return null;e.push(t.name)}return e}function hf(r,e,t){if(!r||typeof r!="object")return t;r.type==="BufferLoadNode"&&r.buffer&&r.buffer.name===e&&t.push(r);for(let n of["a","b","expr","value","condition","thenBody","elseBody","body","initBody"])r[n]&&hf(r[n],e,t);if(Array.isArray(r.args))for(let n of r.args)hf(n,e,t);if(Array.isArray(r.indices))for(let n of r.indices)hf(n,e,t);if(Array.isArray(r.stmts))for(let n of r.stmts)hf(n,e,t);return t}function Ol(r,e){if(!r||typeof r!="object")return e;r.type==="BufferLoadNode"&&r.buffer&&r.buffer.name&&e.add(r.buffer.name);for(let t of["a","b","expr","value","condition","thenBody","elseBody","body","initBody"])r[t]&&Ol(r[t],e);if(Array.isArray(r.args))for(let t of r.args)Ol(t,e);if(Array.isArray(r.indices))for(let t of r.indices)Ol(t,e);if(Array.isArray(r.stmts))for(let t of r.stmts)Ol(t,e);return e}function vy(r,e,t,n){let s=e.byName.get(t);if(!s||s.writes.length!==1)return null;let o=s.writes[0],i=[...new Set((e.consumersByBuf.get(o)||[]).filter(w=>w!==t))];if(i.length!==1)return null;let a=i[0],u=n(r,t),p=n(r,a);if(!u||!p||p.hasReduction)return null;let c=e.byName.get(a);if(!c||c.writes.length!==1||c.writes[0]===o)return null;let h=new Set(s.reads);if(Ol(s.node.body,h),Ol(s.node.initBody,h),h.has(c.writes[0]))return null;let f=s.node.body&&s.node.body.type==="BufferStoreNode"?s.node.body:null,d=c.node.body&&c.node.body.type==="BufferStoreNode"?c.node.body:null;if(!f||!d)return null;let m=u.loops.filter(w=>!u.reductionLoopVars.has(w.loopVar.name)).map(w=>w.loopVar.name),g=DT(f.indices);if(!g||g.join(",")!==m.join(","))return null;let y=p.loops.map(w=>w.loopVar.name);if(y.length!==m.length)return null;let _=DT(d.indices);if(!_||_.join(",")!==y.join(","))return null;let b=hf(d.value,o,[]);if(b.length===0)return null;for(let w of b){let x=DT(w.indices);if(!x||x.join(",")!==_.join(","))return null}return a}var PT=M(()=>{"use strict";l(bF,"walkBlocks");l(df,"buildBlockDAG");l(DT,"indexVarNames");l(hf,"collectLoads");l(Ol,"collectLoadedBuffers");l(vy,"findFusibleConsumer")});function $T(r){return r.isGPU()?ZB():JB()}function wF(r){return r.isGPU()?YB():HB()}function TF(r,e,t,n){let s=dt(r,e);if(!s)return[$T(t)];let o=[],i=VB(s,LB(t));if(i&&o.push(i),t.kind===Ot.CPU){let a=GB(s,EB);a&&o.push(a);let u=WB(s);u&&o.push(u);let p=n?vy(r,n,e,dt):null;p&&o.push(KB(p))}return o.push($T(t)),o}function FT(r,{priority:e=100}={}){Ny.some(t=>t.derive===r.derive&&t.matches===r.matches)||(Ny.push({matches:r.matches,derive:r.derive,priority:e}),Ny.sort((t,n)=>t.priority-n.priority))}function aR(r,e,t,n={}){if(n.richGpu&&t.isGPU()){let i=iR(r,e,t);if(i!==null)return i}if(t.kind!==Ot.CPU&&!t.isGPU())return[];let s=uf(r,e),o=n.dag||df(r);for(let i of Ny)if(i.matches(s,t))return i.derive(r,e,t,o);return[]}var Ny,uR=M(()=>{"use strict";Ks();ro();Al();AT();jB();QB();ky();PT();l($T,"reductionSketch");l(wF,"elementwiseSketch");l(TF,"deriveMultiLevel");Ny=[];l(FT,"registerSketchRule");FT({matches:l(r=>r.hasReduction&&r.spatial>=1&&r.reads>=2,"matches"),derive:TF},{priority:10});FT({matches:l(r=>r.hasReduction,"matches"),derive:l((r,e,t)=>[$T(t)],"derive")},{priority:20});FT({matches:l(()=>!0,"matches"),derive:l((r,e,t)=>[wF(t)],"derive")},{priority:30});l(aR,"deriveSketches")});function zT(r,e,t,n,s={}){return aR(r,e,t,s)}var lR=M(()=>{"use strict";uR();l(zT,"getSketchesForBlock")});function Ay(r){let e=new Map,t=[r];for(;t.length>0;){let n=t.pop();if(n){if(n.type==="BlockNode"&&e.set(n.name,n),n.body&&t.push(n.body),n.stmts)for(let s of n.stmts)t.push(s);n.thenBody&&t.push(n.thenBody),n.elseBody&&t.push(n.elseBody),n.initBody&&t.push(n.initBody)}}return e}function Oy(r,e,t,n=null){let s=n||Ay(r.body),o=s.get(e)||null,i=[];if(o){let a=[];for(let p of o.reads)a.push(`${p.buffer.shape.join("x")}:${p.buffer.dtype}`);for(let p of o.writes)a.push(`${p.buffer.shape.join("x")}:${p.buffer.dtype}`);i.push(a.join(","));let u=[];if(Yt(o.body,u),o.initBody&&Yt(o.initBody,u),i.push(u.join(";")),o.writes.length===1){let p=o.writes[0].buffer.name,c=[];for(let h of xF(s).get(p)||[]){if(h===o)continue;let f=[];Yt(h.body,f),h.initBody&&Yt(h.initBody,f),c.push(f.join(";"))}c.length>0&&(c.sort(),i.push(`consumers:${c.join("|")}`))}}return i.push(t.name),i.push(t.kind),SF(i.join("|"))}function xF(r){if(r.__readersByBuffer)return r.__readersByBuffer;let e=new Map;for(let t of r.values()){if(!t.reads)continue;let n=new Set;for(let s of t.reads)s.buffer&&n.add(s.buffer.name);for(let s of n){let o=e.get(s);o||(o=[],e.set(s,o)),o.push(t)}}return r.__readersByBuffer=e,e}function Yt(r,e){if(!(!r||typeof r!="object"))switch(r.type){case"BufferStoreNode":if(e.push("store"),r.indices)for(let t of r.indices)Yt(t,e);Yt(r.value,e);return;case"BufferLoadNode":if(e.push(`load:${r.buffer.shape.join("x")}:${r.buffer.dtype}`),r.indices)for(let t of r.indices)Yt(t,e);return;case"MathOpNode":e.push(`math:${r.op}`),Yt(r.a,e),r.b&&Yt(r.b,e);return;case"CallExternNode":e.push(`call:${r.externName}`);for(let t of r.args)Yt(t,e);return;case"CompareNode":e.push(`cmp:${r.direction}`),Yt(r.a,e),Yt(r.b,e);return;case"CastNode":e.push(`cast:${r.fromDtype}->${r.toDtype}`),Yt(r.expr,e);return;case"IfThenElseNode":Yt(r.condition,e),Yt(r.thenBody,e),r.elseBody&&Yt(r.elseBody,e);return;case"SeqNode":for(let t of r.stmts)Yt(t,e);return;case"ForNode":Yt(r.body,e);return;case"BlockNode":r.initBody&&Yt(r.initBody,e),Yt(r.body,e);return;case"LetStmtNode":Yt(r.value,e),Yt(r.body,e);return;default:return}}function SF(r){let e=2166136261;for(let t=0;t<r.length;t++)e^=r.charCodeAt(t),e=e*16777619>>>0;return e.toString(16).padStart(8,"0")}var VT=M(()=>{"use strict";l(Ay,"buildBlockMap");l(Oy,"computeWorkloadKey");l(xF,"readersByBuffer");l(Yt,"collectBlockOps");l(SF,"fnv1a")});function kF(r,e){let t=r.medianMs!=null?1:0,n=e.medianMs!=null?1:0;return t!==n?n-t:t===1?r.medianMs-e.medianMs:e.score-r.score}var cR,mf,gf,pR=M(()=>{"use strict";VT();cR="mlfw-codegen-1",mf=class{static{l(this,"TuningRecord")}constructor(e,t,n,s,o,i){this.workloadKey=e,this.sketchName=t,this.params=n,this.score=s,this.traceData=o,this.version=i,this.timestamp=Date.now(),this.medianMs=null,this.minMs=null}};l(kF,"rankRecords");gf=class r{static{l(this,"TuningDatabase")}constructor(e=1){this.version=e,this._records=new Map}computeWorkloadKey(e,t,n,s=null){return Oy(e,t,n,s)}store(e,t){let n=this._records.get(e);n||(n=[],this._records.set(e,n)),n.push(t),n.sort(kF),n.length>10&&(n.length=10)}lookup(e){let t=this._records.get(e);return!t||t.length===0?null:t[0]}lookupTopK(e,t=5){let n=this._records.get(e);return n?n.slice(0,t):[]}has(e){return this._records.has(e)&&this._records.get(e).length>0}get size(){let e=0;for(let[,t]of this._records)e+=t.length;return e}serialize(){let e=[];for(let[t,n]of this._records)for(let s of n)e.push({workloadKey:s.workloadKey,sketchName:s.sketchName,params:s.params,score:s.score,traceData:s.traceData,version:s.version,timestamp:s.timestamp,medianMs:s.medianMs,minMs:s.minMs});return{version:this.version,codegenVersion:cR,entries:e}}static deserialize(e){let t=new r(e.version);if(e.codegenVersion!==void 0&&e.codegenVersion!==cR)return t;for(let n of e.entries){let s=new mf(n.workloadKey,n.sketchName,n.params,n.score,n.traceData,n.version);s.timestamp=n.timestamp,s.medianMs=n.medianMs??null,s.minMs=n.minMs??null,t.store(n.workloadKey,s)}return t}saveToFile(e,t){return t.writeFile(e,JSON.stringify(this.serialize())),e}static loadFromFile(e,t){return t.exists(e)?r.deserialize(JSON.parse(t.readFile(e))):new r}clear(){this._records.clear()}}});function Si(r,e){if(!r)return 0;if(r.type==="ForNode"){let t=r.extent&&r.extent.type==="IntImmNode"?r.extent.value:1,n=r.threadTag?e:e*t;return Si(r.body,n)}if(r.type==="BlockNode")return Math.max(e,Si(r.body,e),r.initBody?Si(r.initBody,e):0);if(r.type==="SeqNode"){let t=e;for(let n of r.stmts)t=Math.max(t,Si(n,e));return t}return r.type==="IfThenElseNode"?Math.max(Si(r.thenBody,e),r.elseBody?Si(r.elseBody,e):e):r.type==="AllocateNode"||r.type==="LetStmtNode"?Si(r.body,e):e}function fR(r,e=.1){let t=r.slice().sort((g,y)=>g-y),n=t.length;if(n===0)return{median:0,min:0,trimmedMean:0,cv:0};let s=t[n>>1],o=t[0],i=Math.floor(n*e),a=i,u=n-i>a?n-i:n,p=0,c=0;for(let g=a;g<u;g++)p+=t[g],c++;let h=c>0?p/c:s,f=0;for(let g=a;g<u;g++){let y=t[g]-h;f+=y*y}let d=c>1?Math.sqrt(f/(c-1)):0,m=h>0?d/h:0;return{median:s,min:o,trimmedMean:h,cv:m}}var vF,By,Ry,hR=M(()=>{"use strict";Oc();vF=1e6;l(Si,"maxSerialTripCount");By=class{static{l(this,"BenchmarkResult")}constructor(e,t,n,s,o=null,i=0){this.medianMs=e,this.minMs=t,this.trimmedMeanMs=o??e,this.cv=i,this.samples=n,this._totalBytes=s}get throughputGBs(){return this.minMs<=0||!this._totalBytes?0:this._totalBytes/(this.minMs*1e6)}};l(fR,"robustStats");Ry=class{static{l(this,"BenchmarkRunner")}constructor(e,t={}){this.target=e,this.warmup=t.warmup??3,this.repeat=t.repeat??10,this.minRepeatMs=t.minRepeatMs??0,this.maxCv=t.maxCv??0,this.maxReMeasures=t.maxReMeasures??1,this.measurer=t.measurer||null,this._warn=t.warn||null,this._bufferCache=new Map}_record(e,t){this._warn&&this._warn(e,null,t)}_getOrAllocBuffers(e){let t=0,n=[];for(let[,i]of e.bufferMap){let a=Math.max(i.numel(),1);n.push(a);let u=i.sizeInBytes();u>0&&(t+=u)}let s=n.join(","),o=this._bufferCache.get(s);o||(o=n.map(i=>new Float32Array(i)),this._bufferCache.set(s,o));for(let i of o)for(let a=0;a<i.length;a++)i[a]=Math.random()*2-1;return{buffers:o,totalBytes:t}}run(e){if(!e||!e.body)return null;if(!this.target.isCPU())return this.measurer?this._runMeasured(e):null;let t=new Ws(this.target),n;try{n=t.compile(e)}catch(p){return this._record("benchmark-compile",p),null}if(n.metadata.kind!=="js")return null;let s;try{s=new Function("return "+n.source)()}catch(p){return this._record("benchmark-construct-fn",p),null}let{buffers:o,totalBytes:i}=this._getOrAllocBuffers(e);for(let p=0;p<this.warmup;p++)try{s(...o)}catch(c){return this._record("benchmark-warmup-run",c),null}let a=[],u=null;for(let p=0;p<=this.maxReMeasures&&(this._collect(s,o,a),u=fR(a),!(this.maxCv<=0||u.cv<=this.maxCv));p++);return new By(u.median,u.min,a,i,u.trimmedMean,u.cv)}_runMeasured(e){if(e.shapeParams&&e.shapeParams.length>0||Si(e.body,1)>vF)return null;let t;try{t=new Ws(this.target).compile(e)}catch(a){return this._record("measured-compile",a),null}let n=[],s=0;for(let[,a]of e.bufferMap){let u=Math.max(a.sizeInBytes(),1);n.push(u),s+=u}let o;try{o=this.measurer(t,n,[],{warmup:this.warmup,repeat:this.repeat})}catch(a){return this._record("measurer",a),null}if(!o||o.length===0)return null;let i=fR(o);return new By(i.median,i.min,o,s,i.trimmedMean,i.cv)}_collect(e,t,n){let s=0,o=this.repeat*3;for(let i=0;i<o&&(i<this.repeat||s<this.minRepeatMs);i++){let a=performance.now();e(...t);let u=performance.now()-a;n.push(u),s+=u}}}});var My,dR=M(()=>{"use strict";My=class{static{l(this,"Deadline")}constructor(e=1/0,t=null){this._clock=t||(()=>performance.now()),this._endAt=e===1/0||e==null?1/0:this._clock()+e}get expired(){return this._clock()>=this._endAt}remainingMs(){return this._endAt===1/0?1/0:Math.max(0,this._endAt-this._clock())}}});function mR(r){let e=null;for(let w of r.ops())if(w.opName==="conv"){if(e)return null;e=w}else if(w.opName!=="return")return null;if(!e||(e.getAttr("groups")||1)!==1)return null;let t=e.getAttr("input_layout"),n=e.getAttr("kernel_layout");if(t&&t!=="NCHW"||n&&n!=="OIHW")return null;let s=e.getOperand(0).type,o=e.getOperand(1).type,i=e.getResult(0).type;if(!s||!o||!i||s.shape.length!==4||o.shape.length!==4||i.shape.length!==4||s.dtype!=="f32"||o.dtype!=="f32"||i.dtype!=="f32")return null;let a=e.getAttr("strides")||[1,1],u=e.getAttr("padding")||[[0,0],[0,0]],p=e.getAttr("dilation")||[1,1],c=s.shape[0],h=s.shape[1],f=s.shape[2],d=s.shape[3],m=o.shape[0],g=o.shape[2],y=o.shape[3],_=i.shape[2],b=i.shape[3];return![c,h,f,d,m,g,y,_,b].every(w=>typeof w=="number"&&w>0)||o.shape[1]!==h||i.shape[1]!==m||i.shape[0]!==c?null:{N:c,Cin:h,H:f,W:d,O:m,Kh:g,Kw:y,Oh:_,Ow:b,sH:a[0],sW:a[1],pH:u[0][0],pW:u[1][0],dH:p[0],dW:p[1]}}function NF(r,e,t){let{weight:n,input:s,output:o}=r,{N:i,Cin:a,H:u,W:p,O:c,Kh:h,Kw:f,Oh:d,Ow:m,sH:g,sW:y,pH:_,pW:b,dH:w,dW:x}=e,{BM:S,BN:T,BK:k,TM:A,TN:N}=t,v=c,O=i*d*m,R=a*h*f,B=h*f,E=d*m,z=T/N,D=S/A,J=z*D,j=Math.ceil(R/k),U=S*k,P=k*T,V=Math.ceil(U/J),he=Math.ceil(P/J),le=new me("ig_As",[U],"f32","shared"),Ie=new me("ig_Bs",[P],"f32","shared"),G=new me("ig_acc",[A*N],"f32","local"),K=new me("ig_af",[A],"f32","local"),ie=new me("ig_bf",[N],"f32","local"),ze=Ae("ig_bx"),Se=Ae("ig_by"),Qe=Ae("ig_tx"),It=Ae("ig_ty"),Jt=Ae("ig_tid"),Gt=Ae("ig_brow"),tt=Ae("ig_bcol"),ye=Ae("ig_k0"),He=l((Cr,Q_)=>we(Te(Cr,$(N)),Q_),"accIdx"),gn=Ae("ig_im"),jt=Ae("ig_in"),Zt=mn(gn,A,mn(jt,N,new Y(G,[He(gn,jt)],yf()))),Qt=v%S!==0,on=R%k!==0,Jn=O%T!==0,Bn=U%J!==0,yr=P%J!==0,Zn=Ae("ig_la"),pt=Ae("ig_aidx"),Qn=we(Gt,kn(pt,$(k))),mt=we(ye,sn(pt,$(k))),Or=kn(mt,$(B)),hs=sn(mt,$(B)),$s=kn(hs,$(f)),Br=sn(hs,$(f)),Xn=new q(n,[Qn,Or,$s,Br]);if(Qt||on){let Cr=Qt?dr(Qn,$(v)):null;on&&(Cr=Cr?ao(Cr,dr(mt,$(R))):dr(mt,$(R))),Xn=new xe(Cr,Xn,yf())}let _r=new Y(le,[pt],Xn);Bn&&(_r=new xe(dr(pt,$(U)),_r));let fo=Iy(Zn,V,new Ye(pt,we(Jt,Te(Zn,$(J))),_r)),Rr=Ae("ig_lb"),Mr=Ae("ig_bidx"),ds=we(ye,kn(Mr,$(T))),Qr=we(tt,sn(Mr,$(T))),Mi=kn(ds,$(B)),Vo=sn(ds,$(B)),Go=kn(Vo,$(f)),Ii=sn(Vo,$(f)),Xl=kn(Qr,$(E)),Fa=sn(Qr,$(E)),Ei=kn(Fa,$(m)),Li=sn(Fa,$(m)),ho=Ey(we(Te(Ei,$(g)),Te(Go,$(w))),$(_)),Ci=Ey(we(Te(Li,$(y)),Te(Ii,$(x))),$(b)),jo=new q(s,[Xl,Mi,ho,Ci]),mo=ao(ao(Ly(ho,$(0)),dr(ho,$(u))),ao(Ly(Ci,$(0)),dr(Ci,$(p))));on&&(mo=ao(mo,dr(ds,$(R)))),Jn&&(mo=ao(mo,dr(Qr,$(O)))),jo=new xe(mo,jo,yf());let qo=new Y(Ie,[Mr],jo);yr&&(qo=new xe(dr(Mr,$(P)),qo));let Di=Iy(Rr,he,new Ye(Mr,we(Jt,Te(Rr,$(J))),qo)),wt=Ae("ig_kk"),Fs=Ae("ig_fi"),ec=Ae("ig_fj"),q_=mn(Fs,A,new Y(K,[Fs],new q(le,[we(Te(we(Te(It,$(A)),Fs),$(k)),wt)]))),U_=mn(ec,N,new Y(ie,[ec],new q(Ie,[we(Te(wt,$(T)),we(Te(Qe,$(N)),ec))]))),za=Ae("ig_mi"),Va=Ae("ig_ni"),K_=mn(za,A,mn(Va,N,new Y(G,[He(za,Va)],we(new q(G,[He(za,Va)]),Te(new q(K,[za]),new q(ie,[Va])))))),W_=mn(wt,k,new te([q_,U_,K_])),ms=Ae("ig_kt"),Xr=new Ye(ye,Te(ms,$(k)),new te([fo,Di,new On,W_,new On])),br=Iy(ms,j,Xr),er=Ae("ig_wm"),Ir=Ae("ig_wn"),Uo=we(we(Gt,Te(It,$(A))),er),zs=we(we(tt,Te(Qe,$(N))),Ir),Er=kn(zs,$(E)),Lr=sn(zs,$(E)),go=kn(Lr,$(m)),Fn=sn(Lr,$(m)),Ko=new Y(o,[Er,Uo,go,Fn],new q(G,[He(er,Ir)]));if(Qt||Jn){let Cr=Qt?dr(Uo,$(v)):null;Jn&&(Cr=Cr?ao(Cr,dr(zs,$(O))):dr(zs,$(O))),Ko=new xe(Cr,Ko)}let yo=mn(er,A,mn(Ir,N,Ko)),H_=new te([Zt,br,yo]),nh=new Ve(G,"local",new Ve(K,"local",new Ve(ie,"local",H_))),Y_=new Ye(Jt,we(Te(It,$(z)),Qe),new Ye(Gt,Te(Se,$(S)),new Ye(tt,Te(ze,$(T)),nh))),J_=Math.ceil(O/T),Z_=Math.ceil(v/S),rh=ki(Se,"blockIdx.y",Z_,ki(ze,"blockIdx.x",J_,ki(It,"threadIdx.y",D,ki(Qe,"threadIdx.x",z,Y_))));return new Ve(le,"shared",new Ve(Ie,"shared",rh))}function gR(r,e,t){if(!yR(r,{M:e.O,N:e.N*e.Oh*e.Ow,K:e.Cin*e.Kh*e.Kw},t))return!1;let{BM:n,BN:s,BK:o,TM:i,TN:a}=t,u=e.O,p=e.N*e.Oh*e.Ow,c=e.Cin*e.Kh*e.Kw;if(u%n!==0||p%s!==0||c%o!==0||[n,s,o,i,a].some(f=>f%GT!==0))return!1;let h=n/i*(s/a);return!(n*o%(GT*h)!==0||o*s%h!==0)}function AF(r,e,t){let{weight:n,input:s,output:o}=r,{N:i,Cin:a,H:u,W:p,O:c,Kh:h,Kw:f,Oh:d,Ow:m,sH:g,sW:y,pH:_,pW:b,dH:w,dW:x}=e,{BM:S,BN:T,BK:k,TM:A,TN:N}=t,v=GT,O=c,R=i*d*m,B=a*h*f,E=h*f,z=d*m,D=u*p,J=a*u*p,j=T/N,U=S/A,P=j*U,V=B/k,he=S*k,le=k*T,Ie=he/P,G=le/P,K=Ie/v,ie=k/v,ze=(d-1)*g+(h-1)*w-_,Se=(m-1)*y+(f-1)*x-b,Qe=_>0||ze>=u,It=b>0||Se>=p,Jt=new me(n.name,[O*B>0?O*B:1],n.dtype,n.scope),Gt=new me(s.name,[i*J>0?i*J:1],s.dtype,s.scope),tt=new me("iv_As",[2*he],"f32","shared");tt.align16=!0;let ye=new me("iv_Bs",[2*le],"f32","shared");ye.align16=!0;let He=new me("iv_acc",[A*N],"f32","local"),gn=new me("iv_af",[A],"f32","local");gn.align16=!0;let jt=new me("iv_bf",[N],"f32","local");jt.align16=!0;let Zt=new me("iv_ra",[Ie],"f32","local");Zt.align16=!0;let Qt=new me("iv_rb",[G],"f32","local"),on=Ae("iv_bx"),Jn=Ae("iv_by"),Bn=Ae("iv_tx"),yr=Ae("iv_ty"),Zn=Ae("iv_tid"),pt=Ae("iv_brow"),Qn=Ae("iv_bcol"),mt=l((ms,Xr)=>we(Te(ms,$(N)),Xr),"accIdx"),Or=0,hs=Ae("iv_im"),$s=Ae("iv_in"),Br=mn(hs,A,mn($s,N,new Y(He,[mt(hs,$s)],yf()))),Xn=l(ms=>{let Xr=Or++,br=Ae("iv_la"+Xr),er=Ae("iv_c"+Xr),Ir=kn(er,$(ie)),Uo=Te(sn(er,$(ie)),$(v)),zs=mn(br,K,new Ye(er,we(Zn,Te(br,$(P))),new iu(Zt,Te(br,$(v)),Jt,we(Te(we(pt,Ir),$(B)),we(ms,Uo)),v))),Er=Ae("iv_lb"+Xr),Lr=Ae("iv_e"+Xr),go=kn(Lr,$(T)),Fn=sn(Lr,$(T)),Ko=we(Qn,Fn),yo=we(ms,go),H_=kn(yo,$(E)),nh=sn(yo,$(E)),Y_=kn(nh,$(f)),J_=sn(nh,$(f)),Z_=kn(Ko,$(z)),rh=sn(Ko,$(z)),Cr=kn(rh,$(m)),Q_=sn(rh,$(m)),X_=Ey(we(Te(Cr,$(g)),Te(Y_,$(w))),$(_)),eb=Ey(we(Te(Q_,$(y)),Te(J_,$(x))),$(b)),_E=we(we(we(Te(Z_,$(J)),Te(H_,$(D))),Te(X_,$(p))),eb),tb=new q(Gt,[_E]),Ga=null;if(Qe&&(Ga=ao(Ly(X_,$(0)),dr(X_,$(u)))),It){let xS=ao(Ly(eb,$(0)),dr(eb,$(p)));Ga=Ga?ao(Ga,xS):xS}Ga&&(tb=new xe(Ga,tb,yf()));let bE=mn(Er,G,new Ye(Lr,we(Zn,Te(Er,$(P))),new Y(Qt,[Er],tb)));return new te([zs,bE])},"prefetch"),_r=l((ms,Xr)=>{let br=Or++,er=Ae("iv_la"+br),Ir=Ae("iv_c"+br),Uo=kn(Ir,$(ie)),zs=Te(sn(Ir,$(ie)),$(v)),Er=[];for(let yo=0;yo<v;yo++)Er.push(new Y(tt,[we(ms,we(Te(we(zs,$(yo)),$(S)),Uo))],new q(Zt,[we(Te(er,$(v)),$(yo))])));let Lr=mn(er,K,new Ye(Ir,we(Zn,Te(er,$(P))),new te(Er))),go=Ae("iv_lb"+br),Fn=Ae("iv_e"+br),Ko=mn(go,G,new Ye(Fn,we(Zn,Te(go,$(P))),new Y(ye,[we(Xr,Fn)],new q(Qt,[go]))));return new te([Lr,Ko])},"commit"),fo=l((ms,Xr)=>{let br=Or++,er=Ae("iv_kk"+br),Ir=[],Uo=we(ms,we(Te(er,$(S)),Te(yr,$(A))));for(let Fn=0;Fn<A/v;Fn++)Ir.push(new iu(gn,$(Fn*v),tt,we(Uo,$(Fn*v)),v));let zs=we(Xr,we(Te(er,$(T)),Te(Bn,$(N))));for(let Fn=0;Fn<N/v;Fn++)Ir.push(new iu(jt,$(Fn*v),ye,we(zs,$(Fn*v)),v));let Er=Ae("iv_mi"+br),Lr=Ae("iv_ni"+br),go=mn(Er,A,mn(Lr,N,new Y(He,[mt(Er,Lr)],we(new q(He,[mt(Er,Lr)]),Te(new q(gn,[Er]),new q(jt,[Lr]))))));return mn(er,k,new te([...Ir,go]))},"computeMMA"),Rr=Ae("iv_kt"),Mr=Ae("iv_p"),ds=Ae("iv_pN"),Qr=Te(Mr,$(he)),Mi=Te(Mr,$(le)),Vo=Te(ds,$(he)),Go=Te(ds,$(le)),Ii=dr(Rr,$(V-1)),Xl=new te([Xn($(0)),_r($(0),$(0)),new On]),Fa=new Ye(Mr,sn(Rr,$(2)),new Ye(ds,sn(we(Rr,$(1)),$(2)),new te([new xe(Ii,Xn(Te(we(Rr,$(1)),$(k)))),fo(Qr,Mi),new xe(Ii,new te([_r(Vo,Go),new On]))]))),Ei=Iy(Rr,V,Fa),Li=Ae("iv_wm"),ho=Ae("iv_wn"),Ci=we(we(pt,Te(yr,$(A))),Li),jo=we(we(Qn,Te(Bn,$(N))),ho),mo=kn(jo,$(z)),qo=sn(jo,$(z)),Di=kn(qo,$(m)),wt=sn(qo,$(m)),Fs=new Y(o,[mo,Ci,Di,wt],new q(He,[mt(Li,ho)])),ec=mn(Li,A,mn(ho,N,Fs)),q_=new te([Br,Xl,Ei,ec]),U_=new Ve(He,"local",new Ve(gn,"local",new Ve(jt,"local",new Ve(Zt,"local",new Ve(Qt,"local",q_))))),za=new Ye(Zn,we(Te(yr,$(j)),Bn),new Ye(pt,Te(Jn,$(S)),new Ye(Qn,Te(on,$(T)),U_))),Va=R/T,K_=O/S,W_=ki(Jn,"blockIdx.y",K_,ki(on,"blockIdx.x",Va,ki(yr,"threadIdx.y",U,ki(Bn,"threadIdx.x",j,za))));return new Ve(tt,"shared",new Ve(ye,"shared",W_))}function yR(r,e,t){let{BM:n,BN:s,BK:o,TM:i,TN:a}=t;if(![n,s,o,i,a].every(m=>typeof m=="number"&&m>0)||n%i!==0||s%a!==0)return!1;let u=s/a,p=n/i,c=u*p,h=r.warpSize||32;return!(c%h!==0||c>(r.maxThreadsPerBlock||1024)||(n*o+o*s)*4*2>(r.sharedMemoryBytes||49152)||i*a+i+a+h>(r.registersPerThread||255))}function BF(r,e){for(let t of OF)if(gR(r,e,t))return t;return null}function _R(r,e,t){let n=r.func,s=n.convInfo;if(!s)return!1;let i=io(n.body).find(_=>/^conv_acc_/.test(_));if(!i)return!1;let a=af(n.body,i);if(!a||a.reads.length<2||a.writes.length<1)return!1;let u=a.reads[0].buffer,p=a.reads[1].buffer,c=a.writes[0].buffer;if(!u||!p||!c)return!1;let h=s.O,f=s.N*s.Oh*s.Ow,d=s.Cin*s.Kh*s.Kw;if(d<128||f<64)return!1;let m=t&&t.convConfig,g=t&&t.convNoVec,y=null;if(!g){let _=m?gR(e,s,m)?m:null:BF(e,s);_&&(y=AF({weight:p,input:u,output:c},s,_))}if(!y){let _=m?yR(e,{M:h,N:f,K:d},m)?m:null:Sy(e,{M:h,N:f,K:d});if(!_)return!1;y=NF({weight:p,input:u,output:c},s,_)}return r.func.body=y,r.func._setChild&&r.func._setChild("body",y),r.func.gpuRegisterBlocked=!0,!0}var $,yf,Ae,we,Ey,Te,kn,sn,dr,Ly,ao,Iy,mn,ki,GT,OF,jT=M(()=>{"use strict";De();xo();Al();ky();$=l(r=>new L(r),"I"),yf=l(()=>new fe(0),"FZERO"),Ae=l(r=>new Ne(r,"i32"),"IV"),we=l((r,e)=>new C("+",r,e),"ADD"),Ey=l((r,e)=>new C("-",r,e),"SUB"),Te=l((r,e)=>new C("*",r,e),"MUL"),kn=l((r,e)=>new C("//",r,e),"DIV"),sn=l((r,e)=>new C("%",r,e),"MOD"),dr=l((r,e)=>new qe("lt",r,e),"LT"),Ly=l((r,e)=>new qe("ge",r,e),"GE"),ao=l((r,e)=>new C("&&",r,e),"AND"),Iy=l((r,e,t)=>new de(r,$(0),$(e),F.SERIAL,t),"forS"),mn=l((r,e,t)=>new de(r,$(0),$(e),F.UNROLLED,t),"forU"),ki=l((r,e,t,n)=>new de(r,$(0),$(t),F.THREAD_BINDING,n,e),"forT");l(mR,"detectPureConv");l(NF,"buildImplicitGemmConv");GT=4;l(gR,"vectorizableConvConfig");l(AF,"buildVectorizedImplicitGemmConv");l(yR,"validConvConfig");OF=[{BM:128,BN:64,BK:8,TM:8,TN:8},{BM:64,BN:64,BK:8,TM:8,TN:8},{BM:64,BN:64,BK:8,TM:4,TN:8},{BM:64,BN:32,BK:8,TM:8,TN:8},{BM:32,BN:64,BK:8,TM:4,TN:8}];l(BF,"pickVectorizedConvConfig");l(_R,"applyImplicitGemmConv")});function bR(r,e,t,n,s,o,i,a){let u=new me(r,[t,n],e.dtype,"shared"),p=new Y(u,[s,o],new q(e,[i,a]));return{tile:u,fill:p}}function IF(r,e,t){let{A:n,B:s,C:o,M:i,N:a,K:u,transB:p}=r,c=u/t,h=Bl("ts_by"),f=Bl("ts_bx"),d=Bl("ts_ty"),m=Bl("ts_tx"),g=Bl("ts_ko"),y=Bl("ts_ki"),_=new me("ts_acc",[1],"f32","local"),b=Rl(Cy(h,Jr(e)),d),w=Rl(Cy(f,Jr(e)),m),x=Cy(g,Jr(t)),S=bR("ts_As",n,e,t,d,m,b,Rl(x,m)),T=bR("ts_Bs",s,t,e,d,m,p?w:Rl(x,d),p?Rl(x,d):w),k=MF(y,t,new Y(_,[Jr(0)],Rl(new q(_,[Jr(0)]),Cy(new q(S.tile,[d,y]),new q(T.tile,[y,m]))))),A=new te([S.fill,T.fill,new On,k,new On]),N=RF(g,c,A),v=new Y(_,[Jr(0)],new fe(0)),O=new Y(o,[b,w],new q(_,[Jr(0)])),R=new Ve(_,"local",new te([v,N,O]));return Dy(h,"blockIdx.y",i/e,Dy(f,"blockIdx.x",a/e,new Ve(S.tile,"shared",new Ve(T.tile,"shared",Dy(d,"threadIdx.y",e,Dy(m,"threadIdx.x",e,R))))))}function EF(r,e){let{M:t,N:n,K:s}=e,o=16,i=16;return o*o>(r.maxThreadsPerBlock||1024)||(o*i+i*o)*4>(r.sharedMemoryBytes||49152)||t%o!==0||n%o!==0||s%i!==0||t<o||n<o||s<i?null:{BS:o,BK:i}}function LF(r,e){let n=io(r.func.body).filter(o=>/^q?conv_(init|acc)_/.test(o));if(n.length===0)return!1;let s=new Rs(e);for(let o of n)s.applyToBlock(r,o);return!0}function CF(r,e,t={}){if(!e.isGPU())return!1;let n=oR(r.func);if(!n)return!1;let s=n.dims;if(t&&t.primitiveMatmul&&(s.batch||1)===1&&!n.epilogue){let a=EF(e,s);if(a){let u=IF(s,a.BS,a.BK);return r.func.body=u,r.func._setChild&&r.func._setChild("body",u),r.func.gpuRegisterBlocked=!0,!0}}let o=Sy(e,s);if(!o)return!1;let i=CT(s,o,n.epilogue);return r.func.body=i,r.func._setChild&&r.func._setChild("body",i),r.func.gpuRegisterBlocked=!0,!0}function Py(r,e,t={}){if(!e.isGPU()||e.isWebGPU&&e.isWebGPU())return!1;let n=CF(r,e,t);return n||(n=_R(r,e,t)),n||(n=LF(r,e)),n}var Jr,Bl,Rl,Cy,RF,MF,Dy,qT=M(()=>{"use strict";ky();Al();jT();ro();De();xo();Jr=l(r=>new L(r),"I"),Bl=l(r=>new Ne(r,"i32"),"IV"),Rl=l((r,e)=>new C("+",r,e),"ADD"),Cy=l((r,e)=>new C("*",r,e),"MUL"),RF=l((r,e,t)=>new de(r,Jr(0),Jr(e),F.SERIAL,t),"forS"),MF=l((r,e,t)=>new de(r,Jr(0),Jr(e),F.UNROLLED,t),"forU"),Dy=l((r,e,t,n)=>new de(r,Jr(0),Jr(t),F.THREAD_BINDING,n,e),"forT");l(bR,"stageTileToShared");l(IF,"buildTiledSharedMatmul");l(EF,"pickTiledConfig");l(LF,"applyDeterministicGpuConv");l(CF,"applyDeterministicGpuMatmul");l(Py,"applyDeterministicGpuSchedule")});function UT(r){return r*1664525+1013904223&2147483647}function PF(r,e,t){let n={...r};for(let s of e.variables)t.rngFloat()<t.mutationRate&&(n[s.name]=s.sample(o=>t.rng(o)));return n}function wR(r={}){return r.strategy==="random"?new KT({numTrials:r.numTrials,seed:r.seed,deadline:r.deadline}):new WT({populationSize:r.populationSize,numGenerations:r.numGenerations,mutationRate:r.mutationRate,eliteRatio:r.eliteRatio,seed:r.seed,deadline:r.deadline})}var $y,KT,WT,DF,TR=M(()=>{"use strict";l(UT,"nextLcg");$y=class{static{l(this,"SearchCandidate")}constructor(e,t,n){this.sketchName=e,this.params=t,this.score=n}},KT=class{static{l(this,"RandomSearch")}constructor(e={}){this.numTrials=e.numTrials||64,this.seed=e.seed||42,this.deadline=e.deadline||null,this._rngState=this.seed}_rng(e){return this._rngState=UT(this._rngState),this._rngState%e}_expired(){return this.deadline?this.deadline.expired:!1}search(e,t){let n=[];for(let s of e){if(this._expired())break;for(let o=0;o<this.numTrials&&!this._expired();o++){let i=s.sampleParams(u=>this._rng(u)),a=t(s,i);a&&n.push(new $y(s.name,i,a.score))}}return n.sort((s,o)=>o.score-s.score),{candidates:n,population:null}}},WT=class{static{l(this,"EvolutionarySearch")}constructor(e={}){this.populationSize=e.populationSize||32,this.numGenerations=e.numGenerations||10,this.mutationRate=e.mutationRate||.3,this.eliteRatio=e.eliteRatio||.2,this.seed=e.seed||42,this.deadline=e.deadline||null,this._rngState=this.seed}_rng(e){return this._rngState=UT(this._rngState),this._rngState%e}_rngFloat(){return this._rngState=UT(this._rngState),this._rngState/2147483647}_expired(){return this.deadline?this.deadline.expired:!1}search(e,t,n=null){let s=n&&n.length?n:this._initPopulation(e),o=new Map,i=l((u,p)=>{let c=u.name+"|"+JSON.stringify(p);if(o.has(c))return o.get(c);let h=t(u,p);return o.set(c,h),h},"evalMemo");for(let u=0;u<this.numGenerations&&!this._expired();u++){let p=[];for(let d of s){let m=i(d.sketch,d.params);m&&p.push({...d,score:m.score})}if(p.sort((d,m)=>m.score-d.score),p.length===0)break;let c=Math.max(1,Math.floor(p.length*this.eliteRatio)),h=p.slice(0,c),f=h.map(d=>({sketch:d.sketch,params:{...d.params}}));for(;f.length<this.populationSize;){let d=h[this._rng(h.length)],m=h[this._rng(h.length)];if(d.sketch!==m.sketch){let g=this._rngFloat()<.5?d:m;f.push(this._mutate(g.sketch,{...g.params}))}else f.push(this._mutate(d.sketch,this._crossover(d,m)))}s=f}let a=[];for(let u of s){let p=i(u.sketch,u.params);p&&a.push(new $y(u.sketch.name,u.params,p.score))}return a.sort((u,p)=>p.score-u.score),{candidates:a,population:s}}_initPopulation(e){let t=[];for(let n=0;n<this.populationSize;n++){let s=e[this._rng(e.length)],o=s.sampleParams(i=>this._rng(i));t.push({sketch:s,params:o})}return t}_crossover(e,t){if(e.sketch!==t.sketch)return{...e.params};let n={};for(let s of e.sketch.variables)n[s.name]=this._rngFloat()<.5?e.params[s.name]:t.params[s.name];return n}_mutate(e,t){let n={rngFloat:l(()=>this._rngFloat(),"rngFloat"),rng:l(i=>this._rng(i),"rng"),mutationRate:this.mutationRate},s=[PF,...DF],o=t;for(let i of s)o=i(o,e,n);return{sketch:e,params:o}}},DF=[];l(PF,"defaultResampleMutator");l(wR,"createSearchStrategy")});function zy(r){let e={"threadIdx.x":1,"threadIdx.y":1,"threadIdx.z":1},t=new Set,n=l(s=>{if(!(!s||typeof s!="object"||t.has(s))){if(t.add(s),s.type==="ForNode"&&e[s.threadTag]!==void 0){let o=s.extent&&s.extent.type==="IntImmNode"?s.extent.value:1;o>e[s.threadTag]&&(e[s.threadTag]=o)}for(let o in s){let i=s[o];Array.isArray(i)?i.forEach(n):i&&typeof i=="object"&&n(i)}}},"visit");return n(r.body),$F.reduce((s,o)=>s*e[o],1)}var $F,Fy,xR=M(()=>{"use strict";fa();Gw();xT();lf();TR();vT();$F=["threadIdx.x","threadIdx.y","threadIdx.z"];l(zy,"gpuThreadBlockSize");Fy=class{static{l(this,"BlockTuningSession")}constructor(e){this.target=e.target,this.primFunc=e.primFunc,this.blockName=e.blockName,this.sketches=e.sketches,this.benchmarkRunner=e.benchmarkRunner||null,this.config=e.config,this.deadline=e.deadline||null,this._warn=e.warn||(()=>{}),this._warnedEvalSketches=new Set;let t=this.sketches.some(o=>o.name==="fused");t?(this.learnedModel=new vl,this.costModel=new Nl(e.costModel.analytical,this.learnedModel)):(this.costModel=e.costModel,this.learnedModel=e.learnedModel);let n=t?null:XB(e.primFunc,e.blockName,e.blockMap);this.evalFunc=n||e.primFunc,this.evalBlockName=e.blockName,this.sketchByName=new Map;for(let o of this.sketches)this.sketchByName.set(o.name,o);let s=this.sketches.length===1&&typeof this.sketches[0].enumerate=="function";this.enumSketch=s?this.sketches[0]:null,this.enumParams=s?this.sketches[0].enumerate():null,this.strategy=wR({...this.config,deadline:this.deadline}),this.population=null,this._best=null,this.plateaued=!1}runRound(){let e=this._best?this._best.measuredScore:-1/0,t=this._produceCandidates();if(t.length===0)return this.plateaued=!0,0;if(this.benchmarkRunner)this._measureAndLearn(t);else{let s=t[0];this._consider({sketchName:s.sketchName,params:s.params,score:s.score,measuredScore:s.score}),this.plateaued=!0}let n=this._best?this._best.measuredScore:-1/0;return Math.max(0,n-e)}best(){return this._best?{sketchName:this._best.sketchName,params:this._best.params,score:this._best.score,medianMs:this._best.medianMs??null,minMs:this._best.minMs??null}:null}bestTrace(){if(!this._best)return null;let e=this.sketchByName.get(this._best.sketchName);if(!e)return null;try{let t=new Sn(Lo(this.primFunc));return e.instantiate(this._best.params)(t,this.blockName,this.target),t.trace.serialize()}catch(t){return this._warn("best-trace",this.blockName,t),null}}_produceCandidates(){if(this.enumSketch){let s=[];for(let o of this.enumParams){let i=this._evaluate(this.enumSketch,o);i&&s.push({sketchName:this.enumSketch.name,params:o,score:i.score})}return s.sort((o,i)=>i.score-o.score),s}let e=l((s,o)=>this._evaluate(s,o),"evaluator"),{candidates:t,population:n}=this.strategy.search(this.sketches,e,this.population);return this.population=n,t}_evaluate(e,t){try{let n=Lo(this.evalFunc),s=new Sn(n);if(e.instantiate(t)(s,this.evalBlockName,this.target),Ku.validate(n).length>0)return null;let i=this.target.maxThreadsPerBlock;return this.target.isGPU&&this.target.isGPU()&&i&&zy(n)>i?null:{score:this.costModel.score(n)}}catch(n){return this._warnedEvalSketches.has(e.name)||(this._warnedEvalSketches.add(e.name),this._warn("evaluate-candidate",this.blockName,n)),null}}_measureAndLearn(e){let t=e.slice(0,this.config.topKForBenchmark);for(let n of t){if(this.deadline&&this.deadline.expired)break;let s=this._measure(n);if(!s)continue;let o=-s.result.medianMs;this.learnedModel.addSample(s.features,o),this._consider({sketchName:n.sketchName,params:n.params,score:o,measuredScore:o,medianMs:s.result.medianMs,minMs:s.result.minMs})}this.learnedModel.train()}_measure(e){let t=this.sketchByName.get(e.sketchName);if(!t)return null;let n,s;try{n=Lo(this.primFunc),t.instantiate(e.params)(new Sn(n),this.blockName,this.target),s=Lo(this.evalFunc),t.instantiate(e.params)(new Sn(s),this.evalBlockName,this.target)}catch(i){return this._warn("measure-candidate",this.blockName,i),null}let o=this.benchmarkRunner.run(n);return o?{result:o,features:Ra.extractStatements(s)}:null}_consider(e){(!this._best||e.measuredScore>this._best.measuredScore)&&(this._best=e)}}});var HT,Vy,SR=M(()=>{"use strict";HT=class{static{l(this,"GradientSchedulerPolicy")}pick(e){let t=e.filter(i=>!i.plateaued);if(t.length===0)return null;let n=t.filter(i=>i.rounds===0);if(n.length>0)return n[0];let s=null,o=-1/0;for(let i of t){let a=i.gainEwma!==void 0?i.gainEwma:i.lastGain,u=i.weight*a;u>o&&(o=u,s=i)}return s}},Vy=class{static{l(this,"TaskScheduler")}constructor(e=null){this.policy=e||new HT}run(e,t,n={}){let s=n.maxRoundsPerTask??8,o=n.plateauPatience??2,i=n.gainEwmaAlpha??.5;for(let a of e)a.rounds=0,a.lastGain=0,a.gainEwma=0,a.stale=0,a.plateaued=!1;for(;!(t&&t.expired);){let a=this.policy.pick(e);if(!a)break;let u=a.session.runRound();a.rounds++,a.lastGain=u,a.gainEwma=i*u+(1-i)*a.gainEwma,u<=0?a.stale++:a.stale=0,(a.session.plateaued||a.stale>=o||a.rounds>=s)&&(a.plateaued=!0)}}}});function FF(r){if(r.isCPU())return null;let e=$v(r.kind);if(!e)throw new Error("hardwareMeasure requested for target '"+r.kind+"' but no measurer is registered for it; the corresponding runtime must be loaded (Node: import '#io/cuda_runtime') before compiling");return e}var Gy,jy,kR=M(()=>{"use strict";fa();vT();lR();pR();hR();dR();VT();Al();PT();ro();qT();xR();lf();De();SR();rw();l(FF,"resolveMeasurer");Gy=class{static{l(this,"AutotuneConfig")}constructor(e={}){this.strategy=e.strategy||"evolutionary",this.numTrials=e.numTrials||64,this.populationSize=e.populationSize||32,this.numGenerations=e.numGenerations||10,this.mutationRate=e.mutationRate,this.eliteRatio=e.eliteRatio,this.seed=e.seed||42,this.timeBudgetMs=e.timeBudgetMs||3e4,this.clock=e.clock||null,this.tuningDB=e.tuningDB||null,this.useTuningDB=e.useTuningDB!==!1,this.measurer=e.measurer||null,this.hardwareMeasure=e.hardwareMeasure??!1,this.enableBenchmark=e.enableBenchmark??(this.hardwareMeasure||!!e.measurer),this.benchmarkWarmup=e.benchmarkWarmup??3,this.benchmarkRepeat=e.benchmarkRepeat??10,this.benchmarkMaxCv=e.benchmarkMaxCv??0,this.topKForBenchmark=e.topKForBenchmark??5,this.maxRoundsPerTask=e.maxRoundsPerTask??8,this.plateauPatience=e.plateauPatience??2,this.schedulerPolicy=e.schedulerPolicy||null,this.onWarning=e.onWarning||null}},jy=class{static{l(this,"Autotuner")}constructor(e,t={},n=null){this.target=e,this.config=t instanceof Gy?t:new Gy(t),this.trace=n,this._funcName=null,this.config.hardwareMeasure&&(this.config.measurer=FF(e)),this.analyticalModel=new by(e),this.learnedModel=new vl,this.costModel=new Nl(this.analyticalModel,this.learnedModel),this.db=this.config.tuningDB instanceof gf?this.config.tuningDB:new gf;let s=l((o,i,a)=>this._warn(o,i,a),"warn");this.benchmarkRunner=this.config.enableBenchmark?new Ry(e,{warmup:this.config.benchmarkWarmup,repeat:this.config.benchmarkRepeat,maxCv:this.config.benchmarkMaxCv,measurer:this.config.measurer,warn:s}):null,this.scheduler=new Vy(this.config.schedulerPolicy)}_warn(e,t,n){let s=n&&n.message?n.message:String(n);if(this.config.onWarning)try{this.config.onWarning({stage:e,func:this._funcName,block:t||null,message:s,error:n})}catch(o){}this.trace&&this.trace.warn("autotune",this._funcName,`${e}${t?" ["+t+"]":""}: ${s}`)}tune(e,t=null){this._funcName=e.name;let n=t?[t]:io(e.body),s=Ay(e.body),o=df(e),i=new My(this.config.timeBudgetMs,this.config.clock),a=new Map,u=new Map;for(let h of n){let f=Oy(e,h,this.target,s);u.set(h,f);let d=a.get(f);if(d){d.weight++;continue}if(this.config.useTuningDB&&this.db.has(f)){a.set(f,{key:f,kind:"cache",cached:this.db.lookup(f),weight:1});continue}let m=zT(e,h,this.target,s,{richGpu:this.config.richGpu??!!this.config.measurer,dag:o});if(m.length===0){a.set(f,{key:f,kind:"empty",weight:1});continue}let g=new Fy({target:this.target,primFunc:e,blockName:h,blockMap:s,sketches:m,costModel:this.costModel,learnedModel:this.learnedModel,benchmarkRunner:this.benchmarkRunner,config:this.config,deadline:i,warn:l((y,_,b)=>this._warn(y,_,b),"warn")});a.set(f,{key:f,kind:"session",session:g,weight:1})}let p=[...a.values()].filter(h=>h.kind==="session");p.length>0&&this.scheduler.run(p,i,this.config);let c=new Map;for(let h of n){let f=a.get(u.get(h));if(f.kind==="cache"){c.set(h,{sketchName:f.cached.sketchName,params:f.cached.params,score:f.cached.score,fromCache:!0});continue}if(f.kind==="empty")continue;let d=f.session.best();if(d){if(this.config.useTuningDB&&!f.stored){let m=new mf(f.key,d.sketchName,d.params,d.score,f.session.bestTrace(),this.db.version);m.medianMs=d.medianMs||null,m.minMs=d.minMs||null,this.db.store(f.key,m),f.stored=!0}c.set(h,{sketchName:d.sketchName,params:d.params,score:d.score,fromCache:!1,medianMs:d.medianMs,minMs:d.minMs})}}return c}tuneAndApply(e,t=null){let n=this.tune(e,t);if(n.size>0){let s=this._applyBestSchedule(e,n);if(s)return{func:s.func,results:n,applied:!0}}return{func:e,results:n,applied:!1}}_applyBestSchedule(e,t){let n=this._buildDefaultSchedule(e),s=!!n&&this._scheduleIsValid(n),o=s&&this._isStrongBackendSchedule(n),i=this._buildTunedSchedule(e,t),a=!!i&&this._scheduleIsValid(i),u=a&&this._isStrongBackendSchedule(i),p=!o||u&&this.config.measurer!=null;return a&&p?(this._adoptSchedule(e,i),{func:e}):(i&&!a?this._warn("tuned-schedule-invalid",null,new Error("tuned schedule exceeds target thread-block limit; falling back to default")):a&&o&&this._warn("baseline-preferred",null,new Error("cost-model-only tuning cannot displace the deterministic GPU schedule without hardware measurement; keeping the deterministic kernel")),s?(this._adoptSchedule(e,n),{func:e}):a?(this._adoptSchedule(e,i),{func:e}):(this._warn("no-valid-schedule",null,new Error("neither tuned nor default schedule is valid; leaving function unscheduled")),null))}_isStrongBackendSchedule(e){return!!e&&e.gpuRegisterBlocked===!0}_buildTunedSchedule(e,t){try{let n=Lo(e),s=new Sn(n),o=Ay(n.body),i=df(n),a=new Set,u=[];for(let c of t)if(c[1].sketchName==="fused"){let h=vy(n,i,c[0],dt);h&&a.add(h),u.unshift(c)}else u.push(c);let p=new Set;for(let[c,h]of u)if(!(a.has(c)||p.has(h))&&(p.add(h),!(!h.sketchName||!h.params)))try{let d=zT(n,c,this.target,o,{richGpu:this.config.richGpu??!!this.config.measurer,dag:i}).find(m=>m.name===h.sketchName);d&&this._fitsThreadBlock(n,c,d,h.params)&&d.instantiate(h.params)(s,c,this.target)}catch(f){this._warn("apply-tuned-block",c,f);continue}return this._scheduleResidualBlocks(s,a),n}catch(n){return this._warn("build-tuned-schedule",null,n),null}}_scheduleResidualBlocks(e,t){let n=null;for(let s of io(e.func.body))if(!(t.has(s)||this._blockIsParallelized(e,s))){n||(n=new Rs(this.target));try{n.applyToBlock(e,s)}catch(o){this._warn("residual-block",s,o)}}}_blockIsParallelized(e,t){let n;try{n=e.getLoops(t)}catch(s){return this._warn("block-loops",t,s),!0}for(let s of n)if(s.kind===F.THREAD_BINDING||s.kind===F.PARALLEL||s.kind===F.VECTORIZED)return!0;return!1}_scheduleIsValid(e){return!this.target.isGPU||!this.target.isGPU()||!this.target.maxThreadsPerBlock?!0:zy(e)<=this.target.maxThreadsPerBlock}_buildDefaultSchedule(e){try{let t=Lo(e),n=new Sn(t);return Py(n,this.target,this.config)||new Rs(this.target).applyToAllBlocks(n),t}catch(t){return this._warn("build-default-schedule",null,t),null}}_adoptSchedule(e,t){Object.assign(e,t),e._setChild("body",e.body)}_fitsThreadBlock(e,t,n,s){if(!this.target.isGPU||!this.target.isGPU()||!this.target.maxThreadsPerBlock)return!0;try{let o=Lo(e);return n.instantiate(s)(new Sn(o),t,this.target),zy(o)<=this.target.maxThreadsPerBlock}catch(o){return this._warn("fits-thread-block",t,o),!1}}}});var qy,vR=M(()=>{"use strict";xi();fa();ro();kR();qT();qy=class extends Wn{static{l(this,"SchedulePass")}constructor(e){super("SchedulePass","scheduling"),this.config=e,this.target=e.target,this.snapshotPoint="afterScheduling",this._autotuner=null,this._policy=null}begin(e){let t=this.config.scheduling;t.autotune?this._autotuner=new jy(this.target,t,e.trace):t.enabled&&(this._policy=new Rs(this.target,null,e.trace))}run(e,t){let n=this.config.scheduling,s=t.trace;if(n.autotune){if(e.cublasInfo||e._tensorIntrin)return;let o=performance.now(),i=this._autotuner.tuneAndApply(e),a=performance.now()-o,u=0,p=0;if(i&&i.results){p=i.results.size;for(let[c,h]of i.results)h.fromCache&&u++,s.explainsEnabled&&s.explain("schedule",c,h.sketchName,`autotuned: best of search${h.fromCache?" (cached)":""}, score ${h.score!=null?h.score.toFixed(3):"n/a"}`,{target:this.target.name,params:h.params})}s.autotuneStats(e.name,{durationMs:a,blockCount:p,applied:!!(i&&i.applied),cacheHits:u})}else if(n.enabled||n.gpuTiling){if(e.cublasInfo||e._tensorIntrin)return;let o=performance.now(),i=new Sn(e);!Py(i,this.target,n)&&n.enabled&&this._policy.applyToAllBlocks(i),s.functionEvent("scheduling",e.name,{durationMs:performance.now()-o})}}}});function NR(r){let e={analyzer:new Os,simp:null};e.simp=new fm(e.analyzer);let t=Zr(r.body,e);return r.body=t,r._setChild("body",t),r}function zF(r,e,t){let n=r.analyzer.getVarBound(e);return t&&t.type==="IntImmNode"&&t.value>0?r.analyzer.bind(e,0,t.value-1):r.analyzer.setVarBound(e,null),n}function Zr(r,e){if(!r||typeof r!="object")return r;switch(r.type){case"ForNode":{let t=zF(e,r.loopVar.name,r.extent),n=Zr(r.body,e);e.analyzer.setVarBound(r.loopVar.name,t);let s=new de(r.loopVar,r.min,r.extent,r.kind,n,r.threadTag);return r.annotations&&(s.annotations=r.annotations),s}case"BlockNode":{let t=[];for(let i of r.iterVars)i.iterVar&&(t.push([i.iterVar.name,e.analyzer.getVarBound(i.iterVar.name)]),e.analyzer.setVarBound(i.iterVar.name,i.binding?la(e.analyzer,i.binding):null));let n=r.iterVars.map(VF(e)),s=Zr(r.body,e),o=r.initBody?Zr(r.initBody,e):null;for(let[i,a]of t)e.analyzer.setVarBound(i,a);return new pe(r.name,n,r.reads,r.writes,s,o)}case"SeqNode":return new te(r.stmts.map(t=>Zr(t,e)));case"IfThenElseNode":{let t=Hn(r.condition,e);return Mo(e.analyzer,t)?Zr(r.thenBody,e):ua(e.analyzer,t)?r.elseBody?Zr(r.elseBody,e):new te([]):new xe(t,Zr(r.thenBody,e),r.elseBody?Zr(r.elseBody,e):null)}case"BufferStoreNode":return new Y(r.buffer,r.indices.map(t=>Hn(t,e)),Hn(r.value,e));case"LetStmtNode":return new Ye(r.variable,Hn(r.value,e),Zr(r.body,e));case"AllocateNode":return new Ve(r.buffer,r.scope,Zr(r.body,e));case"WhileNode":return new Zo(r.condVar,Zr(r.condBody,e),Zr(r.loopBody,e));case"EvaluateNode":return new ou(Hn(r.value,e));default:return r}}function VF(r){return e=>{if(!e.iterVar||!e.binding)return e;let t=Hn(e.binding,r);return new qi(e.iterVar,t,e.kind)}}function Hn(r,e){if(!r||typeof r!="object"||!r.type)return r;switch(r.type){case"IntImmNode":case"FloatImmNode":case"VariableNode":return r;case"BufferLoadNode":return new q(r.buffer,r.indices.map(t=>Hn(t,e)));case"MathOpNode":{let t=Hn(r.a,e),n=r.b?Hn(r.b,e):null;return e.simp.simplify(new C(r.op,t,n))}case"CompareNode":{let t=Hn(r.a,e),n=Hn(r.b,e);return e.simp.simplify(new qe(r.direction,t,n))}case"CastNode":return new rt(Hn(r.expr,e),r.fromDtype,r.toDtype);case"CallExternNode":return new ke(r.externName,r.args.map(t=>Hn(t,e)),r.dtype);case"IfThenElseNode":{let t=Hn(r.condition,e),n=Hn(r.thenBody,e),s=r.elseBody?Hn(r.elseBody,e):null;return Mo(e.analyzer,t)?n:s!==null&&ua(e.analyzer,t)?s:new xe(t,n,s)}default:return r}}var AR=M(()=>{"use strict";Uu();Hc();De();l(NR,"simplifyPrimFunc");l(zF,"bindLoopVar");l(Zr,"simplifyStmt");l(VF,"simplifyIterVar");l(Hn,"simplifyExpr")});var Uy,OR=M(()=>{"use strict";xi();AR();Uy=class extends Wn{static{l(this,"SimplifyPass")}constructor(){super("SimplifyPass","simplify")}run(e,t){let n=performance.now();return NR(e),t.trace.functionEvent("simplify",e.name,{durationMs:performance.now()-n}),e}}});function BR(r){return!!r&&typeof r=="object"&&typeof r.name=="string"&&r.dtype!==void 0&&r.shape!==void 0&&r.type===void 0}var YT,JT,GF,Ky,RR=M(()=>{"use strict";YT=class{static{l(this,"BufferInterval")}constructor(e,t,n,s){this.buffer=e,this.firstUse=t,this.lastUse=n,this.scope=s}get size(){return this.buffer.sizeInBytes()}overlaps(e){return this.firstUse<=e.lastUse&&e.firstUse<=this.lastUse}},JT=class{static{l(this,"BufferLivenessResult")}constructor(e,t,n){this.intervals=e,this.stmtOrder=t,this.paramBuffers=n}isParam(e){return this.paramBuffers.has(e)}getTemporaries(){let e=[];for(let[t,n]of this.intervals)this.paramBuffers.has(t)||e.push(n);return e}interfere(e,t){let n=this.intervals.get(e),s=this.intervals.get(t);return!n||!s?!1:n.overlaps(s)}},GF=new Set(["_parent","_parentKey","_parentIdx"]);l(BR,"isBuffer");Ky=class{static{l(this,"BufferLiveness")}static analyze(e){let t=new Map,n=[],s=0,o=new Set;for(let[,h]of e.bufferMap)o.add(h);let i=[],a=l(h=>{if(!h)return;let f=t.get(h);f?(s<f.firstUse&&(f.firstUse=s),s>f.lastUse&&(f.lastUse=s)):(f=new YT(h,s,s,h.scope),t.set(h,f)),i.push(h)},"touch"),u=l((h,f)=>{if(!(!h||typeof h!="object"||f.has(h))){if(f.add(h),BR(h)){a(h);return}BR(h.buffer)&&a(h.buffer);for(let d of Object.keys(h)){if(GF.has(d)||d==="buffer")continue;let m=h[d];if(!(!m||typeof m!="object"))if(Array.isArray(m))for(let g of m)u(g,f);else u(m,f)}}},"touchAll"),p=l((h,f)=>{for(let d=h;d<i.length;d++){let m=t.get(i[d]);m&&f>m.lastUse&&(m.lastUse=f)}},"extendRegion"),c=l(h=>{if(h)switch(h.type){case"SeqNode":for(let f of h.stmts)c(f);break;case"ForNode":{let f=s,d=i.length;u(h.min,new Set),u(h.extent,new Set),c(h.body);let m=s>f?s-1:f;p(d,m);break}case"WhileNode":{let f=s,d=i.length;u(h.condVar,new Set),c(h.condBody),c(h.loopBody);let m=s>f?s-1:f;p(d,m);break}case"BlockNode":n.push({idx:s,node:h});for(let f of h.reads)a(f.buffer);for(let f of h.writes)a(f.buffer);u(h.body,new Set),h.initBody&&u(h.initBody,new Set),s++;break;case"AllocateNode":a(h.buffer),c(h.body);break;case"IfThenElseNode":{let f=s,d=i.length;u(h.condition,new Set),c(h.thenBody),h.elseBody&&c(h.elseBody);let m=s>f?s-1:f;p(d,m);break}case"LetStmtNode":u(h.value,new Set),c(h.body);break;case"EvaluateNode":u(h.value,new Set);break;default:u(h,new Set);break}},"walk");return c(e.body),new JT(t,n,o)}}});function jF(r,e){if(r.shape.length!==e.shape.length)return!1;for(let t=0;t<r.shape.length;t++)if(r.shape[t]!==e.shape[t])return!1;return!0}function Ml(r,e){if(r===e)return!0;if(!r||!e||typeof r!="object"||typeof e!="object"||r.type!==e.type)return!1;switch(r.type){case"VariableNode":return r.name===e.name;case"IntImmNode":return r.value===e.value;case"FloatImmNode":return r.value===e.value;case"MathOpNode":return r.op===e.op&&Ml(r.a,e.a)&&Ml(r.b,e.b);case"CompareNode":return r.direction===e.direction&&Ml(r.a,e.a)&&Ml(r.b,e.b);case"CastNode":return r.toDtype===e.toDtype&&Ml(r.expr,e.expr);case"BufferLoadNode":return r.buffer===e.buffer&&QT(r.indices,e.indices);default:return!1}}function QT(r,e){if(r.length!==e.length)return!1;for(let t=0;t<r.length;t++)if(!Ml(r[t],e[t]))return!1;return!0}function MR(r,e){r&&Ss(r,e)}function qF(r,e,t){let n=[],s=[];for(let p of[r.body,r.initBody])p&&MR(p,c=>{c.type==="BufferStoreNode"&&c.buffer===t?n.push(c):c.type==="BufferLoadNode"&&c.buffer===e&&s.push(c)});if(n.length!==1||s.length===0)return!1;let o=n[0],i=o.indices;for(let p of s)if(!QT(p.indices,i))return!1;let a=new Set;for(let p of[o.value,...o.indices])p&&MR(p,c=>{c.type==="BufferLoadNode"&&c.buffer===e&&a.add(c)});let u=!0;for(let p of s)if(!a.has(p)){u=!1;break}if(!u){let p=o.value;if(!(p&&p.type==="BufferLoadNode"&&p.buffer===e&&QT(p.indices,i)))return!1}return!0}function UF(r,e){if(r)for(let t of gu(r,n=>n.type==="BlockNode",{kinds:"stmt"}))e.push(t)}var ZT,Wy,IR=M(()=>{"use strict";Gr();ZT=class{static{l(this,"InplaceCandidate")}constructor(e,t,n){this.srcBuffer=e,this.dstBuffer=t,this.reason=n}},Wy=class{static{l(this,"InplaceAnalysis")}static analyze(e,t,n=new Set){let s=[],o=[];UF(e.body,o);let i=new Map;for(let p of t.stmtOrder)i.set(p.node,p.idx);let a=new Map;for(let p of t.stmtOrder)for(let c of p.node.reads){let h=a.get(c.buffer);(h===void 0||p.idx>h)&&a.set(c.buffer,p.idx)}let u=new Set;for(let p of o){let c=i.get(p);if(c!==void 0)for(let h of p.writes){let f=h.buffer;if(!(t.isParam(f)&&!n.has(f)))for(let d of p.reads){let m=d.buffer;if(m===f||u.has(m)||t.isParam(m)&&!n.has(m)||!jF(m,f)||m.dtype!==f.dtype||m.scope!==f.scope)continue;let g=t.intervals.get(m),y=t.intervals.get(f);if(!(!g||!y)&&g.lastUse<=y.firstUse){let _=a.get(m);if(_===void 0||_<=c){if(!qF(p,m,f))continue;s.push(new ZT(m,f,`${m.name} last used at ${g.lastUse}, ${f.name} first used at ${y.firstUse}`)),u.add(m);break}}}}}return s}};l(jF,"shapesMatch");l(Ml,"exprEqual");l(QT,"indexListEqual");l(MR,"walkNodes");l(qF,"isInplaceComputeSafe");l(UF,"collectBlocks")});function ER(r,e){let t=0,n=r.length;for(;t<n;){let s=t+n>>1;r[s].offset<=e.offset?t=s+1:n=s}r.splice(t,0,e)}function LR(r){return{best:null,consider(e,t,n){return t<n?null:r==="best-fit"?((this.best===null||t<this.best.gap)&&(this.best={offset:e,gap:t}),null):e},result(e){return this.best!==null?this.best.offset:e}}}var XT,ex,Hy,CR=M(()=>{"use strict";l(ER,"insertByOffset");l(LR,"gapSelector");XT=class{static{l(this,"MemoryBlock")}constructor(e,t,n){this.offset=e,this.size=t,this.buffer=n}get end(){return this.offset+this.size}overlaps(e){return this.offset<e.end&&e.offset<this.end}},ex=class{static{l(this,"MemoryPool")}constructor(e,t=64,n="best-fit"){this.scope=e,this.alignment=t,this.strategy=n,this.blocks=[],this.peakUsage=0}allocate(e,t){let n=this._align(e),s=this._findFreeOffset(n);return this.placeAt(s,n,t)}placeAt(e,t,n){let s=this._align(t),o=new XT(e,s,n);ER(this.blocks,o);let i=e+s;return i>this.peakUsage&&(this.peakUsage=i),o}_align(e){return Math.ceil(e/this.alignment)*this.alignment}_findFreeOffset(e){let t=this.blocks,n=0,s=LR(this.strategy);for(let o of t){let i=this._align(n),a=o.offset-i,u=s.consider(i,a,e);if(u!==null)return u;o.end>n&&(n=o.end)}return s.result(this._align(n))}fragmentation(){if(this.peakUsage===0)return 0;let e=this.blocks.reduce((t,n)=>t+n.size,0);return Math.max(0,1-e/this.peakUsage)}release(e){let t=this.blocks.indexOf(e);t>=0&&this.blocks.splice(t,1)}},Hy=class{static{l(this,"BufferAssignment")}constructor(){this.assignments=new Map,this.pools=new Map,this.inplaceMap=new Map,this.effLastUse=new Map}assign(e,t=[],n=64,s="best-fit"){for(let c of t)this.inplaceMap.set(c.dstBuffer,c.srcBuffer);let o=new Map;for(let c of e)o.set(c.buffer,c);let i=new Map;for(let c of e)i.set(c.buffer,c.lastUse);let a=new Map;for(let[c,h]of this.inplaceMap)!o.has(c)||!o.has(h)||(a.has(h)||a.set(h,[]),a.get(h).push(c));if(a.size>0){let c=new Map;for(let h of e){let f=h.buffer;if(c.get(f)===1)continue;let d=[f];for(;d.length;){let m=d[d.length-1],g=a.get(m);if(c.get(m)===void 0){if(c.set(m,0),g)for(let y of g)c.get(y)===void 0&&d.push(y);continue}if(d.pop(),c.get(m)!==1){if(g){let y=i.get(m);for(let _ of g){let b=i.get(_);b>y&&(y=b)}i.set(m,y)}c.set(m,1)}}}}this.effLastUse=i;let u=[...e].sort((c,h)=>{let f=c.size,d=h.size,m=f>0,g=d>0;if(m&&g){let y=d-f;if(y!==0)return y}else if(m!==g)return m?-1:1;return c.firstUse-h.firstUse}),p=new Map;for(let c of u){let h=c.buffer;if(this.inplaceMap.has(h)){let w=this.inplaceMap.get(h),x=this.assignments.get(w);if(x){this.assignments.set(h,{offset:x.offset,size:c.size,scope:c.scope,pool:x.pool,inplaceOf:w});continue}}let f=c.size;if(f===0)continue;if(f<0){this.assignments.set(h,{offset:0,size:0,scope:c.scope,pool:c.scope,inplaceOf:null,isDynamic:!0});continue}let d=c.scope,m=this.pools.get(d);m||(m=new ex(d,n),this.pools.set(d,m)),p.has(d)||p.set(d,[]);let g=p.get(d),y=i.get(h)??c.lastUse,_=this._interferenceOffset(g,c.firstUse,y,m._align(f),n,s),b=m.placeAt(_,f,h);ER(g,{firstUse:c.firstUse,lastUseEff:y,offset:b.offset,size:b.size}),this.assignments.set(h,{offset:b.offset,size:b.size,scope:d,pool:d,inplaceOf:null})}return this}_interferenceOffset(e,t,n,s,o,i){let a=[];for(let c of e)c.firstUse<=n&&t<=c.lastUseEff&&a.push([c.offset,c.offset+c.size]);let u=0,p=LR(i);for(let[c,h]of a){let f=Math.ceil(u/o)*o,d=c-f,m=p.consider(f,d,s);if(m!==null)return m;h>u&&(u=h)}return p.result(Math.ceil(u/o)*o)}getOffset(e){let t=this.assignments.get(e);return t?t.offset:-1}getAssignment(e){return this.assignments.get(e)||null}peakMemory(e=null){if(e){let n=this.pools.get(e);return n?n.peakUsage:0}let t=0;for(let[,n]of this.pools)t+=n.peakUsage;return t}}});function Jy(r,e,t){if(!r||typeof r!="object"||t.has(r))return!1;if(t.add(r),r.type==="BufferLoadNode"&&(!e||r.buffer===e))return!0;for(let n of Object.keys(r)){if(n==="_parent"||n==="_parentKey"||n==="_parentIdx")continue;let s=r[n];if(!(!s||typeof s!="object")){if(Array.isArray(s)){for(let o of s)if(Jy(o,e,t))return!0}else if(Jy(s,e,t))return!0}}return!1}function DR(r){let e=new Set,t=new Map,n=new Set,s=[r.body];for(;s.length>0;){let o=s.pop();if(!(!o||typeof o!="object"||n.has(o))){if(n.add(o),o.type==="BufferStoreNode"&&o.buffer){let i=o.buffer;for(let u of o.indices)if(Jy(u,null,new Set)){e.add(i);break}Jy(o.value,i,new Set)&&e.add(i);let a=t.get(i);a||(a=[],t.set(i,a)),a.push(o.value)}for(let i of Object.keys(o)){if(i==="_parent"||i==="_parentKey"||i==="_parentIdx")continue;let a=o[i];if(Array.isArray(a))for(let u of a)u&&typeof u=="object"&&s.push(u);else a&&typeof a=="object"&&s.push(a)}}}for(let[o,i]of t){let a=l(u=>u&&(u.type==="IntImmNode"||u.type==="FloatImmNode")&&u.value===0,"isConstZero");i.every(a)&&e.add(o),i.length===1&&i[0]&&(i[0].type==="IntImmNode"||i[0].type==="FloatImmNode")&&e.add(o)}return e}function KF(r,e){let t=new Set,n=[r];for(;n.length>0;){let s=n.pop();if(!(!s||typeof s!="object"||t.has(s))){t.add(s);for(let o of Object.keys(s)){if(o==="_parent"||o==="_parentKey"||o==="_parentIdx")continue;let i=s[o];if(i&&typeof i=="object"&&e.has(i)){s[o]=e.get(i);continue}if(Array.isArray(i)){for(let a=0;a<i.length;a++){let u=i[a];u&&typeof u=="object"&&e.has(u)?i[a]=e.get(u):u&&typeof u=="object"&&n.push(u)}continue}i&&typeof i=="object"&&n.push(i)}}}}var tx,Yy,PR=M(()=>{"use strict";RR();IR();CR();De();Zb();tx=class{static{l(this,"MemoryPlan")}constructor(e,t,n){this.assignment=e,this.liveness=t,this.inplaceCandidates=n,this.aliasMap=new Map}peakMemory(e=null){return this.assignment.peakMemory(e)}getReport(){let e=new Map;for(let[s,o]of this.assignment.pools)e.set(s,{peakUsage:o.peakUsage,numBuffers:0,numReused:0});for(let[s,o]of this.assignment.assignments){let i=e.get(o.scope);i&&(i.numBuffers++,o.inplaceOf&&i.numReused++)}let t=this.liveness.getTemporaries().length,n=this.inplaceCandidates.length;return{peakMemory:this.assignment.peakMemory(),scopeBreakdown:e,totalTemporaries:t,totalInplace:n,materializedReuse:this.aliasMap.size,assignments:this.assignment.assignments}}},Yy=class{static{l(this,"MemoryPlanner")}constructor(e={}){this.alignment=e.alignment||64,this.enableInplace=e.enableInplace!==!1,this.allocStrategy=e.allocStrategy||"best-fit",this.poolAllocation=e.poolAllocation||!1}plan(e){let t=Ky.analyze(e),n=t.getTemporaries(),s=[];this.enableInplace&&(s=Wy.analyze(e,t));let o=new Hy;return o.assign(n,s,this.alignment,this.allocStrategy),new tx(o,t,s)}planAndRewrite(e){let t=this.plan(e);return{func:this._insertAllocations(e,t),plan:t}}_insertAllocations(e,t){let n=t.liveness.getTemporaries();if(n.length===0)return e;let s=new Map;this.poolAllocation?this._assignPoolOffsets(e,t,n):(s=this._buildReuseAliases(n,t,e),s.size>0&&KF(e.body,s)),t.aliasMap=s;let o=[...n].sort((u,p)=>p.firstUse-u.firstUse),i=e.body,a=new Set;for(let u of o){let p=u.buffer;if(s.has(p))continue;let c=t.assignment.getAssignment(p);c&&(c.inplaceOf||a.has(p)||(a.add(p),i=new Ve(p,c.isDynamic?"dynamic":c.scope,i)))}return e.body=i,e._setChild("body",i),e}_assignPoolOffsets(e,t,n){let s=DR(e);for(let o of n){let i=o.buffer;if(s.has(i)||i.scope!=="global")continue;let a=t.assignment.getAssignment(i);!a||a.inplaceOf||a.isDynamic||a.size>0&&(i.poolByteOffset=a.offset)}}_buildReuseAliases(e,t,n){let s=DR(n),o=new Set(t.assignment.inplaceMap.values()),i=t.assignment.effLastUse,a=l(c=>i.get(c.buffer)??c.lastUse,"lastUseOf"),u=new Map;for(let c of e){let h=c.buffer,f=t.assignment.getAssignment(h);if(!f||f.inplaceOf||f.isDynamic||o.has(h)||h.numel()<=0||s.has(h))continue;let d=`${h.scope}|${h.dtype}|${h.shape.join(",")}|${h.strides.join(",")}`,m=u.get(d);m||(m=[],u.set(d,m)),m.push(c)}let p=new Map;for(let c of u.values()){if(c.length<2)continue;c.sort((f,d)=>f.firstUse-d.firstUse||a(f)-a(d));let h=new ku((f,d)=>f.lastUse-d.lastUse);for(let f of c){let d=h.peek();d&&d.lastUse<f.firstUse?(h.pop(),d.lastUse=a(f),h.push(d),p.set(f.buffer,d.rep)):h.push({rep:f.buffer,lastUse:a(f)})}}return p}};l(Jy,"exprLoadsBuffer");l(DR,"collectFreshZeroDependent");l(KF,"rewriteBufferAliases")});var Zy,$R=M(()=>{"use strict";xi();PR();Zy=class extends Wn{static{l(this,"MemoryPlanPass")}constructor(e){super("MemoryPlanPass","memoryPlanning"),this.config=e,this._planner=null}begin(e){let t=this.config.memory.alignment||this.config.target?.cacheLineSizeBytes||64;this._planner=new Yy({alignment:t,enableInplace:this.config.memory.inplaceReuse,allocStrategy:this.config.memory.allocStrategy,poolAllocation:this.config.memory.poolAllocation})}run(e,t){if(e.gpuRegisterBlocked)return;let n=performance.now(),{plan:s}=this._planner.planAndRewrite(e),o=s.getReport();t.trace.memoryStats(e.name,{durationMs:performance.now()-n,peakMemory:o.peakMemory,totalTemporaries:o.totalTemporaries,totalInplace:o.totalInplace})}}});function Il(r){return r&&r.type==="IntImmNode"?r.value:null}function WF(r){if(!r||r.type!=="ForNode"||!FR.has(r.kind)||Il(r.min)!==0)return null;let e=Il(r.extent);if(e===null)return null;let t=r.body;if(!t||t.type!=="ForNode"||!FR.has(t.kind)||Il(t.min)!==0)return null;let n=Il(t.extent);if(n===null||n<=0)return null;let s=t.body;if(!s||s.type!=="IfThenElseNode"||s.elseBody)return null;let o=HF(s.condition,r.loopVar.name,t.loopVar.name,n);if(o===null||o<=0)return null;let i=Math.floor(o/n),a=o-i*n;if(a===0||i<1||e!==i+1)return null;let u=new Os;if(u.bind(r.loopVar.name,0,i-1),u.bind(t.loopVar.name,0,n-1),!Mo(u,s.condition))return null;let p=s.thenBody,c=new de(t.loopVar,new L(0),new L(n),t.kind,p,t.threadTag),h=new de(r.loopVar,new L(0),new L(i),r.kind,c,r.threadTag),f=ti(Ma(p),g=>g.type==="VariableNode"&&g.name===r.loopVar.name?new L(i):g),d=new Ne(t.loopVar.name,t.loopVar.dtype),m=new de(d,new L(0),new L(a),t.kind,f,t.threadTag);return new te([h,m])}function HF(r,e,t,n){if(!r)return null;let s,o;if(r.type==="MathOpNode"&&r.op==="<")s=r.a,o=r.b;else if(r.type==="CompareNode"&&r.direction==="lt")s=r.a,o=r.b;else return null;let i=Il(o);if(i===null||!s||s.type!=="MathOpNode"||s.op!=="+")return null;let a=s.a,u=s.b;return!u||u.type!=="VariableNode"||u.name!==t||!a||a.type!=="MathOpNode"||a.op!=="*"||!a.a||a.a.type!=="VariableNode"||a.a.name!==e||Il(a.b)!==n?null:i}var FR,Qy,zR=M(()=>{"use strict";xi();De();Gr();lf();Uu();Hc();FR=new Set([F.SERIAL,F.PARALLEL,F.VECTORIZED,F.UNROLLED]),Qy=class extends Wn{static{l(this,"LoopPartitionPass")}constructor(){super("LoopPartitionPass","loopPartition")}run(e,t){let n=ti(e.body,s=>WF(s)||s);return n&&n!==e.body&&(e.body=n,e._setChild("body",n)),e}};l(Il,"intImm");l(WF,"partitionLoop");l(HF,"matchFlatGuard")});function Xy(r){let e=r.body;if(!e||e.type!=="BlockNode")return null;let t=e.body;if(!t||t.type!=="BufferStoreNode")return null;let n=t,s=n.value;if(!s||s.type!=="MathOpNode"||!YF.has(s.op))return null;let o=null,i=null;if(s.a&&s.a.type==="BufferLoadNode"&&s.a.buffer===n.buffer?(o=s.a,i=s.b):s.b&&s.b.type==="BufferLoadNode"&&s.b.buffer===n.buffer&&(o=s.b,i=s.a),!o)return null;let a=nx(n.indices),u=nx(o.indices);if(a!==u||a.includes("?"))return null;let p=n.indices.map(f=>{if(f.type!=="VariableNode")return f;for(let d of e.iterVars)if(d.iterVar&&d.iterVar.name===f.name&&d.binding)return d.binding;return f}),c=r.loopVar.name,h=nx(p);return h.includes("?")||h.includes("$"+c)?null:{store:n,loadSide:o,valueSide:i,outerIndices:p,block:e,op:n.value.op}}function nx(r){return r.map(rx).join(",")}function rx(r){return r?r.type==="VariableNode"?"$"+r.name:r.type==="IntImmNode"?String(r.value):r.type==="MathOpNode"?"("+rx(r.a)+r.op+(r.b?rx(r.b):"")+")":"?":"?"}var YF,sx=M(()=>{"use strict";YF=new Set(["+","*","max","min"]);l(Xy,"detectAccumulator");l(nx,"indicesKey");l(rx,"exprKey")});function uo(r){if(!(!r||typeof r!="object"||!r.type))switch(r.type){case"ForNode":{let e=Xy(r);r.accumulator=e,e||uo(r.body);return}case"BlockNode":uo(r.body),r.initBody&&uo(r.initBody);return;case"SeqNode":for(let e of r.stmts)uo(e);return;case"LetStmtNode":case"AllocateNode":uo(r.body);return;case"IfThenElseNode":uo(r.thenBody),r.elseBody&&uo(r.elseBody);return;case"WhileNode":uo(r.condBody),uo(r.loopBody);return;default:return}}var e_,VR=M(()=>{"use strict";xi();sx();e_=class extends Wn{static{l(this,"AccumulatorDetectionPass")}constructor(){super("AccumulatorDetectionPass","accumulatorDetect")}run(e,t){return uo(e.body),e}};l(uo,"annotateStmt")});function ZF(r){let e=[],t=[r.body];for(;t.length>0;){let s=t.pop();if(s){if(s.type==="BlockNode"){e.push(s),t.push(s.body);continue}if(s.body&&t.push(s.body),s.stmts)for(let o of s.stmts)t.push(o);s.thenBody&&t.push(s.thenBody),s.elseBody&&t.push(s.elseBody)}}if(e.length===0)return null;let n=null;for(let s of e){if(!s.name.includes("matmul"))return null;s.reads.length>=2&&s.writes.length>=1&&(n=s)}return n}function QF(r){let e=ZF(r);if(!e)return null;let t=e.reads[0].buffer,n=e.reads[1].buffer,s=e.writes[0].buffer;if(!GR.has(t.dtype)||!GR.has(n.dtype)||s.dtype!=="f32"||t.shape.length!==2||n.shape.length!==2||s.shape.length!==2)return null;let o=s.shape[0],i=s.shape[1],a=t.shape[1];if(![o,i,a].every(p=>typeof p=="number"&&p>0&&p%JF===0))return null;let u=new Set;for(let[,p]of r.bufferMap)u.add(p.name);return!u.has(t.name)||!u.has(n.name)||!u.has(s.name)?null:{M:o,N:i,K:a,a:t.name,b:n.name,c:s.name}}var JF,GR,t_,jR=M(()=>{"use strict";xi();fa();JF=16,GR=new Set(["f16","bf16"]);l(ZF,"findMatmulBlock");l(QF,"detectWmmaMatmul");t_=class extends Wn{static{l(this,"AutoTensorizePass")}constructor(e){super("AutoTensorizePass","scheduling"),this.config=e,this.target=e.target}run(e,t){if(e.cublasInfo||e._tensorIntrin||!this.target||!this.target.isGPU())return;let n=QF(e);n&&(new Sn(e).tensorize("wmma_16x16x16_f16f16f32",n),t&&t.trace&&t.trace.explainsEnabled&&t.trace.explain("tensorize",e.name,"wmma_16x16x16_f16f16f32",`auto-tensorized ${n.M}x${n.N}x${n.K} f16 GEMM`,{target:this.target.name}))}}});function XF(r){let e=[],t=new Map,n=new Map,s=[{node:r,block:null}];for(;s.length>0;){let{node:o,block:i}=s.pop();if(o.type==="BufferLoadNode"&&o.buffer)t.set(o.buffer.name,(t.get(o.buffer.name)||0)+1),i&&i.reads.add(o.buffer.name);else if(o.type==="BufferStoreNode"&&o.buffer&&i){i.writes.add(o.buffer.name);let u=n.get(o.buffer.name);u||(u=new Set,n.set(o.buffer.name,u)),u.add(i.name)}let a=i;o.type==="BlockNode"&&(a={name:o.name,writes:new Set,reads:new Set,hasInit:o.initBody!=null},e.push(a));for(let u of en(o))s.push({node:u,block:a})}return{blocks:e,loadCount:t,storeWriters:n}}var n_,qR=M(()=>{"use strict";xi();fa();ro();Gr();l(XF,"analyzeFunc");n_=class extends Wn{static{l(this,"InlineReindexPass")}constructor(e){super("InlineReindexPass","scheduling"),this.config=e,this.target=e.target,this.snapshotPoint="afterInlineReindex"}run(e,t){if(!this.target.isGPU()||this.target.isWebGPU()||e.cublasInfo||e._tensorIntrin||i0(e))return;let n=this.config.scheduling;if(!(n.enabled||n.gpuTiling||n.autotune))return;let s=new Set;for(let[,c]of e.bufferMap)s.add(c.name);let{blocks:o,loadCount:i,storeWriters:a}=XF(e.body),u=new Sn(e),p=!1;for(let c of o){if(c.hasInit||c.writes.size===0)continue;let h=[...c.writes];if(h.every(f=>!s.has(f)&&a.get(f).size===1)&&h.some(f=>(i.get(f)||0)>0)&&[...c.reads].every(f=>(a.get(f)?a.get(f).size:0)<=1))try{u.computeInlineBlock(c.name),p=!0}catch{}}p&&s0(e)}}});function ox(r,e,t=ez){return t.filter(n=>n.phase===r).sort((n,s)=>n.priority-s.priority).map(n=>n.factory(e)).filter(Boolean)}var ez,UR=M(()=>{"use strict";ez=[];l(ox,"tirPassesForPhase")});function KR(r){let e=[];for(let t of ox("pre",r))e.push(t);e.push(new n_(r)),r.optimization.tensorize&&e.push(new t_(r)),e.push(new qy(r)),r.optimization.loopPartition&&e.push(new Qy),e.push(new Uy),e.push(new Zy(r)),r.optimization.detectAccumulators&&e.push(new e_);for(let t of ox("post",r))e.push(t);return e}var WR=M(()=>{"use strict";vR();OR();$R();zR();VR();jR();qR();UR();l(KR,"buildTirPipeline")});var _f,HR=M(()=>{"use strict";De();_f=class{static{l(this,"TensorVerifier")}constructor(){this.errors=[],this.boundVars=new Set}verify(e){if(this.errors=[],this.boundVars.clear(),!(e instanceof Jo))return this.errors.push("Expected PrimFunc at root"),this.errors;for(let t of e.params)this.boundVars.add(t.name);if(e.shapeParams)for(let t of e.shapeParams)this.boundVars.add(t.name);return this.visit(e.body),this.errors}visit(e){let t=[e];for(;t.length>0;){let n=t.pop();if(n)switch(n.type){case"SeqNode":for(let s=n.stmts.length-1;s>=0;s--)t.push(n.stmts[s]);break;case"ForNode":this.boundVars.has(n.loopVar.name)&&this.errors.push(`Loop variable ${n.loopVar.name} already bound`),this.boundVars.add(n.loopVar.name),t.push({type:"_unbind",name:n.loopVar.name}),t.push(n.body);break;case"BlockNode":for(let s of n.iterVars)s.iterVar&&(this.boundVars.has(s.iterVar.name)&&this.errors.push(`Block variable ${s.iterVar.name} already bound`),this.boundVars.add(s.iterVar.name));for(let s=n.iterVars.length-1;s>=0;s--)n.iterVars[s].iterVar&&t.push({type:"_unbind",name:n.iterVars[s].iterVar.name});t.push(n.body),n.initBody&&t.push(n.initBody);break;case"AllocateNode":n.buffer||this.errors.push("Allocate missing buffer"),t.push(n.body);break;case"LetStmtNode":this._visitExpr(n.value),this.boundVars.add(n.variable.name),t.push({type:"_unbind",name:n.variable.name}),t.push(n.body);break;case"IfThenElseNode":this._visitExpr(n.condition),n.elseBody&&t.push(n.elseBody),t.push(n.thenBody);break;case"WhileNode":n.condVar||this.errors.push("WhileNode missing condition variable"),t.push(n.loopBody),t.push(n.condBody);break;case"BufferStoreNode":if(n.buffer||this.errors.push("BufferStore missing buffer"),(!n.indices||n.indices.length!==n.buffer.shape.length)&&this.errors.push(`BufferStore rank mismatch for ${n.buffer?n.buffer.name:"unknown"}`),n.indices)for(let s of n.indices)this._visitExpr(s);this._visitExpr(n.value);break;case"EvaluateNode":this._visitExpr(n.value);break;case"_unbind":this.boundVars.delete(n.name);break;default:this._visitExpr(n);break}}}_visitExpr(e){if(e)switch(e.type){case"BufferLoadNode":if(e.buffer||this.errors.push("BufferLoad missing buffer"),(!e.indices||e.indices.length!==e.buffer.shape.length)&&this.errors.push(`BufferLoad rank mismatch for ${e.buffer?e.buffer.name:"unknown"}`),e.indices)for(let t of e.indices)this._visitExpr(t);break;case"MathOpNode":this._visitExpr(e.a),e.b&&this._visitExpr(e.b);break;case"CompareNode":this._visitExpr(e.a),this._visitExpr(e.b);break;case"CastNode":this._visitExpr(e.expr);break;case"CallExternNode":e.externName||this.errors.push("CallExtern missing function name");for(let t of e.args)this._visitExpr(t);break;case"IfThenElseNode":this._visitExpr(e.condition),this._visitExpr(e.thenBody),e.elseBody&&this._visitExpr(e.elseBody);break;case"VariableNode":this.boundVars.has(e.name)||this.errors.push(`Unbound variable used: ${e.name}`);break;case"BlockRealizeNode":e.binding&&this._visitExpr(e.binding);break;case"IntImmNode":case"FloatImmNode":break}}}});function ix(r){let e=[];r.functionCount===0&&e.push(new vt("Module has no functions"));for(let t of r)bf(t,e);return e}function bf(r,e=[]){if(!r.entryBlock)return e.push(new vt("Function has no entry block",null,r)),e;r.entryBlock.arguments.length!==r.inputTypes.length&&e.push(new vt(`Entry block has ${r.entryBlock.arguments.length} args, expected ${r.inputTypes.length}`,null,r));for(let s=0;s<r.entryBlock.arguments.length;s++){let o=r.entryBlock.arguments[s],i=r.inputTypes[s];i&&!o.type.equals(i)&&e.push(new vt(`Block arg ${s} type ${Rn(o.type)} != expected ${Rn(i)}`,null,r))}let t=new Set;for(let s of r.entryBlock.arguments)t.add(s);for(let s of r.body)YR(s,t);for(let s of r.body)JR(s,r,t,e);let n=r.getReturnOp();if(!n)e.push(new vt("Missing return op",null,r));else{n.numOperands!==r.outputTypes.length&&e.push(new vt(`Return has ${n.numOperands} operands, function declares ${r.outputTypes.length} outputs`,n,r));for(let s=0;s<Math.min(n.numOperands,r.outputTypes.length);s++){let o=n.getOperand(s).type,i=r.outputTypes[s];i instanceof I&&o instanceof I&&(o.dtype!==i.dtype&&e.push(new vt(`Return operand ${s} dtype ${o.dtype} != expected ${i.dtype}`,n,r)),o.shapeCompatible(i)||e.push(new vt(`Return operand ${s} shape incompatible: ${Rn(o)} vs ${Rn(i)}`,n,r)))}}return e}function YR(r,e){for(let t of r.arguments)e.add(t);for(let t of r)for(let n=0;n<t.numResults;n++){let s=t.getResult(n);s&&e.add(s)}}function tz(r,e,t){let n=new Set;for(let u of r)n.add(u);let s=1,o=2,i=new Map,a=new Set;for(let u of r){if(i.get(u)!==void 0)continue;let p=[{op:u,i:0}];for(i.set(u,s);p.length>0;){let c=p[p.length-1],h=c.op;if(c.i<h.numOperands){let f=h.getOperand(c.i);c.i++;let d=f&&f.definingOp;if(!d||!n.has(d))continue;let m=i.get(d);m===s?a.has(d)||(a.add(d),t.push(new vt("participates in a value dependency cycle",d,e))):m===void 0&&(i.set(d,s),p.push({op:d,i:0}));continue}i.set(h,o),p.pop()}}}function JR(r,e,t,n){tz(r,e,n);for(let s of r)nz(s,e,t,n);if(r.size>0){let s=r.lastOp;if(r.parentRegion&&r.parentRegion.parentOp){let o=r.parentRegion.parentOp.opName,i=ne.get(o);i&&i.hasRegions&&(s.isTerminator()||n.push(new vt(`Block in ${o} region must end with terminator, got '${s.opName}'`,s,e)))}}}function nz(r,e,t,n){for(let o=0;o<r.numOperands;o++){let i=r.getOperand(o);if(!i){n.push(new vt(`Operand ${o} is null`,r,e));continue}if(!(i instanceof Wo)){n.push(new vt(`Operand ${o} is not a Value`,r,e));continue}t.has(i)||n.push(new vt(`Operand ${o} used before definition`,r,e))}for(let o=0;o<r.numResults;o++){let i=r.getResult(o);i||n.push(new vt(`Result ${o} is null`,r,e)),i&&i.definingOp!==r&&n.push(new vt(`Result ${o} definingOp mismatch`,r,e))}let s=ne.get(r.opName);if(!s){!["return","yield"].includes(r.opName)&&!ne.has(r.opName)&&n.push(new vt(`Unknown op '${r.opName}'`,r,e));return}s.numOperands>=0&&r.numOperands!==s.numOperands&&n.push(new vt(`'${r.opName}' expects ${s.numOperands} operands, got ${r.numOperands}`,r,e)),s.numResults>=0&&r.numResults!==s.numResults&&n.push(new vt(`'${r.opName}' expects ${s.numResults} results, got ${r.numResults}`,r,e));for(let o of s.attrs)o.required&&!r.hasAttr(o.name)&&n.push(new vt(`'${r.opName}' missing required attribute '${o.name}'`,r,e));if(s.hasRegions&&s.numRegions>0&&r.numRegions!==s.numRegions&&n.push(new vt(`'${r.opName}' expects ${s.numRegions} regions, got ${r.numRegions}`,r,e)),s.verify){let o=s.verify(r);if(o)for(let i of o)n.push(new vt(i,r,e))}if(s.inferResultTypes&&r.numResults>0){let o=[];for(let a=0;a<r.numOperands;a++)o.push(r.getOperand(a).type);let i=s.inferResultTypes(o,r.attributes,r.results.map(a=>a.type));if(i)for(let a=0;a<Math.min(i.length,r.numResults);a++){let u=r.getResult(a).type,p=i[a];p instanceof I&&u instanceof I&&(u.dtype!==p.dtype&&n.push(new vt(`Result ${a} dtype ${u.dtype} != inferred ${p.dtype}`,r,e)),u.shapeCompatible(p)||n.push(new vt(`Result ${a} shape ${Rn(u)} incompatible with inferred ${Rn(p)}`,r,e)))}}for(let o of r.regions){let i=new Set(t);for(let a of o)YR(a,i);for(let a of o)JR(a,e,i,n)}}var vt,ZR=M(()=>{"use strict";Oe();lh();At();vt=class{static{l(this,"VerificationError")}constructor(e,t=null,n=null){this.message=e,this.op=t,this.func=n}toString(){let e="";return this.func&&(e+=`[${this.func.name}] `),this.op&&(e+=`op '${this.op.opName}' (id=${this.op.id}): `),e+this.message}};l(ix,"verifyModule");l(bf,"verifyFunction");l(YR,"collectScopeDefs");l(tz,"detectCycles");l(JR,"verifyBlock");l(nz,"verifyOperation")});var r_,ax,El,ux,lx=M(()=>{"use strict";_T();Oe();r_=class{static{l(this,"ValueObserver")}constructor(){this.min=1/0,this.max=-1/0,this.count=0,this.histogram=null}update(e){for(let t=0;t<e.length;t++){let n=e[t];n<this.min&&(this.min=n),n>this.max&&(this.max=n)}this.count+=e.length,this.histogram&&this.histogram.update(e)}enableHistogram(e=2048){this.histogram=new ax(e)}},ax=class{static{l(this,"HistogramCollector")}constructor(e=2048){this.numBins=e,this.bins=new Float64Array(e),this.rangeMin=0,this.rangeMax=0,this.initialized=!1,this.totalCount=0}update(e){this.initialized||this._initRange(e);let t=this.rangeMax-this.rangeMin;if(t<=0)return;for(let s=0;s<e.length;s++){let o=e[s];if(o<this.rangeMin||o>this.rangeMax){this._expandAndRebucket(e);return}}let n=this.numBins/t;for(let s=0;s<e.length;s++){let o=Math.min(this.numBins-1,Math.floor((e[s]-this.rangeMin)*n));this.bins[o]++}this.totalCount+=e.length}_initRange(e){let t=1/0,n=-1/0;for(let s=0;s<e.length;s++)e[s]<t&&(t=e[s]),e[s]>n&&(n=e[s]);t===n&&(n=t+1),this.rangeMin=t,this.rangeMax=n,this.initialized=!0}_expandAndRebucket(e){let t=this.rangeMin,n=this.rangeMax;for(let c=0;c<e.length;c++)e[c]<t&&(t=e[c]),e[c]>n&&(n=e[c]);let s=new Float64Array(this.bins),o=this.rangeMin,a=this.rangeMax-o;this.rangeMin=t,this.rangeMax=n;let u=n-t,p=this.numBins/u;if(this.bins.fill(0),a>0){let c=a/this.numBins;for(let h=0;h<this.numBins;h++){if(s[h]===0)continue;let f=o+(h+.5)*c,d=Math.min(this.numBins-1,Math.floor((f-t)*p));this.bins[d]+=s[h]}}for(let c=0;c<e.length;c++){let h=Math.min(this.numBins-1,Math.floor((e[c]-t)*p));this.bins[h]++}this.totalCount+=e.length}computePercentileThreshold(e){let t=this.totalCount*e,n=0,s=(this.rangeMax-this.rangeMin)/this.numBins;for(let o=0;o<this.numBins;o++)if(n+=this.bins[o],n>=t)return this.rangeMin+(o+1)*s;return this.rangeMax}computeEntropyThreshold(e){if(this.totalCount===0)return this.rangeMax;let t=new Float64Array(this.numBins);for(let i=0;i<this.numBins;i++)t[i]=this.bins[i]/this.totalCount;let n=this.numBins,s=1/0;for(let i=e;i<=this.numBins;i++){let a=i/e,u=new Float64Array(i);for(let c=0;c<e;c++){let h=Math.floor(c*a),f=Math.min(i,Math.floor((c+1)*a)),d=0;for(let g=h;g<f;g++)d+=t[g];let m=f-h;if(m>0){let g=d/m;for(let y=h;y<f;y++)u[y]=g}}let p=0;for(let c=0;c<i;c++)t[c]>0&&u[c]>0&&(p+=t[c]*Math.log(t[c]/u[c]));p<s&&(s=p,n=i)}let o=(this.rangeMax-this.rangeMin)/this.numBins;return this.rangeMin+n*o}},El=class{static{l(this,"CalibrationCollector")}constructor(e="minmax"){this.observers=new Map,this.mode=e}attach(e){for(let t of e.ops())for(let n=0;n<t.numResults;n++){let s=t.getResult(n);if(s.type instanceof I&&Lt(s.type.dtype)){let o=new r_;(this.mode==="entropy"||this.mode==="percentile")&&o.enableHistogram(),this.observers.set(s,o)}}for(let t of e.args)if(t.type instanceof I&&Lt(t.type.dtype)){let n=new r_;(this.mode==="entropy"||this.mode==="percentile")&&n.enableHistogram(),this.observers.set(t,n)}}observe(e,t){let n=this.observers.get(e);n&&n.update(t)}getResult(){return new ux(this.observers,this.mode)}},ux=class{static{l(this,"CalibrationResult")}constructor(e,t){this._observers=e,this._mode=t}getRange(e){let t=this._observers.get(e);return!t||t.count===0?null:{min:t.min,max:t.max}}getQuantParams(e,t,n){let s=this.getRange(e);if(!s)return null;let o=t===kr.PER_TENSOR_ASYMMETRIC;if(this._mode==="percentile"&&this._observers.get(e).histogram){if(o)return hr.fromRange(s.min,s.max,t,n);let a=this._observers.get(e).histogram.computePercentileThreshold(.999);return hr.fromRange(-a,a,t,n)}if(this._mode==="entropy"&&this._observers.get(e).histogram){if(o)return hr.fromRange(s.min,s.max,t,n);let i=this._observers.get(e).histogram,a=n==="ui8"?256:255,u=i.computeEntropyThreshold(a);return hr.fromRange(-u,u,t,n)}return hr.fromRange(s.min,s.max,t,n)}hasData(e){let t=this._observers.get(e);return t&&t.count>0}values(){return this._observers.keys()}}});function sz(r,e){let t=[],n=new Set;for(let s of r.ops())if(e.has(s.opName))for(let o=0;o<s.numOperands;o++){let i=s.getOperand(o);if(n.has(i)||!(i.type instanceof I)||!Lt(i.type.dtype))continue;let a=i.definingOp;a&&a.opName==="constant"||(n.add(i),t.push(i))}return t}function oz(r,e){let t=e.map(d=>d.type),n=new un(r.name,r.inputTypes,[...r.outputTypes,...t]),s=new Map,o=r.entryBlock.arguments,i=n.entryBlock.arguments;for(let d=0;d<o.length;d++)s.set(o[d],i[d]);let a=r.entryBlock.opsArray(),u=new Set(a),p=wo(a,d=>u.has(d),"ignore"),c=new Map;for(let d of p)d.opName!=="return"&&c.set(d,d.clone(s));for(let d of a)d.opName!=="return"&&n.entryBlock.pushOp(c.get(d));let h=r.getReturnOp(),f=[];if(h)for(let d of h.operands)f.push(s.get(d)||d);for(let d of e)f.push(s.get(d));return n.entryBlock.pushOp(new Re("return",f,[],{})),n}function QR(r,e,t,n={}){if(e.isGPU&&e.isGPU())throw new Error("collectCalibration: synchronous calibration is unavailable for async (GPU) targets; precompute calibration on a CPU/WASM target or supply quantization.calibration directly");if(!t||t.length===0)throw new Error("collectCalibration: at least one calibration batch is required");let s=n.quantizableOps||rz,o=n.mode||"minmax",i=sz(r,s),a=new El(o);a.attach(r);let u=a.getResult();if(i.length===0)return u;let p=oz(r,i),c=new wn("__calibrate__");c.addFunction(p);let h=r.outputTypes.length,d=(n.compileFn||iz)(c,e);for(let m of t){let g=Array.isArray(m)?m:[m],y=p.outputTypes.map(_=>new Float32Array(Math.max(1,_.numel())));d.run(r.name,...g,...y);for(let _=0;_<i.length;_++)a.observe(i[_],y[h+_])}return u}function iz(r,e){throw new Error("collectCalibration: opts.compileFn is required (pass the compileModule function to avoid a circular import)")}var rz,XR=M(()=>{"use strict";xs();Yo();sr();Oe();ji();lx();rz=new Set(["dot","conv","add","mul","sub"]);l(sz,"activationTargets");l(oz,"buildCaptureFunction");l(QR,"collectCalibration");l(iz,"defaultCompileFn")});function px(r){let e=0;for(let t=0;t<r.numResults;t++){let n=r.getResult(t).type;n instanceof I&&n.isFullyStatic&&(e+=n.sizeInBytes())}return e}var cx,Ll,fx,s_,eM=M(()=>{"use strict";At();Oe();nf();cx=class{static{l(this,"Partition")}constructor(e,t){this.id=e,this.target=t,this.ops=[],this.opSet=new Set,this._inputValues=null,this._outputValues=null,this._memoryBytes=0}addOp(e){this.opSet.has(e)||(this.ops.push(e),this.opSet.add(e),this._inputValues=null,this._outputValues=null,this._memoryBytes+=px(e))}hasOp(e){return this.opSet.has(e)}merge(e){for(let t of e.ops)this.addOp(t)}computeIO(){if(this._inputValues&&this._outputValues)return;let{inputs:e,outputs:t}=_i(this.opSet,this.ops);this._inputValues=e,this._outputValues=t}getInputValues(){return this.computeIO(),this._inputValues}getOutputValues(){return this.computeIO(),this._outputValues}get size(){return this.ops.length}get memoryBytes(){return this._memoryBytes}};l(px,"estimateOpMemory");Ll=class{static{l(this,"PartitionerConfig")}constructor(e={}){this.targets=e.targets||[],this.defaultTarget=e.defaultTarget||null,this.opTargetOverrides=e.opTargetOverrides||new Map,this.memoryLimits=e.memoryLimits||new Map,this.minPartitionSize=e.minPartitionSize||1,this.costWeights={transferCost:1,loadBalance:.5,...e.costWeights||{}}}},fx=class{static{l(this,"PartitionResult")}constructor(e,t,n){this.partitions=e,this.opToPartition=t,this.transferEdges=n}getPartition(e){return this.opToPartition.get(e)||null}getPartitionsForTarget(e){return this.partitions.filter(t=>t.target===e||t.target.name===e.name)}get numPartitions(){return this.partitions.length}},s_=class{static{l(this,"GraphPartitioner")}constructor(e){this.config=e instanceof Ll?e:new Ll(e),this._supportCache=new Map,this._buildSupportMap()}partition(e){let t=this._collectPartitionableOps(e),n=this._assignTargets(t),s=this._buildPartitions(t,n),o=this._mergeSmallPartitions(s,n),i=this._computeTransferEdges(o),a=new Map;for(let u of o)for(let p of u.ops)a.set(p,u);return new fx(o,a,i)}_buildSupportMap(){for(let e of this.config.targets){let t=new Set;for(let n of ne.names())this._targetSupportsOp(e,n)&&t.add(n);this._supportCache.set(e,t)}}_targetSupportsOp(e,t){if(e.hasLibraryOp(t))return!0;let n=ne.get(t);return n?n.isConstant||n.isTerminator?!0:e.isGPU()?n.isElementwise||n.isReduction||n.isBroadcast||n.isInjective||n.getAttr("gpuCapable")===!0:e.isCPU()?!0:e.isWasm()?!n.isOpaque&&t!=="custom_call":!1:!1}_collectPartitionableOps(e){let t=[];for(let n of e.ops()){let s=ne.get(n.opName);!s||s.isTerminator||t.push(n)}return t}_assignTargets(e){let t=new Map;for(let n of e){let s=n.getAttr("device");if(s){let a=this._resolveDeviceAttr(s);if(a){t.set(n,a);continue}}let o=this.config.opTargetOverrides.get(n.opName);if(o){t.set(n,o);continue}let i=this._selectBestTarget(n);t.set(n,i)}return t}_resolveDeviceAttr(e){return typeof e=="string"?this.config.targets.find(t=>t.name===e||t.kind===e)||null:e}_selectBestTarget(e){let t=this.config.defaultTarget||this.config.targets[0],n=-1/0;for(let s of this.config.targets){let o=this._supportCache.get(s);if(!o||!o.has(e.opName))continue;let i=this._scoreTargetForOp(s,e);i>n&&(n=i,t=s)}return t}_scoreTargetForOp(e,t){let n=ne.get(t.opName);if(!n)return 0;let s=0;if(e.hasLibraryOp(t.opName)&&(s+=100),n.isReduction||n.isElementwise){let o=0;for(let i=0;i<t.numOperands;i++){let a=t.getOperand(i).type;a instanceof I&&a.isFullyStatic&&(o+=a.numel())}e.isGPU()&&o>1024?s+=50:e.isCPU()&&o<=1024&&(s+=30)}return s+=e.computeTFLOPs*10,s}_buildPartitions(e,t){let{partitions:n}=tf(e,{sort:l(i=>this._topologicalSort(i),"sort"),labelOf:l(i=>t.get(i),"labelOf"),sameLabel:l((i,a)=>i===a||i.name===a.name,"sameLabel"),canMerge:l((i,a,u)=>this._fitsMemoryLimit(i,a,u),"canMerge"),onAttach:l((i,a)=>{i.memoryBytes=(i.memoryBytes||0)+px(a)},"onAttach")}),s=new Map;for(let i of n){let a=new cx(i.id,i.label);for(let u of i.ops)a.addOp(u);s.has(i.label.name)||s.set(i.label.name,[]),s.get(i.label.name).push(a)}let o=[];for(let i of s.values())for(let a of i)o.push(a);return o}_fitsMemoryLimit(e,t,n){let s=this.config.memoryLimits.get(n.name);if(!s)return!0;let o=px(t);return e.memoryBytes+o<=s}_mergeSmallPartitions(e,t){if(e.length<=1)return e;let n=new Map;for(let d of e)for(let m of d.ops)n.set(m,d);let s=new Set,o=new Map,i=new Map;for(let d of e)s.has(d)||o.set(d,new Set);for(let d of e){if(s.has(d))continue;let m=o.get(d);for(let g of d.ops)for(let y=0;y<g.numResults;y++)for(let _ of g.getResult(y).uses()){let b=n.get(_.user);b&&b!==d&&m.add(b)}}let a=l((d,m)=>{let g=o.get(d);for(let y of o.get(m))y!==d&&g.add(y);g.delete(m);for(let[y,_]of o)_.has(m)&&(_.delete(m),y!==d&&_.add(d));o.delete(m),i=new Map},"mergeSucc"),u=l(d=>{let m=i.get(d);if(m)return m;m=new Set;let g=[...o.get(d)];for(;g.length>0;){let y=g.pop();if(m.has(y))continue;m.add(y);let _=o.get(y);if(_)for(let b of _)g.push(b)}return i.set(d,m),m},"reachOf"),p=l((d,m)=>{for(let g of o.get(d))if(g!==m&&u(g).has(m))return!0;return!1},"pathThroughIntermediate"),c=l((d,m)=>p(d,m)||p(m,d),"mergeCreatesCycle"),h=[];for(let d=0;d<e.length;d++){let m=e[d];if(s.has(m))continue;if(m.size>=this.config.minPartitionSize){h.push(m);continue}let g=-1,y=-1/0;for(let _=0;_<e.length;_++){if(d===_)continue;let b=e[_];if(s.has(b)||b.target.name!==m.target.name||c(m,b))continue;let w=this.config.costWeights.transferCost*this._mergeScore(m,b);w>y&&(y=w,g=_)}if(g>=0){let _=e[g];a(_,m),_.merge(m);for(let b of m.ops)n.set(b,_);s.add(m)}else h.push(m)}let f=new Set;for(let d of h)for(let m of d.ops)f.add(m);for(let d of e){let m=!1;for(let g of d.ops)if(!f.has(g)){m=!0;break}if(m){h.push(d);for(let g of d.ops)f.add(g)}}return h}_mergeScore(e,t){let n=0,s=new Set;for(let o of e.ops)for(let i=0;i<o.numResults;i++)s.add(o.getResult(i));for(let o of t.ops)for(let i=0;i<o.numOperands;i++)s.has(o.getOperand(i))&&n++;return n}_computeTransferEdges(e){let t=[],n=new Set,s=new Map;for(let o of e)for(let i of o.ops)s.set(i,o);for(let o of e)for(let i of o.ops)for(let a=0;a<i.numOperands;a++){let u=i.getOperand(a).definingOp;if(!u)continue;let p=s.get(u);if(!p||p===o)continue;let c=i.getOperand(a),h=`${p.id}|${o.id}|${c.id}`;n.has(h)||(n.add(h),t.push({src:p,dst:o,value:c,sizeBytes:c.type instanceof I&&c.type.isFullyStatic?c.type.sizeInBytes():0}))}return t}_topologicalSort(e){let t=new Set(e),n=new Map,s=new Map;for(let u of e)n.set(u,0),s.set(u,[]);for(let u of e)for(let p=0;p<u.numOperands;p++){let c=u.getOperand(p).definingOp;c&&t.has(c)&&(s.get(c).push(u),n.set(u,n.get(u)+1))}let o=[];for(let u of e)n.get(u)===0&&o.push(u);let i=[],a=0;for(;a<o.length;){let u=o[a++];i.push(u);for(let p of s.get(u)){let c=n.get(p)-1;n.set(p,c),c===0&&o.push(p)}}return i}}});var o_,i_,tM=M(()=>{"use strict";nn();sr();xs();eM();nf();Kt();o_=class extends Ge{static{l(this,"GraphPartitionPass")}constructor(e={}){super("GraphPartitionPass"),this.partitionerConfig=new Ll(e),this.partitionResult=null}run(e,t){if(this.partitionerConfig.targets.length<2)return oe.UNCHANGED;let n=new s_(this.partitionerConfig);return this.partitionResult=n.partition(e),this.partitionResult.numPartitions<=1?oe.UNCHANGED:(this._annotateOps(),this._insertTransferOps(e),this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,numPartitions:this.partitionResult.numPartitions,transferEdges:this.partitionResult.transferEdges.length,level:ue.DEBUG}),e.bumpVersion(),oe.CHANGED)}_annotateOps(){for(let e of this.partitionResult.partitions)for(let t of e.ops)t.setAttr("partition_id",e.id),t.setAttr("partition_target",e.target.name)}_insertTransferOps(e){let t=e.entryBlock,n=this._buildOrderIndex(t),{useMap:s,firstInPart:o}=this._buildInsertionIndex(t);for(let i of this.partitionResult.transferEdges){let a=i.value,u=i.src.target.name,p=i.dst.target.name;if(u===p)continue;let c=new Re("copy_to_device",[a],[a.type],{src_device:u,dst_device:p});c.setAttr("partition_id",i.dst.id),c.setAttr("partition_target",p);let h=s.get(i.dst),f=h&&h.get(a)||o.get(i.dst)||null,d=a.definingOp;if(f&&d&&n.has(d)&&n.has(f)&&n.get(d)>=n.get(f))t.insertAfter(c,d);else if(f)t.insertBefore(c,f);else if(d&&n.has(d))t.insertAfter(c,d);else{let g=e.getReturnOp();g?t.insertBefore(c,g):t.pushOp(c)}let m=c.getResult(0);for(let g of i.dst.ops)for(let y=0;y<g.numOperands;y++)g.getOperand(y)===a&&g.replaceOperand(y,m)}}_buildInsertionIndex(e){let t=[...new Set(this.partitionResult.transferEdges.map(o=>o.dst))],n=new Map,s=new Map;for(let o of t)n.set(o,new Map);for(let o of e.ops())for(let i of t){if(!i.hasOp(o))continue;s.has(i)||s.set(i,o);let a=n.get(i);for(let u=0;u<o.numOperands;u++){let p=o.getOperand(u);a.has(p)||a.set(p,o)}}return{useMap:n,firstInPart:s}}_buildOrderIndex(e){let t=new Map,n=0;for(let s of e.ops())t.set(s,n++);return t}},i_=class extends Ge{static{l(this,"PartitionMaterializationPass")}constructor(e={}){super("PartitionMaterializationPass"),this.targets=e.targets||[]}run(e,t){let n=this._collectPartitions(e);if(n.size<=1)return oe.UNCHANGED;let s=this._getModule(e);if(!s)return oe.UNCHANGED;let o=this._materializePartitions(e,n);for(let i of o)s.addFunction(i);return this._rewriteOriginalFunction(e,o,n),this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,partitions:n.size,subFunctions:o.length,level:ue.DEBUG}),e.bumpVersion(),oe.CHANGED}_collectPartitions(e){let t=new Map;for(let n of e.ops()){let s=n.getAttr("partition_id");s!==void 0&&(t.has(s)||t.set(s,{id:s,target:n.getAttr("partition_target"),ops:[]}),t.get(s).ops.push(n))}return t}_getModule(e){return e._module||null}_materializePartitions(e,t){let n=[];for(let[s,o]of t){let i=new Set(o.ops),{inputs:a,outputs:u}=_i(i,o.ops),p=a.map(_=>_.type),c=u.map(_=>_.type),h=`${e.name}_partition_${s}`,f=new un(h,p,c);f._partitionTarget=o.target;let d=new Map;for(let _=0;_<a.length;_++)d.set(a[_],f.args[_]);let m=Ba(o.ops);for(let _ of m)f.entryBlock.pushOp(_.clone(d));let g=u.map(_=>d.get(_)||_),y=new Re("return",g,[]);f.entryBlock.pushOp(y),n.push(f)}return n}_rewriteOriginalFunction(e,t,n){for(let s of t)e.setAttr?.(`sub_${s.name}`,s._partitionTarget)}}});function wf(r){return sc(r,-1)}function az(r,e,t){let n=t.sharedMemoryBytes||16384,s=0;for(let u of e.entryBlock.ops())if(u.opName!=="yield")for(let p=0;p<u.numResults;p++){let c=u.getResult(p).type;if(!c||!c.shape)continue;let h=wf(c.shape);if(h<0)return!0;let f=h*bn(c.dtype);f>s&&(s=f)}let o=r.getAttr("num_carry"),i=r.getAttr("num_xs"),a=0;for(let u=0;u<o;u++){let p=r.getOperand(i+u).type,c=p&&p.shape?wf(p.shape):-1;c>0&&(a+=c*bn(p.dtype))}return 3*s+2*a>n}function uz(r,e){let n=r.regions[0].entryBlock,s=r.getAttr("num_carry"),o=[],i=null;for(let b of n.ops()){if(b.opName==="yield"){i=b;continue}o.push(b)}if(!i)return null;let a=new Set(o),u=new Set(n.arguments),p=[],c=new Set,h=[],f=new Set;for(let b of o)for(let w=0;w<b.numOperands;w++){let x=b.getOperand(w),S=x.definingOp;if(!u.has(x)&&!(S&&a.has(S))){if(S&&oo(S)){f.has(S)||(f.add(S),h.push(S));continue}c.has(x)||(c.add(x),p.push(x))}}let d=[...n.arguments,...p];for(let b of d)if(!b.type||!b.type.isFullyStatic)return null;let m=[];for(let b=0;b<i.numOperands;b++)m.push(i.getOperand(b));let g=new un(e,d.map(b=>b.type),m.map(b=>b.type)),y=new Map;for(let b=0;b<d.length;b++)y.set(d[b],g.args[b]);for(let b of h)g.entryBlock.pushOp(b.clone(y));for(let b of o)g.entryBlock.pushOp(b.clone(y));let _=m.map(b=>y.get(b));return _.some(b=>b===void 0)?null:(g.entryBlock.pushOp(new Re("return",_,[])),{bodyFunc:g,captured:p,numCarry:s,numYs:m.length-s})}function lz(r,e,t,n,s){let{getSlot:o,newSlot:i,steps:a,addedFuncs:u}=n,p=new wn(r.name+"_mod");p.addFunction(r);let c=Ti(p,s);if(c){let h=c.plan,f=e.length,d=new Map,m=!0;for(let g of h.returnFixups||[]){if(g.kind!=="copy"){m=!1;break}d.set(g.pos,g.srcSlot)}if(m){let g=l(x=>{let S=f+x;return d.has(S)?d.get(S):h.argSlots[S]},"retSlot"),y=new Map;for(let x=0;x<f;x++)y.set(h.argSlots[x],o(e[x]));for(let x of h.intermediates)y.set(x.slot,i(x.shape,x.dtype));for(let x=0;x<t.length;x++){let S=g(x);y.has(S)||y.set(S,o(t[x]))}let _=l(x=>{let S=y.get(x);return S===void 0?null:S},"mapSlot"),b=[],w=!0;for(let x of h.steps){let S=x.inputSlots.map(_),T=x.outputSlots.map(_);if(S.includes(null)||T.includes(null)){w=!1;break}b.push({name:x.name,inputSlots:S,outputSlots:T})}if(w){for(let x of b)a.push(x);for(let x of p.functions())u.push(x);return!0}}}return a.push({name:r.name,inputSlots:e.map(o),outputSlots:t.map(o)}),u.push(r),!0}function nM(r,e,t){let n=dy({ops:r,opSet:new Set(r)},e,new Map);return n?lz(n.subFunc,n.inputs,n.outputs,t,1):!1}function cz(r,e,t){let{getSlot:n,newSlot:s,steps:o,scanLoops:i,addedFuncs:a}=t,u=r.regions[0];if(!u||!u.entryBlock)return!1;let p=uz(r,e);if(!p)return!1;let{bodyFunc:c,captured:h,numCarry:f,numYs:d}=p,m=r.getAttr("num_xs"),g=new wn(e+"_mod");g.addFunction(c);let y=Ti(g,2),_=y?y.plan:null,b=[],w=[];for(let P=0;P<f;P++){let V=r.getOperand(m+P).type;b.push(V.shape),w.push(V.dtype)}let x=[],S=[];for(let P=0;P<m;P++){let V=r.getOperand(P).type;x.push(V.shape.slice(1)),S.push(V.dtype)}let T=[],k=[];for(let P=0;P<d;P++){let V=r.getResult(f+P).type;T.push(V.shape.slice(1)),k.push(V.dtype)}let A=b.map((P,V)=>s(P,w[V])),N=b.map((P,V)=>s(P,w[V])),v=x.map((P,V)=>s(P,S[V])),O=[],R=[],B=[],E=[];for(let P=0;P<f;P++)R.push(n(r.getOperand(m+P)));for(let P=0;P<f;P++)B.push(n(r.getResult(P)));for(let P of h)E.push(n(P));let z=[],D=[];for(let P=0;P<m;P++)z.push(n(r.getOperand(P)));for(let P=0;P<d;P++)D.push(n(r.getResult(f+P)));let J=o.length;if(!_)O=T.map((P,V)=>s(P,k[V])),o.push({name:c.name,inputSlots:[...v,...A,...E],outputSlots:[...N,...O]}),a.push(c);else{let P=m+f+h.length,V=new Map;for(let G of _.returnFixups||[]){if(G.kind!=="copy")return!1;V.set(G.pos,G.srcSlot)}let he=l(G=>{let K=P+G;return V.has(K)?V.get(K):_.argSlots[K]},"bodyReturnSlot"),le=new Map;for(let G=0;G<m;G++)le.set(_.argSlots[G],v[G]);for(let G=0;G<f;G++)le.set(_.argSlots[m+G],A[G]);for(let G=0;G<h.length;G++)le.set(_.argSlots[m+f+G],E[G]);for(let G of _.intermediates)le.set(G.slot,s(G.shape,G.dtype));for(let G=0;G<f;G++){let K=he(G);le.has(K)||le.set(K,N[G])}for(let G=0;G<d;G++){let K=he(f+G),ie=le.get(K);ie===void 0&&(ie=s(T[G],k[G]),le.set(K,ie)),O.push(ie)}let Ie=l(G=>{let K=le.get(G);return K===void 0?null:K},"mapSlot");for(let G of _.steps){let K=G.inputSlots.map(Ie),ie=G.outputSlots.map(Ie);if(K.includes(null)||ie.includes(null))return!1;o.push({name:G.name,inputSlots:K,outputSlots:ie})}for(let G of g.functions())a.push(G)}let j=o.length,U=r.getOperand(0).type.shape[0];return typeof U!="number"||U<0?!1:(i.push({T:U,loopStart:J,loopEnd:j,carry:b.map((P,V)=>({a:A[V],b:N[V],initSlot:R[V],finalSlot:B[V],bytes:wf(P)*bn(w[V])})),xs:v.map((P,V)=>({xtSlot:P,xsSlot:z[V],stepBytes:wf(x[V])*bn(S[V])})),ys:O.map((P,V)=>({ytSlot:P,ysSlot:D[V],stepBytes:wf(T[V])*bn(k[V])}))}),!0)}function hx(r,e,t=!1){if(!e||typeof e.isWebGPU!="function"||!e.isWebGPU()||r.functionCount!==1)return null;let n=r.functions().next().value,s=n.getReturnOp();if(!s)return null;let o=[];for(let R of n.ops())R.opName==="scan"&&o.push(R);if(o.length===0)return null;let i=new Set,a=l(R=>{if(!(!R||i.has(R))){i.add(R);for(let B=0;B<R.numOperands;B++)a(R.getOperand(B).definingOp)}},"visit");for(let R=0;R<s.numOperands;R++)a(s.getOperand(R).definingOp);for(let R of o)if(!i.has(R))return null;if(!t){let R=!1;for(let B of o){let E=B.regions[0];if(E&&E.entryBlock&&az(B,E,e)){R=!0;break}}if(!R)return null}let u=[];for(let R of i)oo(R)||qt(R.opName)||u.push(R);let p=Ba(u),c=new Map,h=0,f=l(R=>{let B=c.get(R);return B===void 0&&(B=h++,c.set(R,B)),B},"getSlot");for(let R of n.args)f(R);let d=[],m=l((R,B)=>{let E=h++;return d.push({slot:E,shape:[...R],dtype:B}),E},"newSlot"),g=[],y=[],_=[],b={getSlot:f,newSlot:m,steps:g,scanLoops:y,addedFuncs:_},w=new Set(o),x=[],S=0,T=0;for(let R of p)if(w.has(R)){if(x.length&&!nM(x,n.name+"_seg"+S++,b)||(x=[],!cz(R,n.name+"_scan"+T++,b)))return null}else x.push(R);if(x.length&&!nM(x,n.name+"_seg"+S++,b))return null;let k=n.args.map(f),A=[],N=new Set;for(let R=0;R<s.numOperands;R++){let B=s.getOperand(R),E=k.length;if(c.has(B)&&!(B.isBlockArgument&&B.isBlockArgument())){let z=f(B);N.has(z)?(k.push(h++),A.push({pos:E,kind:"copy",srcSlot:z})):(N.add(z),k.push(z))}else if(B.isBlockArgument&&B.isBlockArgument())k.push(h++),A.push({pos:E,kind:"copy",srcSlot:f(B)});else return null}let v=new Set(k),O=new Set;for(let[R,B]of c)if(!(v.has(B)||O.has(B))){if(O.add(B),!R.type||!R.type.isFullyStatic)return null;d.some(E=>E.slot===B)||d.push({slot:B,shape:[...R.type.shape],dtype:R.type.dtype})}r.removeFunction(n.name);for(let R of _)r.addFunction(R);return{plan:{numSlots:h,argSlots:k,intermediates:d,steps:g,returnFixups:A,scanLoops:y}}}var rM=M(()=>{"use strict";sr();xs();Yo();my();Tr();Vn();Oe();l(wf,"numel");l(az,"isScanOversized");l(uz,"buildScanBodyFunction");l(lz,"inlineNativeSplit");l(nM,"emitSegment");l(cz,"emitScanLoop");l(hx,"splitGraphForScan")});function Tf(r){dx.push(r),dx.sort((e,t)=>(e.priority??100)-(t.priority??100))}function pz(r){for(let e of dx)if(e.applies(r))return e;return null}function sM(r,e){let t=pz(e);return t?t.run(r,e):null}var dx,oM=M(()=>{"use strict";my();rM();dx=[];l(Tf,"registerGraphSplitStrategy");l(pz,"selectGraphSplitStrategy");l(sM,"splitGraph");Tf({name:"cublas",priority:10,applies:l(r=>r.config.matmulBackend==="cublas","applies"),run:l(r=>TB(r),"run")});Tf({name:"cuda-attention",priority:15,applies:l(r=>r.cudaAttention,"applies"),run:l(r=>Ti(r,1),"run")});Tf({name:"cuda-native-matmul-chain",priority:20,applies:l(r=>r.cudaMatmulChain,"applies"),run:l(r=>Ti(r),"run")});Tf({name:"cuda-native-conv-chain",priority:25,applies:l(r=>r.cudaConvChain,"applies"),run:l(r=>Ti(r),"run")});Tf({name:"webgpu",priority:30,applies:l(r=>r.isWebGPU,"applies"),run:l((r,e)=>{let t=hx(r,e.target);return!t&&bB(r,e.target.maxThreadsPerBlock||256)&&(t=hx(r,e.target,!0),t||(t=Ti(r,2))),t},"run")})});function iM(r,e){let t=new od;for(let[,n]of r.bufferMap)t.paramBuffers.add(n.name);return fz(r.body,t,e),dz(r,t,e),mz(r.body,t),t}function fz(r,e,t){let n=[r];for(;n.length>0;){let s=n.pop();if(!(!s||typeof s!="object")){switch(s.type){case"ForNode":if(s.loopVar&&e.locals.set(s.loopVar.name,"i32"),s.kind===F.THREAD_BINDING&&s.threadTag){let o=s.extent&&s.extent.type==="IntImmNode"?s.extent.value:0,i=!s.extent||s.extent.type!=="IntImmNode",a={varName:s.loopVar.name,extent:o,isDynamic:i,extentNode:s.extent};e.threadBindings.has(s.threadTag)?e.threadBindings.get(s.threadTag).push(a):e.threadBindings.set(s.threadTag,[a])}break;case"LetStmtNode":s.variable&&e.locals.set(s.variable.name,ei(s.variable.dtype));break;case"BufferStoreNode":case"BufferLoadNode":s.buffer&&e.usedBuffers.set(s.buffer.name,s.buffer);break;case"AllocateNode":s.buffer&&e.allocatedBuffers.add(s.buffer.name),s.scope==="shared"&&s.buffer&&e.sharedBuffers.push(s.buffer);break;case"CallExternNode":s.externName&&!cv(s.externName)&&e.externCalls.set(s.externName,{argCount:s.args?s.args.length:0,dtype:s.dtype||"f32"});break;case"BlockNode":if(s.reads)for(let o of s.reads)o.buffer&&e.usedBuffers.set(o.buffer.name,o.buffer);if(s.writes)for(let o of s.writes)o.buffer&&e.usedBuffers.set(o.buffer.name,o.buffer);if(s.iterVars)for(let o of s.iterVars)o.iterVar&&e.locals.set(o.iterVar.name,ei(o.iterVar.dtype)),o.binding&&typeof o.binding=="object"&&o.binding.type&&n.push(o.binding);break}if(s.body&&n.push(s.body),s.value&&typeof s.value=="object"&&s.value.type&&n.push(s.value),s.stmts)for(let o of s.stmts)n.push(o);if(s.thenBody&&n.push(s.thenBody),s.elseBody&&n.push(s.elseBody),s.initBody&&n.push(s.initBody),s.condBody&&n.push(s.condBody),s.loopBody&&n.push(s.loopBody),s.condition&&typeof s.condition=="object"&&s.condition.type&&n.push(s.condition),s.a&&typeof s.a=="object"&&s.a.type&&n.push(s.a),s.b&&typeof s.b=="object"&&s.b.type&&n.push(s.b),s.expr&&typeof s.expr=="object"&&s.expr.type&&n.push(s.expr),s.args)for(let o of s.args)typeof o=="object"&&o!==null&&o.type&&n.push(o);if(s.indices)for(let o of s.indices)typeof o=="object"&&o!==null&&o.type&&n.push(o)}}}function dz(r,e,t){let n=e.memoryLayout.alignment,s=0,o=l(i=>{let a=i.shape.some(c=>typeof c!="number"||c<0),u=i.numel();if(!a&&u>=0)return u*bn(i.dtype);let p=1;for(let c of i.shape)p*=typeof c=="number"&&c>0?c:1;return Math.max(hz,p*bn(i.dtype))},"bufBytes");for(let[,i]of r.bufferMap)s=Math.ceil(s/n)*n,e.memoryLayout.bufferOffsets.set(i.name,s),s+=o(i);for(let[i,a]of e.usedBuffers)e.memoryLayout.bufferOffsets.has(i)||(s=Math.ceil(s/n)*n,e.memoryLayout.bufferOffsets.set(i,s),s+=o(a));e.memoryLayout.totalBytes=s}function mz(r,e){let t=new Map;for(let n of gu(r,s=>s.type==="BufferStoreNode"&&s.buffer&&!e.paramBuffers.has(s.buffer.name))){let s=n.buffer.name;t.has(s)||t.set(s,[]),t.get(s).push(n.value)}for(let[n,s]of t)if(s.every(i=>i&&i.type==="IntImmNode"&&i.value===0||i&&i.type==="FloatImmNode"&&i.value===0)&&e.zeroBuffers.add(n),s.length===1&&s[0]){let i=s[0];(i.type==="IntImmNode"||i.type==="FloatImmNode")&&e.constantBuffers.set(n,i.value)}}var hz,aM=M(()=>{"use strict";Tc();Vn();De();Gr();l(iM,"scanMetadata");l(fz,"walkTree");hz=65536;l(dz,"computeMemoryLayout");l(mz,"detectZeroBuffers")});function xf(r,e,t){let n=typeof r.offset=="number"?r.offset:0;if(e.length===0)return new L(n);if(e.length===1)return n===0?e[0]:new C("+",e[0],new L(n));let s=[];n!==0&&s.push(new L(n));for(let o=0;o<e.length;o++){let i=e[o];if(i.type==="IntImmNode"&&i.value===0)continue;let a=r.strides[o];if(a===1)s.push(i);else if(typeof a=="number"&&a>=0)s.push(new C("*",i,new L(a)));else{let u=gz(r,o,t);s.push(new C("*",i,u))}}return s.length===0?new L(0):s.reduce((o,i)=>new C("+",o,i))}function gz(r,e,t){let n=[];for(let s=e+1;s<r.shape.length;s++){let o=r.shape[s];typeof o=="number"&&o>=0?n.push(new L(o)):n.push(yz(r,s,t))}return n.length===0?new L(1):n.reduce((s,o)=>new C("*",s,o))}function yz(r,e,t){let n=r.shape[e];if(n instanceof Ce)return uu(n,s=>new Ne(qa(s),"index"));if(t){let s=`${r.name}:${e}`,o=t.get(s);if(o)return new Ne(o.name,"index")}return new L(1)}var uM=M(()=>{"use strict";De();Vs();Eb();l(xf,"flattenIndex");l(gz,"computeDynamicStride");l(yz,"resolveShapeParam")});function lM(r,e){let t=iM(r,e),n={shapeParamMap:r.shapeParamMap,accCounter:0,metadata:t},s=is(r.body,n);return new rd(r.name,r.params,s,r.bufferMap,r.shapeParams,r.shapeParamMap,t)}function is(r,e){if(!r||typeof r!="object")return r;switch(r.type){case"ForNode":return _z(r,e);case"BlockNode":return bz(r,e);case"SeqNode":return wz(r,e);case"BufferStoreNode":return Tz(r,e);case"LetStmtNode":return xz(r,e);case"AllocateNode":return Sz(r,e);case"IfThenElseNode":return kz(r,e);case"WhileNode":return vz(r,e);case"EvaluateNode":return new ou(mr(r.value,e));case"SyncThreadsNode":return r;default:return r}}function _z(r,e){let t=r.accumulator!==void 0?r.accumulator:Xy(r);if(t)return Nz(r,t,e);let n=is(r.body,e);return new de(r.loopVar,r.min,r.extent,r.kind,n,r.threadTag)}function bz(r,e){let t=[];for(let o of r.iterVars)o.iterVar&&o.binding&&t.push({name:o.iterVar.name,dtype:o.iterVar.dtype,expr:mr(o.binding,e)});let n=is(r.body,e),s=r.initBody?is(r.initBody,e):null;return t.length>0?s?(n=new te([new mu(t,s),new mu(t,n)]),s=null):n=new mu(t,n):s&&(n=new te([s,n])),n}function wz(r,e){let t=[];for(let n of r.stmts)t.push(is(n,e));return new te(t)}function Tz(r,e){let t=xf(r.buffer,r.indices,e.shapeParamMap),n=mr(r.value,e),s=r.buffer.dtype||ir(r.value);return new wc(r.buffer,t,n,s)}function xz(r,e){return new Ye(r.variable,mr(r.value,e),is(r.body,e))}function Sz(r,e){return new Ve(r.buffer,r.scope,is(r.body,e))}function kz(r,e){return new xe(mr(r.condition,e),is(r.thenBody,e),r.elseBody?is(r.elseBody,e):null)}function vz(r,e){return new Zo(r.condVar,is(r.condBody,e),is(r.loopBody,e))}function mr(r,e){if(!r||typeof r!="object"||!r.type)return r;switch(r.type){case"BufferLoadNode":{let t=xf(r.buffer,r.indices,e.shapeParamMap),n=new bc(r.buffer,t,r.buffer.dtype);return No(n),n}case"MathOpNode":{let t=mr(r.a,e),n=r.b?mr(r.b,e):null,s={...r,a:t,b:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),No(s),s}case"CompareNode":{let t=mr(r.a,e),n=mr(r.b,e),s={...r,a:t,b:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),No(s),s}case"CastNode":{let t=mr(r.expr,e),n={...r,expr:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),No(n),n}case"CallExternNode":{let t=r.args.map(s=>mr(s,e)),n={...r,args:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),No(n),n}case"IfThenElseNode":{let t=mr(r.condition,e),n=mr(r.thenBody,e),s=r.elseBody?mr(r.elseBody,e):null,o={...r,condition:t,thenBody:n,elseBody:s};return Object.setPrototypeOf(o,Object.getPrototypeOf(r)),No(o),o}default:return No(r),r}}function Nz(r,e,t){let n=`_acc_${t.accCounter++}`,s=ir(e.loadSide);t.metadata.locals.set(n,s);let o=new Map;for(let d of e.block.iterVars)d.iterVar&&d.binding&&o.set(d.iterVar.name,d.binding);let i=o.size>0?Ar(e.valueSide,o):e.valueSide,a=xf(e.store.buffer,e.outerIndices,t.shapeParamMap),u=new bc(e.store.buffer,a,e.store.buffer.dtype);No(u);let p=mr(i,t),c=xf(e.store.buffer,e.outerIndices,t.shapeParamMap),h=new wc(e.store.buffer,c,null,e.store.buffer.dtype),f=e.block.initBody?o.size>0?Sf(e.block.initBody,o):e.block.initBody:null;return new sd({localName:n,dtype:s,op:e.op,initLoad:u,loopVar:r.loopVar,extent:r.extent,loopKind:r.kind,body:p,flushStore:h,initBody:f?is(f,t):null})}function Ar(r,e){if(!r||typeof r!="object"||!r.type)return r;if(r.type==="VariableNode"&&e.has(r.name))return e.get(r.name);if(r.type==="BufferLoadNode"){let t=r.indices.map(o=>Ar(o,e));if(!t.some((o,i)=>o!==r.indices[i]))return r;let s={...r,indices:t};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}if(r.type==="MathOpNode"){let t=Ar(r.a,e),n=r.b?Ar(r.b,e):null;if(t===r.a&&n===r.b)return r;let s={...r,a:t,b:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}if(r.type==="CompareNode"){let t=Ar(r.a,e),n=Ar(r.b,e);if(t===r.a&&n===r.b)return r;let s={...r,a:t,b:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}if(r.type==="CastNode"){let t=Ar(r.expr,e);if(t===r.expr)return r;let n={...r,expr:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),n}if(r.type==="CallExternNode"){let t=r.args.map(o=>Ar(o,e));if(!t.some((o,i)=>o!==r.args[i]))return r;let s={...r,args:t};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}if(r.type==="IfThenElseNode"){let t=Ar(r.condition,e),n=Ar(r.thenBody,e),s=r.elseBody?Ar(r.elseBody,e):null;if(t===r.condition&&n===r.thenBody&&s===r.elseBody)return r;let o={...r,condition:t,thenBody:n,elseBody:s};return Object.setPrototypeOf(o,Object.getPrototypeOf(r)),o}return r}function Sf(r,e){if(!r||typeof r!="object"||!r.type)return r;switch(r.type){case"BufferStoreNode":{let t=r.indices.map(o=>Ar(o,e)),n=Ar(r.value,e);if(t.every((o,i)=>o===r.indices[i])&&n===r.value)return r;let s={...r,indices:t,value:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}case"SeqNode":{let t=r.stmts.map(s=>Sf(s,e)),n={...r,stmts:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),n}case"ForNode":{let t=Sf(r.body,e);if(t===r.body)return r;let n={...r,body:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),n}case"IfThenElseNode":{let t=Ar(r.condition,e),n=Sf(r.thenBody,e),s=r.elseBody?Sf(r.elseBody,e):null,o={...r,condition:t,thenBody:n,elseBody:s};return Object.setPrototypeOf(o,Object.getPrototypeOf(r)),o}default:return r}}var cM=M(()=>{"use strict";Tc();aM();uM();De();sx();l(lM,"lowerToLIR");l(is,"lowerStmt");l(_z,"lowerForNode");l(bz,"lowerBlockNode");l(wz,"lowerSeqNode");l(Tz,"lowerBufferStore");l(xz,"lowerLetStmt");l(Sz,"lowerAllocate");l(kz,"lowerIfThenElse");l(vz,"lowerWhile");l(mr,"lowerExpr");l(Nz,"lowerAccumulator");l(Ar,"substituteVars");l(Sf,"substituteVarsStmt")});function pM(r){let e=[],t={errors:e,boundVars:new Set,path:[],bufferNames:new Set};if(!r||r.type!=="LIRFunc")return e.push(new lo("root must be LIRFunc")),e;for(let[,n]of r.bufferMap)t.bufferNames.add(n.name);for(let n of r.shapeParams)t.boundVars.add(n.name);if(r.metadata)for(let[n]of r.metadata.locals)t.boundVars.add(n);return as(r.body,t),e}function as(r,e){if(!(!r||typeof r!="object")){switch(e.path.push(r.type),r.type){case"ForNode":Az(r,e);break;case"SeqNode":for(let t of r.stmts)as(t,e);break;case"LIRFlatStoreNode":Oz(r,e);break;case"LIRAccumulatorNode":Bz(r,e);break;case"LIRBindingsNode":Rz(r,e);break;case"LetStmtNode":{vn(r.value,e);let t=e.boundVars.has(r.variable.name);e.boundVars.add(r.variable.name),as(r.body,e),t||e.boundVars.delete(r.variable.name);break}case"AllocateNode":{let t=r.buffer?e.bufferNames.has(r.buffer.name):!0;r.buffer&&e.bufferNames.add(r.buffer.name),as(r.body,e),r.buffer&&!t&&e.bufferNames.delete(r.buffer.name);break}case"IfThenElseNode":vn(r.condition,e),as(r.thenBody,e),r.elseBody&&as(r.elseBody,e);break;case"WhileNode":as(r.condBody,e),as(r.loopBody,e);break;case"EvaluateNode":vn(r.value,e);break;default:break}e.path.pop()}}function Az(r,e){r.extent?vn(r.extent,e):e.errors.push(new lo("ForNode missing extent",[...e.path]));let t=!0;r.loopVar?(t=e.boundVars.has(r.loopVar.name),e.boundVars.add(r.loopVar.name)):e.errors.push(new lo("ForNode missing loopVar",[...e.path])),as(r.body,e),r.loopVar&&!t&&e.boundVars.delete(r.loopVar.name)}function Oz(r,e){r.buffer||e.errors.push(new lo("LIRFlatStoreNode missing buffer",[...e.path])),vn(r.offsetExpr,e),vn(r.value,e)}function Bz(r,e){r.localName||e.errors.push(new lo("LIRAccumulatorNode missing localName",[...e.path])),r.dtype||e.errors.push(new lo("LIRAccumulatorNode missing dtype",[...e.path]));let t=r.localName?e.boundVars.has(r.localName):!0;r.localName&&e.boundVars.add(r.localName),vn(r.initLoad,e),vn(r.body,e),as(r.flushStore,e),r.initBody&&as(r.initBody,e),r.localName&&!t&&e.boundVars.delete(r.localName)}function Rz(r,e){let t=[];for(let n of r.bindings)vn(n.expr,e),e.boundVars.has(n.name)||t.push(n.name),e.boundVars.add(n.name);as(r.body,e);for(let n of t)e.boundVars.delete(n)}function vn(r,e){if(!(!r||typeof r!="object"||!r.type))switch(r.type){case"LIRFlatLoadNode":r.buffer||e.errors.push(new lo("LIRFlatLoadNode missing buffer",[...e.path])),vn(r.offsetExpr,e);break;case"MathOpNode":vn(r.a,e),r.b&&vn(r.b,e);break;case"CompareNode":vn(r.a,e),vn(r.b,e);break;case"CastNode":vn(r.expr,e);break;case"CallExternNode":if(r.args)for(let t of r.args)vn(t,e);break;case"IfThenElseNode":vn(r.condition,e),vn(r.thenBody,e),r.elseBody&&vn(r.elseBody,e);break;case"VariableNode":r.name!==void 0&&!e.boundVars.has(r.name)&&e.errors.push(new lo(`unbound variable '${r.name}'`,[...e.path]));break;case"IntImmNode":case"FloatImmNode":break;default:break}}var lo,fM=M(()=>{"use strict";lo=class{static{l(this,"LIRVerificationError")}constructor(e,t){this.message=e,this.nodePath=t||[]}toString(){let e=this.nodePath.length>0?` at ${this.nodePath.join(" > ")}`:"";return`LIR verification: ${this.message}${e}`}};l(pM,"verifyLIR");l(as,"verifyStmt");l(Az,"verifyForNode");l(Oz,"verifyFlatStore");l(Bz,"verifyAccumulator");l(Rz,"verifyBindings");l(vn,"verifyExpr")});function hM(r,e,t={}){return new Do({target:e,...t}).compileFunction(r)}var a_,mx,Do,u_=M(()=>{"use strict";Yo();xs();VO();am();GO();AB();WR();Ui();Oc();tm();HR();ZR();lx();XR();tM();oM();jT();Kt();P0();cT();cM();fM();Kt();a_=class{static{l(this,"CompilerConfig")}constructor(e={}){this.target=e.target,this.verify=e.verify!==!1,this.verifyMode=e.verify==="full"?"full":"normal",this.errorMode=e.errorMode||"strict";let t=this.target&&typeof this.target.isWebGPU=="function"&&this.target.isWebGPU(),s=this.target&&typeof this.target.isGPU=="function"&&this.target.isGPU()&&!t;this.fusion={enabled:!0,strategy:"priority",epilogue:void 0,...e.fusion},this.scheduling={enabled:t,autotune:!1,gpuTiling:s,...e.scheduling},this.matmulBackend=e.matmulBackend||"native",this.quantization={enabled:!1,...e.quantization},this.optimization={layout:!1,rematerialization:!1,rematConfig:{},fastMath:!1,maxSimplifyIterations:8,loopPartition:!1,detectAccumulators:!1,tensorize:!1,...e.optimization},this.memory={alignment:64,inplaceReuse:!0,allocStrategy:"best-fit",poolAllocation:!1,...e.memory},this.partition={enabled:!1,targets:[],defaultTarget:null,opTargetOverrides:new Map,memoryLimits:new Map,minPartitionSize:1,costWeights:{},...e.partition},this.passContext=e.passContext||null,this.loweringRules=e.loweringRules||null,this.codegenEntries=e.codegenEntries||null;let o=e.trace||{};this.trace={level:o.level??ue.SILENT,sink:o.sink??null,irSnapshot:{afterGraphPasses:!1,afterLowering:!1,afterScheduling:!1,...o.irSnapshot||{}}}}get usePartition(){return this.partition.enabled&&this.partition.targets.length>=2}},mx=class{static{l(this,"CompilationResult")}constructor(e,t,n){this.module=e,this.trace=t,this.errors=n||[]}get succeeded(){return this.errors.length===0}get failedFunctions(){let e=new Set;for(let t of this.errors)t.funcName&&e.add(t.funcName);return e}run(e,...t){return this.module.run(e,...t)}async runAsync(e,...t){return this.module.runAsync(e,...t)}isAsync(e){return this.module.isAsync(e)}getSource(e){return this.module.getKernelSource(e)}listKernels(){return this.module.listKernels()}},Do=class r{static{l(this,"Compiler")}constructor(e){if(this.config=e instanceof a_?e:new a_(e),!this.config.target)throw new Error("Compiler requires a target");this.context=new Gg({loweringRules:this.config.loweringRules,codegenEntries:this.config.codegenEntries})}compile(e){let t=new im(this.config.trace),n=this.config.errorMode==="resilient",s=[],o=new Set,i=performance.now();t.phaseStart("compile");let a={compiler:this,trace:t,errors:s,failed:o,resilient:n,original:e,working:n?Ok(e):e,cudaMatmulChain:!1,split:null,primFuncs:null,lirFuncs:null,runtimeModule:null};for(let u of this._compilePhases())u.when&&!u.when(a)||u.run(a);if(t.phaseEnd("compile",performance.now()-i),!n&&s.length>0)throw new Error(s[0].toString());return new mx(a.runtimeModule,t,s)}_compilePhases(){return[{name:"verify:pre",when:l(e=>e.compiler.config.verify,"when"),run:l(e=>e.compiler._verifyGraph(e.working,"before graph passes",e.trace,e.errors,e.failed,e.resilient),"run")},{name:"calibrate",when:l(e=>{let t=e.compiler.config.quantization;return t.enabled&&t.calibrationData&&!t.calibration},"when"),run:l(e=>e.compiler._runCalibration(e.working,e.trace),"run")},{name:"graphPasses",run:l(e=>{e.cudaMatmulChain=e.compiler._runGraphPasses(e.working,e.original,e.trace,e.errors,e.failed,e.resilient)},"run")},{name:"partition",when:l(e=>e.compiler.config.usePartition,"when"),run:l(e=>e.compiler._runPartitioning(e.working,e.trace),"run")},{name:"split",run:l(e=>{let t=e.compiler.config,n=typeof t.target.isWebGPU=="function"&&t.target.isWebGPU(),s=typeof t.target.isGPU=="function"&&t.target.isGPU()&&!n,o=0,i=0;for(let p of e.working)for(let c of p.ops())c.opName==="conv"||c.opName==="quantized_conv"?o++:c.opName==="scaled_dot_product_attention"&&i++;let a=s&&o>=2,u=s&&i>0;e.split=sM(e.working,{config:t,target:t.target,cudaMatmulChain:e.cudaMatmulChain,cudaConvChain:a,cudaAttention:u,isWebGPU:n})},"run")},{name:"verify:post",when:l(e=>e.compiler.config.verify,"when"),run:l(e=>e.compiler._verifyGraph(e.working,"after graph passes",e.trace,e.errors,e.failed,e.resilient),"run")},{name:"lowering",run:l(e=>{if(e.primFuncs=e.compiler._lowerAll(e.working,e.trace,e.errors,e.failed,e.resilient),e.compiler.config.matmulBackend==="cublas")for(let t of e.primFuncs)t.cublasInfo=e.split&&e.split.cublasInfos?e.split.cublasInfos.get(t.name)||null:Cv(t)},"run")},{name:"tirPasses",run:l(e=>e.compiler._runTirPasses(e),"run")},{name:"verify:tensor",when:l(e=>e.compiler.config.verify,"when"),run:l(e=>e.compiler._verifyAll(e.primFuncs,e.errors,e.failed,e.resilient),"run")},{name:"lirLowering",run:l(e=>{e.lirFuncs=e.compiler._lowerToLIR(e.primFuncs,e.trace,e.errors,e.failed,e.resilient)},"run")},{name:"codegen",run:l(e=>{e.runtimeModule=e.compiler._codegen(e.lirFuncs,e.trace,e.errors,e.failed,e.resilient),e.split&&(e.runtimeModule.executionPlan=e.split.plan)},"run")}]}compileFunction(e){let t=new wn("single");return t.addFunction(e),this.compile(t)}calibrate(e,t="minmax"){let n=new El(t);for(let s of e)n.attach(s);return n}_runCalibration(e,t){let n=this.config.quantization,s=e.functionNames()[0],o=e.getFunction(s);if(!o)return;t.phaseStart("calibrate");let i=performance.now(),a=this.config.target,u=l((c,h)=>new r({target:h,verify:this.config.verify}).compile(c),"compileFn"),p=QR(o,a,n.calibrationData,{mode:n.calibrationMode||"minmax",quantizableOps:n.quantizableOps,compileFn:u});this.config.quantization={...n,calibration:p},t.phaseEnd("calibrate",performance.now()-i)}_runGraphPasses(e,t,n,s,o,i){let a=new oa,u=0;for(let m of e)for(let g of m.ops())g.opName==="dot"&&u++;let p=this.config.target,c=(p.getAttr&&p.getAttr("matmulChainThreshold"))??(p.kind==="cuda"?2:1/0),h=u>=c;for(let m of NB(this.config,this.config.target,{cudaMatmulChain:h,context:this.context}))a.addPass(m);a.setTrace(n),this.config.verifyMode==="full"&&a.setVerifyHook((m,g)=>(g?ix(m):bf(m)).map(_=>_.toString())),n.phaseStart("graphPasses");let f=performance.now(),d=a.run(e,{errorMode:i?"resilient":"strict",passContext:this.config.passContext});if(d.errors){for(let m of d.errors)s.push(m),n.errorEvent(m.phase,m.funcName,m.message,m.passName);if(d.failedFunctions){for(let m of d.failedFunctions)if(o.add(m),i&&t&&t!==e){let g=t.getFunction(m);g&&e.addFunction(zh(g))}}}if(n.phaseEnd("graphPasses",performance.now()-f),n.shouldSnapshot("afterGraphPasses")){let m=new lp;n.irDump("afterGraphPasses",m.printModule(e))}return h}_runPartitioning(e,t){let n=new oa;n.addPass(new o_(this.config.partition)),n.addPass(new i_({targets:this.config.partition.targets})),n.setTrace(t),t.phaseStart("partition");let s=performance.now();n.run(e),t.phaseEnd("partition",performance.now()-s)}_eachFunc(e,t,n,s,o,i,a){for(let u of e)if(!o.has(u.name))try{a(u)}catch(p){if(s.push(new Dn(t,u.name,p.message)),o.add(u.name),n.errorEvent(t,u.name,p.message),!i)break}}_lowerAll(e,t,n,s,o){t.phaseStart("lowering");let i=performance.now(),a=[];return this._eachFunc(e,"lowering",t,n,s,o,u=>{let p=performance.now(),c=Qh(u,this.config.target,this.context);this.config.target.isGPU&&this.config.target.isGPU()&&!(this.config.target.isWebGPU&&this.config.target.isWebGPU())&&(c.convInfo=mR(u)),t.functionEvent("lowering",u.name,{durationMs:performance.now()-p}),a.push(c),t.shouldSnapshot("afterLowering")&&t.irDump("afterLowering:"+u.name,jg(c))}),t.phaseEnd("lowering",performance.now()-i),a}_runTirPasses(e){let t=new qg;for(let n of KR(this.config))t.addPass(n);if(t.setTrace(e.trace),this.config.verifyMode==="full"){let n=new _f;t.setVerifyHook(s=>n.verify(s))}t.run(e.primFuncs,{trace:e.trace,errors:e.errors,failed:e.failed,resilient:e.resilient})}_verifyGraph(e,t,n,s,o,i){if(i){for(let u of e){if(o.has(u.name))continue;let p=bf?bf(u):[];if(p.length>0){let c=p.map(h=>h.toString()).join("; ");s.push(new Dn("verification",u.name,c)),o.add(u.name),n.errorEvent("verification",u.name,c)}}return}let a=ix(e);if(a.length>0)throw new Error("Graph verification failed ("+t+"): "+a.map(u=>u.toString()).join("; "))}_verifyAll(e,t,n,s){let o=new _f;for(let i of e){if(n.has(i.name))continue;let a=o.verify(i);if(a.length>0){let u=a.join("; ");if(s)t.push(new Dn("verification",i.name,u)),n.add(i.name);else throw new Error("TensorIR verification failed for "+i.name+": "+u)}}}_lowerToLIR(e,t,n,s,o){t.phaseStart("lirLowering");let i=performance.now(),a=[];return this._eachFunc(e,"lirLowering",t,n,s,o,u=>{let p=performance.now(),c=lM(u,this.config.target);if(u.cublasInfo&&(c.cublasInfo=u.cublasInfo),u.gpuRegisterBlocked&&(c.gpuRegisterBlocked=!0),this.config.verifyMode==="full"){let h=pM(c);if(h.length>0)throw new Error("LIR verification failed: "+h.map(f=>f.toString()).join("; "))}t.functionEvent("lirLowering",u.name,{durationMs:performance.now()-p}),a.push(c)}),t.phaseEnd("lirLowering",performance.now()-i),a}_codegen(e,t,n,s,o){t.phaseStart("codegen");let i=performance.now(),a=new Fu("compiled"),u=this.config.usePartition,p={matmulBackend:this.config.matmulBackend,context:this.context},c=new Map,h=l(d=>(c.has(d.name)||c.set(d.name,new Ws(d,p)),c.get(d.name)),"getBackend"),f=u?null:new Ws(this.config.target,p);return this._eachFunc(e,"codegen",t,n,s,o,d=>{let m=performance.now(),g;if(u){let _=d._partitionTarget,b=_?this.config.partition.targets.find(w=>w.name===_):this.config.target;g=h(b||this.config.target)}else g=f;let y=g.compile(d);a.addCompiledKernel(y),d.shapeParamMap&&d.shapeParamMap.size>0&&a.setShapeParamMap(d.name,d.shapeParamMap,d.bufferMap),t.codegenStats(d.name,{durationMs:performance.now()-m,sourceSize:y.source.length,targetName:y.target.name})}),t.phaseEnd("codegen",performance.now()-i),a}};l(hM,"compileGraph")});function Iz(r){return typeof r=="object"&&r!==null}function Ez(r){return Iz(r)&&"_impl"in r}function Lz(r){return(e,...t)=>{let n=ss();if(!n)throw new Error(`TRACING dispatch key active but no tracer is set for op '${r}'`);if(Mz.has(r)){let c=Nt.findOp(r);return Nt.redispatch(c,e,...t)}let s=[],o={},i=Sm(r),a=0,u=l(c=>c instanceof ft||Ez(c),"isTensor"),p=l(c=>{c instanceof ft||c.isSymbolic?s.push(c):s.push(n.captureConstant(c))},"pushTensor");for(let c of t)if(Array.isArray(c)&&c.length>0&&u(c[0]))for(let h of c)p(h);else u(c)?p(c):i&&(c!=null&&a<i.length&&(o[i[a]]=c),a++);return n.recordOp(r,s,o)}}function mM(){let r=Nt.listOps();for(let e of r){let t=Nt.findOp(e);if(!t)continue;let n=t.name,s=wr.fromUnboxed(Lz(n));t.entry.registerKernel(48,s)}}var Mz,gM=M(()=>{"use strict";Nn();tc();$i();ya();ga();km();Mz=new Set(["scatter","repeat","tile","split","chunk","roll","flip","cumsum","sort","argsort","topk"]);l(Iz,"_isObject");l(Ez,"_hasImpl");l(Lz,"_tracingKernel");l(mM,"registerTracingDispatch")});function Dz(r){return r&&r.shape&&r.shape.length>=2&&Cz.has(r.dtype)}function yM(r,e,t=Dz){let n=r.graph.functions().next().value;if(!n)return r;let s=n.entryBlock,o=r.numUserInputs,i=r.capturedParams,a=[];for(let h=0;h<i.length;h++){let f=o+h,d=s.getArgument(f);d&&t(i[h],h,d)&&a.push({j:h,argIndex:f,arg:d,param:i[h]})}if(a.length===0)return r;for(let h of a){let f=e(h.param),d=h.arg.type,m=Re,g=new m("constant",[],[d],{value:f,tensor_type:d}),y=s.firstOp;y?s.insertBefore(g,y):s.pushOp(g),h.arg.replaceAllUsesWith(g.getResult(0))}let u=new Set(a.map(h=>h.argIndex));s.removeArguments(u),n.inputTypes=Object.freeze(n.inputTypes.filter((h,f)=>!u.has(f)));let p=new Set(a.map(h=>h.j)),c=i.filter((h,f)=>!p.has(f));return{...r,capturedParams:c}}var Cz,_M=M(()=>{"use strict";sr();Cz=new Set(["f16","f32","f64"]);l(Dz,"defaultWeightPredicate");l(yM,"foldWeightParams")});function kf(r,e,t){let n=t.get(e.id);if(n)return n;let s=r.scalarConstant(0,e.type.dtype).getResult(0);return r.broadcast(s,e.type.shape,[]).getResult(0)}var co,vf=M(()=>{"use strict";l(kf,"gradOrZero");co=class{static{l(this,"GradAccumulator")}constructor(e){this._builder=e,this._pending=new Map,this._reduced=new Map}accumulate(e,t){if(!t)return;let n=this._pending.get(e);n||(n=[],this._pending.set(e,n)),n.push(t),this._reduced.delete(e)}_treeReduce(e){let t=e;for(;t.length>1;){let n=[];for(let s=0;s<t.length;s+=2)s+1<t.length?n.push(this._builder.add(t[s],t[s+1]).getResult(0)):n.push(t[s]);t=n}return t[0]}get(e){if(this._reduced.has(e))return this._reduced.get(e);let t=this._pending.get(e);if(!t||t.length===0)return null;let n=this._treeReduce(t);return this._reduced.set(e,n),n}has(e){let t=this._pending.get(e);return!!t&&t.length>0}}});function ce(r,e){gx.set(r,e)}function Ia(r){return gx.get(r)||null}function f_(r){let e=gx.get(r);if(e)return e;if(yx.has(r))return null;throw new Error(`autodiff: op '${r}' is on the gradient path but has no VJP rule and is not a registered gradient barrier. Register one with registerVJPRule('${r}', ...) or registerGradientBarrier('${r}').`)}function _x(r,e){bM.set(r,e)}function h_(r){return bM.get(r)||null}function bx(r){yx.add(r)}function d_(r){return yx.has(r)}var gx,yx,bM,ls=M(()=>{"use strict";gx=new Map,yx=new Set;l(ce,"registerVJPRule");l(Ia,"getVJPRule");l(f_,"requireVJPRuleOrBarrier");bM=new Map;l(_x,"registerRegionVJP");l(h_,"getRegionVJP");l(bx,"registerGradientBarrier");l(d_,"isGradientBarrier")});var Po,wx=M(()=>{"use strict";Po=new Set(["scan","if"])});function Nf(r){let e=new Set(r.arguments.map(i=>i.id)),t=l(i=>{for(let a of i.ops()){for(let u of a.results)e.add(u.id);for(let u of a.regions||[])for(let p of u.blocks){for(let c of p.arguments)e.add(c.id);t(p)}}},"addLocals");t(r);let n=new Set,s=[],o=l(i=>{for(let a of i.ops()){for(let u of a.operands)e.has(u.id)||n.has(u.id)||u.definingOp&&u.definingOp.opName==="constant"||(n.add(u.id),s.push(u));for(let u of a.regions||[])for(let p of u.blocks)o(p)}},"scan");return o(r),s}function m_(r,e){let t=r.scalarConstant(0,e.type.dtype).getResult(0);return r.broadcast(t,e.type.shape,[]).getResult(0)}function wM(r,e,t){let n=e.type.shape,s=n.map((a,u)=>u===0?t:0),o=n.map((a,u)=>u===0?t+1:a),i=r.slice(e,s,o).getResult(0);return r.reshape(i,n.slice(1)).getResult(0)}function $z(r,e,t){let n=t.slice(1),s=e.map(o=>r.reshape(o,[1,...n]).getResult(0));return s.length===1?r.reshape(s[0],t).getResult(0):r.concat(s,0).getResult(0)}function g_(r,e,t,n,s,o,i=new Map,a=[]){let u=new Map,p=new Map;for(let _=0;_<e.arguments.length;_++)u.set(e.arguments[_].id,t[_]),p.set(e.arguments[_],t[_]);for(let[_,b]of n)u.set(_,b);for(let _ of a)n.has(_.id)&&p.set(_,n.get(_.id));let c=[],h=null;for(let _ of e.ops())_.opName==="yield"?h=_:c.push(_);let f=l(_=>{if(u.has(_.id))return u.get(_.id);let b=_.definingOp;if(b&&b.opName==="constant"){let w=i.get(_.id);return w===void 0&&(w=r._buildOp("constant",[],[_.type],new Map(b.attributes),null).getResult(0),i.set(_.id,w)),u.set(_.id,w),p.set(_,w),w}return _},"matOperand");for(let _ of c){if(Po.has(_.opName)){for(let S of _.operands)f(S),p.set(S,u.get(S.id)??S);let x=_.clone(p);r.block.pushOp(x);for(let S=0;S<_.numResults;S++)u.set(_.getResult(S).id,x.getResult(S)),p.set(_.getResult(S),x.getResult(S));continue}let b=_.operands.map(f),w=r._buildOp(_.opName,b,_.results.map(x=>x.type),new Map(_.attributes),null);for(let x=0;x<_.numResults;x++)u.set(_.getResult(x).id,w.getResult(x)),p.set(_.getResult(x),w.getResult(x))}let d=h.operands.map(_=>u.get(_.id));if(o)return{forwardYields:d};let m=new co(r);for(let _=0;_<h.operands.length;_++)s[_]&&m.accumulate(h.operands[_].id,s[_]);for(let _=c.length-1;_>=0;_--){let b=c[_];if(b.opName==="constant")continue;let w=b.results.map(k=>m.get(k.id));if(w.every(k=>k===null))continue;if(Po.has(b.opName)){let k=h_(b.opName);k&&k(b,{accumulator:m,builder:r,materialize:f,needsGrad:Pz,scanCheckpoint:null});continue}let x=f_(b.opName);if(!x)continue;let S={builder:r,op:b,operands:b.operands.map(f),results:b.results.map(k=>u.get(k.id)),gradOutputs:w,attrs:b.attributes,full:l((k,A)=>r.broadcast(r.scalarConstant(k,A.dtype).getResult(0),A.shape,[]).getResult(0),"full")},T=x(S);if(T)for(let k=0;k<b.numOperands;k++)T[k]&&m.accumulate(b.getOperand(k).id,Tx(r,T[k],b.getOperand(k).type.shape))}let g=e.arguments.map(_=>m.get(_.id)),y=new Map;for(let _ of n.keys())y.set(_,m.get(_));return{forwardYields:d,gradArgs:g,gradFree:y}}function Fz(r,e,t,n,s){let o=r.regions[0].blocks[0],i=r.regions[1].blocks[0],a=n(r.getOperand(0)),u=[];for(let y=0;y<r.numResults;y++)u.push(e.get(r.getResult(y).id));let p=Nf(o),c=Nf(i),h=new Map(p.map(y=>[y.id,n(y)])),f=new Map(c.map(y=>[y.id,n(y)])),{gradFree:d}=g_(t,o,[],h,u,!1,new Map,p),{gradFree:m}=g_(t,i,[],f,u,!1,new Map,c),g=new Map;for(let y of p)g.set(y.id,y);for(let y of c)g.set(y.id,y);for(let[y,_]of g){if(!s.has(y))continue;let b=d.get(y),w=m.get(y);if(!b&&!w)continue;let x=m_(t,_),S=t.broadcast(a,_.type.shape,[]).getResult(0);e.accumulate(y,t.select(S,b??x,w??x).getResult(0))}}function zz(r,e){if(!r||e<=1)return null;if(r==="sqrt"||r===!0)return Math.max(1,Math.ceil(Math.sqrt(e)));if(typeof r=="number"&&r>=1){let t=Math.floor(r);return t>=e?null:t}return null}function Vz(r,e,t,n,s,o=null){let i=r.regions[0].blocks[0],a=r.getAttr("num_carry"),u=r.getAttr("num_xs"),p=r.numResults-a,c=[];for(let v=0;v<u;v++)c.push(r.getOperand(v));let h=[];for(let v=0;v<a;v++)h.push(r.getOperand(u+v));let f=c[0].type.shape[0],d=Nf(i),m=c.map(n),g=h.map(n),y=new Map(d.map(v=>[v.id,n(v)])),_=new Map,b=l(v=>m.map(O=>wM(t,O,v)),"sliceX"),w=l((v,O)=>g_(t,i,[...v,...O],y,null,!0,_,d).forwardYields.slice(0,a),"stepForward"),x=[];for(let v=0;v<p;v++)x.push(e.get(r.getResult(a+v).id));let S=[];for(let v=0;v<a;v++){let O=e.get(r.getResult(v).id);S.push(O??m_(t,g[v]))}let T=new Map,k=m.map(()=>new Array(f)),A=l((v,O,R)=>{let B=[...O,...R],E=x.map(j=>j===null?null:wM(t,j,v)),z=[...S,...E],{gradArgs:D,gradFree:J}=g_(t,i,B,y,z,!1,_,d);for(let j=0;j<u;j++)k[j][v]=D[j]??m_(t,O[j]);S=[];for(let j=0;j<a;j++)S.push(D[u+j]??m_(t,R[j]));for(let[j,U]of J){if(!U)continue;let P=T.get(j);T.set(j,P?t.add(P,U).getResult(0):U)}},"backwardStep"),N=zz(o,f);if(N){let v=Math.ceil(f/N),O=new Array(v),R=g;O[0]=R;for(let B=0;B<f;B++){R=w(b(B),R);let E=(B+1)/N;Number.isInteger(E)&&E<v&&(O[E]=R)}for(let B=v-1;B>=0;B--){let E=B*N,z=Math.min(E+N,f),D=[],J=[O[B]],j=O[B];for(let U=E;U<z;U++){let P=b(U);D.push(P),j=w(P,j),J.push(j)}for(let U=z-1;U>=E;U--)A(U,D[U-E],J[U-E])}}else{let v=g,O=[v],R=[];for(let B=0;B<f;B++){let E=b(B);R.push(E),v=w(E,v),O.push(v)}for(let B=f-1;B>=0;B--)A(B,R[B],O[B])}for(let v=0;v<u;v++)s.has(c[v].id)&&e.accumulate(c[v].id,$z(t,k[v],m[v].type.shape));for(let v=0;v<a;v++)s.has(h[v].id)&&e.accumulate(h[v].id,S[v]);for(let[v,O]of T)O&&s.has(v)&&e.accumulate(v,O)}var Pz,TM=M(()=>{"use strict";vf();ls();Af();wx();Pz={has:l(()=>!0,"has")};_x("scan",(r,e)=>Vz(r,e.accumulator,e.builder,e.materialize,e.needsGrad,e.scanCheckpoint));_x("if",(r,e)=>Fz(r,e.accumulator,e.builder,e.materialize,e.needsGrad));l(Nf,"regionFreeVars");l(m_,"zeroLike");l(wM,"sliceStep");l($z,"stackSteps");l(g_,"diffBodyStep");l(Fz,"buildCondBackward");l(zz,"resolveSegmentLength");l(Vz,"buildScanBackward")});function Of(r,{accumulator:e,builder:t,needsGrad:n,resolveValue:s,handleRegionOp:o=null}){for(let i=r.length-1;i>=0;i--){let a=r[i];if(a.opName==="return"||a.opName==="constant"||!a.results.some(g=>n.has(g.id)))continue;let p=[];for(let g=0;g<a.numResults;g++)p.push(e.get(a.getResult(g).id));if(p.every(g=>g===null)||o&&o(a))continue;let c=f_(a.opName);if(!c)continue;let h=new Array(a.numOperands);for(let g=0;g<a.numOperands;g++)h[g]=s(a.getOperand(g));let f=new Array(a.numResults);for(let g=0;g<a.numResults;g++)f[g]=s(a.getResult(g));let d=l((g,y)=>t.broadcast(t.scalarConstant(g,y.dtype).getResult(0),y.shape,[]).getResult(0),"full"),m=c({builder:t,op:a,operands:h,results:f,gradOutputs:p,attrs:a.attributes,full:d});if(m)for(let g=0;g<a.numOperands;g++){if(g>=m.length||!m[g])continue;let y=a.getOperand(g);n.has(y.id)&&e.accumulate(y.id,Tx(t,m[g],y.type.shape))}}}function xM(r){let e=[];for(let t of r.regions)t.blocks[0]&&e.push(...Nf(t.blocks[0]));return e}function Tx(r,e,t){let n=e.type.shape;if(n.length===t.length&&n.every((u,p)=>u===t[p]))return e;let s=n.length-t.length,o=[];for(let u=0;u<s;u++)o.push(u);for(let u=0;u<t.length;u++)t[u]===1&&n[s+u]!==1&&o.push(s+u);let i=e;if(o.length>0){let u=r.scalarConstant(0,e.type.dtype).getResult(0);i=r.reduce(e,u,o,"sum").getResult(0)}let a=i.type.shape;return a.length===t.length&&a.every((u,p)=>u===t[p])||(i=r.reshape(i,t).getResult(0)),i}var Gz,Bf,Af=M(()=>{"use strict";xs();es();Es();vf();ls();TM();wx();Gz=new Set(["neg","abs","sign","floor","ceil"]);l(Of,"backpropOps");l(xM,"regionControlFlowFreeVars");l(Tx,"reduceGradToOperandShape");Bf=class{static{l(this,"BackwardGraphBuilder")}constructor(e={}){this._rematPolicy=e.rematPolicy||null,this._checkpointPolicy=e.checkpointPolicy||null,this._scanCheckpoint=e.scanCheckpoint||null}build(e){if(this._checkpointPolicy)return this._buildCheckpointed(e);let n=at.compute(e).topologicalOrder,s=e.getReturnOp();if(!s)throw new Error("Forward function has no return op");let o=s.operands,i=e.args,a=this._computeGradReachability(e,n),{savedValues:u,savedValueIndices:p}=this._identifySavedValues(n,a,i),c=o.map(T=>T.type),h=u.map(T=>T.type),f=[...c,...h],d=[];for(let T=0;T<i.length;T++)a.has(i[T].id)&&d.push(i[T].type);let m=new un(`backward_${e.name}`,f,d),g=new An(m),y=m.args,_=y.slice(0,c.length),b=y.slice(c.length),w=new Map;for(let T=0;T<u.length;T++)w.set(u[T].id,b[T]);for(let T=0;T<i.length;T++)p.has(i[T].id)&&w.set(i[T].id,b[p.get(i[T].id)]);let x=new co(g);for(let T=0;T<o.length;T++){let k=o[T];x.accumulate(k.id,_[T])}Of(n,{accumulator:x,builder:g,needsGrad:a,resolveValue:l(T=>this._materialize(T,w,g),"resolveValue"),handleRegionOp:l(T=>{let k=h_(T.opName);return k?(k(T,{accumulator:x,builder:g,materialize:l(A=>this._materialize(A,w,g),"materialize"),needsGrad:a,scanCheckpoint:this._scanCheckpoint}),!0):!1},"handleRegionOp")});let S=[];for(let T=0;T<i.length;T++)a.has(i[T].id)&&S.push(kf(g,i[T],x));return g.returnOp(S),{backwardFunc:m,savedValues:u,gradInputIndices:this._getGradInputIndices(i,a)}}_materialize(e,t,n){if(t.has(e.id))return t.get(e.id);if(!e.definingOp)return e;let s=new Set([e.id]),o=[{val:e,i:0}];for(;o.length>0;){let i=o[o.length-1],a=i.val,u=a.definingOp;if(t.has(a.id)||!u){s.delete(a.id),o.pop();continue}if(i.i<u.numOperands){let f=u.getOperand(i.i);i.i++,f.definingOp&&!t.has(f.id)&&!s.has(f.id)&&(s.add(f.id),o.push({val:f,i:0}));continue}let p=new Array(u.numOperands);for(let f=0;f<u.numOperands;f++){let d=u.getOperand(f);p[f]=t.has(d.id)?t.get(d.id):d}let c=u.results.map(f=>f.type),h=n._buildOp(u.opName,p,c,new Map(u.attributes),null);for(let f=0;f<u.numResults;f++)t.set(u.getResult(f).id,h.getResult(f));s.delete(a.id),o.pop()}return t.has(e.id)?t.get(e.id):e}_computeGradReachability(e,t){let n=new Set,s=e.getReturnOp();for(let o of s.operands)n.add(o.id);for(let o=t.length-1;o>=0;o--){let i=t[o];if(!(i.opName==="return"||!i.results.some(u=>n.has(u.id)))){if(Po.has(i.opName)){for(let u=0;u<i.numOperands;u++)n.add(i.getOperand(u).id);for(let u of xM(i))n.add(u.id);continue}if(Ia(i.opName)&&!d_(i.opName))for(let u=0;u<i.numOperands;u++)n.add(i.getOperand(u).id)}}return n}_identifySavedValues(e,t,n){let s=[],o=new Map,i=new Set(n.map(h=>h.id));for(let h of e)if(!(h.opName==="return"||h.opName==="constant"||!Ia(h.opName)||!h.results.some(m=>t.has(m.id)))){for(let m=0;m<h.numOperands;m++){let g=h.getOperand(m);i.has(g.id)&&!o.has(g.id)&&(o.set(g.id,s.length),s.push(g))}for(let m=0;m<h.numResults;m++){let g=h.getResult(m);t.has(g.id)&&!o.has(g.id)&&this._shouldSaveResult(h)&&(o.set(g.id,s.length),s.push(g))}}let a=new Set(o.keys()),u=new Map(n.map(h=>[h.id,h])),p=new Set,c=l(h=>{if(a.has(h.id)||p.has(h.id))return;p.add(h.id);let f=h.definingOp;if(!f){i.has(h.id)&&!o.has(h.id)&&(o.set(h.id,s.length),s.push(u.get(h.id)));return}for(let d=0;d<f.numOperands;d++)c(f.getOperand(d))},"collect");for(let h of e)if(!(h.opName==="return"||h.opName==="constant")&&h.results.some(f=>t.has(f.id))){if(Po.has(h.opName)){for(let f of h.operands)c(f);for(let f of xM(h))c(f);continue}if(Ia(h.opName)){for(let f=0;f<h.numOperands;f++)c(h.getOperand(f));for(let f=0;f<h.numResults;f++)c(h.getResult(f))}}return{savedValues:s,savedValueIndices:o}}_shouldSaveResult(e){return this._rematPolicy?!this._rematPolicy.shouldRematerialize(e):!Gz.has(e.opName)}_getGradInputIndices(e,t){let n=[];for(let s=0;s<e.length;s++)t.has(e[s].id)&&n.push(s);return n}_buildCheckpointed(e){let n=at.compute(e).topologicalOrder,s=e.getReturnOp();if(!s)throw new Error("Forward function has no return op");let o=s.operands,i=e.args,a=this._computeGradReachability(e,n),u=this._checkpointPolicy.segment(n,e);for(let O of u)for(let R of O.ops)if(Po.has(R.opName))throw new Error(`Checkpointed backward does not support region control-flow op '${R.opName}'; build the backward without a checkpointPolicy, which differentiates scan/if via buildScanBackward/buildCondBackward.`);let p=new Set,c=[],h=new Map,f=new Map;for(let O of n)for(let R=0;R<O.numResults;R++){let B=O.getResult(R);f.set(B.id,B)}let d=new Set(i.map(O=>O.id));for(let O of i)a.has(O.id)&&!p.has(O.id)&&(p.add(O.id),h.set(O.id,c.length),c.push(O));for(let O of u){for(let R of O.boundaryInputs)if(!d.has(R)&&!p.has(R)){p.add(R);let B=f.get(R)||null;B&&(h.set(R,c.length),c.push(B))}for(let R of O.boundaryOutputs)if(!p.has(R)){p.add(R);let B=f.get(R)||null;B&&(h.set(R,c.length),c.push(B))}}for(let O of o)p.has(O.id)||(p.add(O.id),h.set(O.id,c.length),c.push(O));let m=o.map(O=>O.type),g=c.map(O=>O.type),y=[...m,...g],_=[];for(let O=0;O<i.length;O++)a.has(i[O].id)&&_.push(i[O].type);let b=new un(`backward_${e.name}`,y,_),w=new An(b),x=b.args,S=x.slice(0,m.length),T=x.slice(m.length),k=new Map;for(let O=0;O<c.length;O++)k.set(c[O].id,T[O]);let A=new co(w);for(let O=0;O<o.length;O++)A.accumulate(o[O].id,S[O]);let N=new Map;for(let O of n)if(O.opName==="constant"){let R=O.getResult(0).type,B=w._buildOp("constant",[],[R],new Map(O.attributes),null);N.set(O.getResult(0).id,B.getResult(0))}for(let O=u.length-1;O>=0;O--){let R=u[O],B=new Map;for(let E of R.ops){let z=new Array(E.numOperands);for(let j=0;j<E.numOperands;j++){let U=E.getOperand(j),P=B.get(U.id)||k.get(U.id)||N.get(U.id);z[j]=P||U}let D=E.results.map(j=>j.type),J=w._buildOp(E.opName,z,D,new Map(E.attributes),null);for(let j=0;j<E.numResults;j++)B.set(E.getResult(j).id,J.getResult(j))}Of(R.ops,{accumulator:A,builder:w,needsGrad:a,resolveValue:l(E=>B.get(E.id)||k.get(E.id)||N.get(E.id)||E,"resolveValue")})}let v=[];for(let O=0;O<i.length;O++)a.has(i[O].id)&&v.push(kf(w,i[O],A));return w.returnOp(v),{backwardFunc:b,savedValues:c,gradInputIndices:this._getGradInputIndices(i,a)}}}});var Ea,y_=M(()=>{"use strict";Ea=class{static{l(this,"RematPolicy")}constructor(e={}){this._maxRematDepth=e.maxRematDepth||1,this._sizeThreshold=e.sizeThreshold||1024*1024,this._alwaysRemat=new Set(e.alwaysRemat||["neg","abs","sign","floor","ceil","exp","log","sqrt","rsqrt","sin","cos","tanh"]),this._neverRemat=new Set(e.neverRemat||["matmul","dot","conv","reduce","custom_call","pool2d"])}shouldRematerialize(e){if(this._alwaysRemat.has(e.opName))return!0;if(this._neverRemat.has(e.opName))return!1;let t=e.numResults>0?e.getResult(0).type:null;return!t||!t.shape||t.numel()>this._sizeThreshold?!1:this._isElementwise(e)}_isElementwise(e){if(e.numOperands===0||e.numResults===0)return!1;let t=e.getResult(0).type.shape;for(let n=0;n<e.numOperands;n++){let s=e.getOperand(n).type;if(!s||!s.shape||s.shape.length!==t.length)return!1;for(let o=0;o<t.length;o++)if(s.shape[o]!==t[o])return!1}return!0}}});var Rf,xx=M(()=>{"use strict";xs();es();Es();vf();ls();y_();Af();Rf=class{static{l(this,"JointGraphBuilder")}constructor(e={}){this._rematPolicy=e.rematPolicy||new Ea(e.remat||{}),this._checkpointPolicy=e.checkpointPolicy||null}build(e){if(this._checkpointPolicy)return this._buildCheckpointed(e);let t=this._buildScaffold(e);return Of(t.topoOrder,{accumulator:t.accumulator,builder:t.builder,needsGrad:t.needsGrad,resolveValue:l(n=>t.valueMap.get(n.id)||n,"resolveValue")}),this._finish(t)}_buildCheckpointed(e){let t=this._buildScaffold(e),n=this._checkpointPolicy.segment(t.topoOrder,e);for(let s=n.length-1;s>=0;s--){let o=n[s],i=new Map;for(let a of o.ops){let u=new Array(a.numOperands);for(let h=0;h<a.numOperands;h++){let f=a.getOperand(h);u[h]=i.get(f.id)||t.valueMap.get(f.id)||f}let p=a.results.map(h=>h.type),c=t.builder._buildOp(a.opName,u,p,new Map(a.attributes),null);for(let h=0;h<a.numResults;h++)i.set(a.getResult(h).id,c.getResult(h))}Of(o.ops,{accumulator:t.accumulator,builder:t.builder,needsGrad:t.needsGrad,resolveValue:l(a=>i.get(a.id)||t.valueMap.get(a.id)||a,"resolveValue")})}return this._finish(t)}_buildScaffold(e){let n=at.compute(e).topologicalOrder;this._assertNoRegionControlFlow(n);let s=e.getReturnOp();if(!s)throw new Error("Forward function has no return op");let o=s.operands,i=e.args,a=o.map(w=>w.type),u=[...e.inputTypes,...a],p=[...e.outputTypes,...e.inputTypes],c=new un(`joint_${e.name}`,u,p),h=new An(c),f=c.args,d=f.slice(0,e.inputTypes.length),m=f.slice(e.inputTypes.length),g=new Map;for(let w=0;w<i.length;w++)g.set(i[w].id,d[w]);for(let w of n){if(w.opName==="return")continue;let x=new Array(w.numOperands);for(let k=0;k<w.numOperands;k++){let A=w.getOperand(k);x[k]=g.get(A.id)||A}let S=w.results.map(k=>k.type),T=h._buildOp(w.opName,x,S,new Map(w.attributes),null);for(let k=0;k<w.numResults;k++)g.set(w.getResult(k).id,T.getResult(k))}let y=o.map(w=>g.get(w.id)),_=this._computeGradReachability(e,n),b=new co(h);for(let w=0;w<o.length;w++)b.accumulate(o[w].id,m[w]);return{topoOrder:n,forwardInputs:i,forwardOutputs:o,fwdOutputValues:y,valueMap:g,builder:h,needsGrad:_,accumulator:b,jointFunc:c}}_finish(e){let t=[];for(let n=0;n<e.forwardInputs.length;n++)t.push(kf(e.builder,e.forwardInputs[n],e.accumulator));return e.builder.returnOp([...e.fwdOutputValues,...t]),{jointFunc:e.jointFunc,numForwardOutputs:e.forwardOutputs.length,numGradInputs:t.length}}_computeGradReachability(e,t){let n=new Set,s=e.getReturnOp();for(let o of s.operands)n.add(o.id);for(let o=t.length-1;o>=0;o--){let i=t[o];if(!(i.opName==="return"||!i.results.some(u=>n.has(u.id)))&&Ia(i.opName)&&!d_(i.opName))for(let u=0;u<i.numOperands;u++)n.add(i.getOperand(u).id)}return n}_assertNoRegionControlFlow(e){for(let t of e)if(Po.has(t.opName))throw new Error(`JointGraphBuilder does not support region control-flow op '${t.opName}'; use BackwardGraphBuilder (separate mode) without a checkpointPolicy, which differentiates scan/if.`)}}});var SM=M(()=>{"use strict"});function kM(r,e){let t=r.gradOutputs[0],[n,s]=r.operands,o=r.full(0,n.type),i=r.builder.compare(n,s,e).getResult(0),a=r.builder.select(i,t,o).getResult(0),u=r.builder.select(i,o,t).getResult(0);return[a,u]}function vM(r){let e=r.gradOutputs[0],[t]=r.operands,n=r.full(0,e.type),s=r.builder.where(t,e,n).getResult(0),o=r.builder.where(t,n,e).getResult(0);return[null,s,o]}var NM=M(()=>{"use strict";ls();ce("add",r=>{let e=r.gradOutputs[0];return[e,e]});ce("sub",r=>{let e=r.gradOutputs[0],t=r.builder.neg(e).getResult(0);return[e,t]});l(kM,"_minMaxVJP");ce("maximum",r=>kM(r,"ge"));ce("minimum",r=>kM(r,"le"));ce("clamp",r=>{let e=r.gradOutputs[0],[t,n,s]=r.operands,o=r.full(0,n.type),i=r.builder.compare(n,t,"ge").getResult(0),a=r.builder.where(i,e,o).getResult(0),u=r.builder.compare(n,s,"le").getResult(0);return[null,r.builder.where(u,a,o).getResult(0),null]});l(vM,"_whereVJP");ce("where",vM);ce("select",vM);ce("mul",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.builder.mul(e,n).getResult(0),o=r.builder.mul(e,t).getResult(0);return[s,o]});ce("div",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.builder.div(e,n).getResult(0),o=r.builder.mul(n,n).getResult(0),i=r.builder.neg(t).getResult(0),a=r.builder.mul(e,i).getResult(0),u=r.builder.div(a,o).getResult(0);return[s,u]});ce("neg",r=>{let e=r.gradOutputs[0];return[r.builder.neg(e).getResult(0)]});ce("pow",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.full(1,t.type),o=r.builder.sub(n,s).getResult(0),i=r.builder.pow(t,o).getResult(0),a=r.builder.mul(n,i).getResult(0),u=r.builder.mul(e,a).getResult(0),p=r.builder.pow(t,n).getResult(0),c=r.builder.log(t).getResult(0),h=r.builder.mul(p,c).getResult(0),f=r.builder.mul(e,h).getResult(0);return[u,f]})});function AM(r,e){let t=r.full(2/Math.sqrt(Math.PI),e.type),n=r.builder.mul(e,e).getResult(0),s=r.builder.neg(n).getResult(0),o=r.builder.exp(s).getResult(0);return r.builder.mul(t,o).getResult(0)}function OM(r,e){let t=r.builder,n=r.full(1,e.type),s=t.add(e,r.full(Yi,e.type)).getResult(0),o=t.div(n,s).getResult(0),i=t.log(s).getResult(0),a=t.sub(i,t.mul(r.full(.5,e.type),o).getResult(0)).getResult(0),u=t.mul(o,o).getResult(0),p=u;for(let c of vc)a=t.add(a,t.mul(r.full(c,e.type),p).getResult(0)).getResult(0),p=t.mul(p,u).getResult(0);for(let c=0;c<Yi;c++){let h=t.add(e,r.full(c,e.type)).getResult(0);a=t.sub(a,t.div(n,h).getResult(0)).getResult(0)}return a}var BM=M(()=>{"use strict";ls();Ji();ce("exp",r=>{let e=r.gradOutputs[0],t=r.results[0];return[r.builder.mul(e,t).getResult(0)]});ce("log",r=>{let e=r.gradOutputs[0],[t]=r.operands;return[r.builder.div(e,t).getResult(0)]});ce("sqrt",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.full(2,t.type),s=r.builder.mul(n,t).getResult(0);return[r.builder.div(e,s).getResult(0)]});ce("tanh",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.builder.mul(t,t).getResult(0),s=r.builder.sub(r.full(1,t.type),n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("sigmoid",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.builder.sub(r.full(1,t.type),t).getResult(0),s=r.builder.mul(t,n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("relu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(0,t.type),s=r.builder.compare(t,n,"gt").getResult(0);return[r.builder.select(s,e,n).getResult(0)]});ce("gelu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(1.702,t.type),s=r.full(1,t.type),o=r.builder.mul(n,t).getResult(0),i=r.builder.sigmoid(o).getResult(0),a=r.builder.sub(s,i).getResult(0),u=r.builder.mul(o,a).getResult(0),p=r.builder.add(s,u).getResult(0),c=r.builder.mul(i,p).getResult(0);return[r.builder.mul(e,c).getResult(0)]});ce("silu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.sigmoid(t).getResult(0),s=r.builder.sub(r.full(1,t.type),n).getResult(0),o=r.builder.mul(t,s).getResult(0),i=r.builder.add(n,r.builder.mul(n,o).getResult(0)).getResult(0);return[r.builder.mul(e,i).getResult(0)]});ce("sin",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.cos(t).getResult(0);return[r.builder.mul(e,n).getResult(0)]});ce("cos",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.sin(t).getResult(0),s=r.builder.neg(n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("abs",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.sign(t).getResult(0);return[r.builder.mul(e,n).getResult(0)]});l(AM,"_erfDerivIR");l(OM,"_digammaIR");ce("erf",r=>{let e=r.gradOutputs[0],[t]=r.operands;return[r.builder.mul(e,AM(r,t)).getResult(0)]});ce("erfc",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.neg(AM(r,t)).getResult(0);return[r.builder.mul(e,n).getResult(0)]});ce("lgamma",r=>{let e=r.gradOutputs[0],[t]=r.operands;return[r.builder.mul(e,OM(r,t)).getResult(0)]});ce("gamma",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.results[0],s=r.builder.mul(n,OM(r,t)).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("log2",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(Math.LN2,t.type),s=r.builder.mul(t,n).getResult(0);return[r.builder.div(e,s).getResult(0)]});ce("log10",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(Math.LN10,t.type),s=r.builder.mul(t,n).getResult(0);return[r.builder.div(e,s).getResult(0)]});ce("exp2",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.full(Math.LN2,t.type),s=r.builder.mul(t,n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("square",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(2,t.type),s=r.builder.mul(n,t).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("reciprocal",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.builder.mul(t,t).getResult(0),s=r.builder.neg(n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("rsqrt",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.builder.mul(r.builder.mul(t,t).getResult(0),t).getResult(0),s=r.full(-.5,t.type),o=r.builder.mul(s,n).getResult(0);return[r.builder.mul(e,o).getResult(0)]})});var RM=M(()=>{"use strict";ls();ce("dot",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.op.getAttr("lhs_contracting"),o=r.op.getAttr("rhs_contracting"),i=r.op.getAttr("lhs_batch")||[],a=r.op.getAttr("rhs_batch")||[],u=t.type.rank,p=n.type.rank,c=e.type.rank,h=[];for(let _=0;_<u;_++)!s.includes(_)&&!i.includes(_)&&h.push(_);let f=[];for(let _=0;_<p;_++)!o.includes(_)&&!a.includes(_)&&f.push(_);let d=[];for(let _=i.length;_<i.length+f.length;_++)d.push(_+h.length);let m=[];for(let _=i.length;_<i.length+h.length;_++)m.push(_);let g=r.builder.dot(e,n,d,f,Array.from({length:i.length},(_,b)=>b),a).getResult(0),y=r.builder.dot(t,e,h,m,i,Array.from({length:i.length},(_,b)=>b)).getResult(0);return[g,y]});ce("matmul",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=l(p=>{let c=Array.from({length:p},(h,f)=>f);return c[p-2]=p-1,c[p-1]=p-2,c},"swapLastTwo"),o=r.builder.transpose(n,s(n.type.rank)).getResult(0),i=r.builder.matmul(e,o).getResult(0),a=r.builder.transpose(t,s(t.type.rank)).getResult(0),u=r.builder.matmul(a,e).getResult(0);return[i,u]});ce("conv",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.builder,o=r.op.getAttr("strides"),i=r.op.getAttr("padding"),a=r.op.getAttr("dilation")||o.map(()=>1),u=r.op.getAttr("groups")||1,p=r.op.getAttr("input_layout"),c=r.op.getAttr("kernel_layout");if(!(o.length===2&&o.every(A=>A===1)&&a.every(A=>A===1)&&u===1&&p==="NCHW"&&c==="OIHW"))throw new Error("conv VJP supports only 2D stride-1 dilation-1 groups-1 NCHW/OIHW conv");let f=n.type.shape,d=f[2],m=f[3],[g,y]=i,_=s.reverse(s.transpose(n,[1,0,2,3]).getResult(0),[2,3]).getResult(0),b=[[d-1-g[0],d-1-g[1]],[m-1-y[0],m-1-y[1]]],w=s.conv(e,_,[1,1],b).getResult(0),x=s.transpose(t,[1,0,2,3]).getResult(0),S=s.transpose(e,[1,0,2,3]).getResult(0),T=s.conv(x,S,[1,1],[g,y]).getResult(0),k=s.transpose(T,[1,0,2,3]).getResult(0);return[w,k]})});var MM=M(()=>{"use strict";ls();ce("reduce",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("reduce_type"),s=r.op.getAttr("dimensions"),o=t.type.shape,i=t.type.dtype,a=new Set(s),u=o.map((h,f)=>a.has(f)?1:h),p=o.map((h,f)=>f),c=r.builder.reshape(e,u).getResult(0);if(n==="sum")return[r.builder.broadcast(c,o,p).getResult(0),null];if(n==="mean"){let h=1;for(let y=0;y<o.length;y++)a.has(y)&&(h*=o[y]);let f=r.builder.broadcast(c,o,p).getResult(0),d=r.builder.scalarConstant(h,i).getResult(0),m=r.builder.broadcast(d,o,[]).getResult(0);return[r.builder.div(f,m).getResult(0),null]}if(n==="max"||n==="min"){let h=r.builder.reshape(r.results[0],u).getResult(0),f=r.builder.broadcast(h,o,p).getResult(0),d=r.builder.compare(t,f,"eq").getResult(0),m=r.builder.convert(d,i).getResult(0),g=r.builder.broadcast(c,o,p).getResult(0);return[r.builder.mul(g,m).getResult(0),null]}throw n==="prod"?new Error("reduce VJP for reduce_type='prod' is not implemented; provide a gradient rule or avoid differentiating reduce_prod"):new Error(`reduce VJP: unsupported reduce_type '${n}' on the gradient path (would silently drop the gradient)`)})});var IM=M(()=>{"use strict";ls();ce("reshape",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=t.type.shape;return[r.builder.reshape(e,n).getResult(0)]});ce("reverse",r=>[r.builder.reverse(r.gradOutputs[0],r.op.getAttr("dimensions")).getResult(0)]);ce("transpose",r=>{let e=r.gradOutputs[0],t=r.op.getAttr("permutation"),n=new Array(t.length);for(let s=0;s<t.length;s++)n[t[s]]=s;return[r.builder.transpose(e,n).getResult(0)]});ce("broadcast_in_dim",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("broadcast_dimensions"),s=t.type.shape,o=r.results[0].type.shape,i=t.type.dtype,a=[];for(let h=0;h<o.length;h++)if(!n.includes(h))a.push(h);else{let f=n.indexOf(h);s[f]===1&&o[h]!==1&&a.push(h)}if(a.length===0)return[e];let u=r.builder.scalarConstant(0,i).getResult(0),p=r.builder.reduce(e,u,a,"sum").getResult(0),c=p.type.shape;return c.length!==s.length||!c.every((h,f)=>h===s[f])?[r.builder.reshape(p,s).getResult(0)]:[p]});ce("slice",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("starts"),s=t.type.shape,o=e.type.shape,i=r.op.getAttr("strides")||s.map(()=>1),a=t.type.dtype,u=[...n],p=new Array(s.length),c=new Array(s.length);for(let f=0;f<s.length;f++)c[f]=i[f]-1,p[f]=s[f]-n[f]-(o[f]-1)*i[f]-1;let h=r.builder.scalarConstant(0,a).getResult(0);return[r.builder.pad(e,h,u,p,c).getResult(0)]});ce("concat",r=>{let e=r.gradOutputs[0],t=r.op.getAttr("dimension"),n=r.operands,s=[],o=0;for(let i=0;i<n.length;i++){let a=n[i].type.shape,u=new Array(a.length).fill(0),p=[...e.type.shape];u[t]=o,p[t]=o+a[t],s.push(r.builder.slice(e,u,p).getResult(0)),o+=a[t]}return s});ce("gather",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.builder.scalarConstant(0,t.type.dtype).getResult(0),o=r.builder.broadcast(s,t.type.shape,[]).getResult(0);return[r.builder.scatterAdd(o,n,e,{updateWindowDims:r.op.getAttr("offset_dims"),insertedWindowDims:r.op.getAttr("collapsed_slice_dims"),scatterDimsToOperandDims:r.op.getAttr("start_index_map"),indexVectorDim:r.op.getAttr("index_vector_dim")}).getResult(0),null]});ce("scatter",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.op.getAttr("inserted_window_dims"),o=t.type.shape.map((a,u)=>s.includes(u)?1:a),i=r.builder.gather(e,n,{offsetDims:r.op.getAttr("update_window_dims"),collapsedSliceDims:s,startIndexMap:r.op.getAttr("scatter_dims_to_operand_dims"),indexVectorDim:r.op.getAttr("index_vector_dim"),sliceSizes:o}).getResult(0);return[e,null,i]});ce("pad",r=>{let e=r.gradOutputs[0],t=r.op.getAttr("low"),n=r.op.getAttr("high"),[s]=r.operands,o=s.type.shape,i=[...t],a=new Array(o.length);for(let u=0;u<o.length;u++)a[u]=t[u]+o[u];return[r.builder.slice(e,i,a).getResult(0),null]})});var EM=M(()=>{"use strict";ls();es();ce("softmax",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.op.getAttr("axis"),s=t.type.dtype,o=t.type.shape,i=r.builder.mul(e,t).getResult(0),a=r.builder.scalarConstant(0,s).getResult(0),u=r.builder.reduce(i,a,[n],"sum").getResult(0),p=To(o.length,n),c=r.builder.broadcast(u,o,p).getResult(0),h=r.builder.sub(e,c).getResult(0);return[r.builder.mul(t,h).getResult(0)]});ce("log_softmax",r=>{let e=r.gradOutputs[0],t=r.results[0],[n]=r.operands,s=r.op.getAttr("axis"),o=t.type.dtype,i=t.type.shape,a=r.builder.exp(t).getResult(0),u=r.builder.scalarConstant(0,o).getResult(0),p=r.builder.reduce(e,u,[s],"sum").getResult(0),c=To(i.length,s),h=r.builder.broadcast(p,i,c).getResult(0),f=r.builder.mul(a,h).getResult(0);return[r.builder.sub(e,f).getResult(0)]});ce("layer_norm",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.op.getAttr("axis"),o=r.op.getAttr("epsilon"),i=t.type.dtype,a=t.type.shape,u=[s<0?a.length+s:s],p=a[u[0]],c=To(a.length,u[0]),h=r.builder.scalarConstant(0,i).getResult(0),f=r.builder.reduce(t,h,u,"mean").getResult(0),d=r.builder.broadcast(f,a,c).getResult(0),m=r.builder.sub(t,d).getResult(0),g=r.builder.mul(m,m).getResult(0),y=r.builder.reduce(g,h,u,"mean").getResult(0),_=r.full(o,y.type),b=r.builder.add(y,_).getResult(0),w=r.builder.rsqrt(b).getResult(0),x=r.builder.broadcast(w,a,c).getResult(0),S=[];for(let he=0;he<a.length;he++)u.includes(he)&&S.push(he);let T=r.builder.broadcast(n,a,S).getResult(0),k=r.builder.mul(e,T).getResult(0),A=r.full(p,t.type),N=r.builder.mul(A,k).getResult(0),v=r.builder.reduce(k,h,u,"sum").getResult(0),O=r.builder.broadcast(v,a,c).getResult(0),R=r.builder.mul(m,x).getResult(0),B=r.builder.mul(k,R).getResult(0),E=r.builder.reduce(B,h,u,"sum").getResult(0),z=r.builder.broadcast(E,a,c).getResult(0),D=r.builder.mul(R,z).getResult(0),J=r.builder.sub(N,r.builder.add(O,D).getResult(0)).getResult(0),j=r.builder.div(x,A).getResult(0),U=r.builder.mul(j,J).getResult(0),P=r.builder.reduce(r.builder.mul(e,R).getResult(0),h,c,"sum").getResult(0),V=r.builder.reduce(e,h,c,"sum").getResult(0);return[U,P,V]});ce("scaled_dot_product_attention",r=>{let e=r.gradOutputs[0],[t,n,s]=r.operands,o=r.op.getAttr("scale");if(r.op.getAttr("causal"))throw new Error("causal scaled_dot_product_attention VJP not supported");let i=r.builder,a=t.type.dtype,u=t.type.rank,p=[];for(let N=0;N<u;N++)p.push(N);p[u-2]=u-1,p[u-1]=u-2;let c=l(N=>i.transpose(N,p).getResult(0),"lastT"),h=i.matmul(t,c(n)).getResult(0),f=i.mul(h,r.full(o,h.type)).getResult(0),d=i.softmax(f,u-1).getResult(0),m=i.matmul(c(d),e).getResult(0),g=i.matmul(e,c(s)).getResult(0),y=i.scalarConstant(0,a).getResult(0),_=i.mul(g,d).getResult(0),b=i.reduce(_,y,[u-1],"sum").getResult(0),w=[];for(let N=0;N<u-1;N++)w.push(N);let x=i.broadcast(b,d.type.shape,w).getResult(0),S=i.mul(d,i.sub(g,x).getResult(0)).getResult(0),T=i.mul(S,r.full(o,S.type)).getResult(0),k=i.matmul(T,n).getResult(0),A=i.matmul(c(T),t).getResult(0);return[k,A,m]});ce("pool2d",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder,s=r.op.getAttr("pool_type"),o=r.op.getAttr("kernel_size"),i=r.op.getAttr("strides"),a=r.op.getAttr("padding"),u=r.op.getAttr("layout")||"NCHW",p=a.every(k=>k[0]===0&&k[1]===0),c=i[0]===o[0]&&i[1]===o[1];if(u!=="NCHW"||!p||!c||s!=="avg"&&s!=="max")throw new Error("pool2d VJP supports only non-overlapping (stride=kernel) avg/max pooling without padding, NCHW");let[h,f,d,m]=e.type.shape,[g,y]=o,_=t.type.shape,b=l(k=>n.reshape(n.broadcast(k,[h,f,d,g,m,y],[0,1,2,4]).getResult(0),_).getResult(0),"upsample"),w=b(e);if(s==="avg"){let k=r.full(g*y,t.type);return[n.div(w,k).getResult(0)]}let x=b(r.results[0]),S=n.compare(t,x,"eq").getResult(0),T=r.full(0,t.type);return[n.select(S,w,T).getResult(0)]});ce("batch_norm",r=>{let e=r.gradOutputs[0],[t,n,,s,o]=r.operands,i=r.op.getAttr("axis"),a=r.op.getAttr("epsilon"),u=t.type.dtype,p=t.type.shape,c=r.builder,h=To(p.length,i),f=c.scalarConstant(0,u).getResult(0),d=r.full(a,o.type),m=c.rsqrt(c.add(o,d).getResult(0)).getResult(0),g=c.broadcast(m,p,[i]).getResult(0),y=c.broadcast(s,p,[i]).getResult(0),_=c.broadcast(n,p,[i]).getResult(0),b=c.sub(t,y).getResult(0),w=c.mul(b,g).getResult(0),x=c.mul(e,_).getResult(0),S=c.mul(x,g).getResult(0),T=c.reduce(c.mul(e,w).getResult(0),f,h,"sum").getResult(0),k=c.reduce(e,f,h,"sum").getResult(0),A=c.neg(c.reduce(S,f,h,"sum").getResult(0)).getResult(0),N=c.mul(c.mul(g,g).getResult(0),g).getResult(0),v=r.full(-.5,t.type),O=c.mul(c.mul(c.mul(x,b).getResult(0),N).getResult(0),v).getResult(0),R=c.reduce(O,f,h,"sum").getResult(0);return[S,T,k,A,R]});ce("elu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.results[0],s=r.op.getAttr("alpha")??1,o=r.full(0,t.type),i=r.full(1,t.type),a=r.full(s,t.type),u=r.builder.compare(t,o,"gt").getResult(0),p=r.builder.add(n,a).getResult(0),c=r.builder.select(u,i,p).getResult(0);return[r.builder.mul(e,c).getResult(0)]});ce("leaky_relu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("negative_slope")??.01,s=r.full(0,t.type),o=r.full(1,t.type),i=r.full(n,t.type),a=r.builder.compare(t,s,"gt").getResult(0),u=r.builder.select(a,o,i).getResult(0);return[r.builder.mul(e,u).getResult(0)]});ce("celu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("alpha")??1,s=r.full(0,t.type),o=r.full(1,t.type),i=r.builder.compare(t,s,"gt").getResult(0),a=r.full(n,t.type),u=r.builder.div(t,a).getResult(0),p=r.builder.exp(u).getResult(0),c=r.builder.select(i,o,p).getResult(0);return[r.builder.mul(e,c).getResult(0)]});ce("selu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=1.0507009873554805,s=1.6732632423543772,o=r.full(0,t.type),i=r.full(n,t.type),a=r.builder.compare(t,o,"gt").getResult(0),u=r.full(s,t.type),p=r.builder.exp(t).getResult(0),c=r.builder.mul(u,p).getResult(0),h=r.builder.select(a,i,r.builder.mul(i,c).getResult(0)).getResult(0);return[r.builder.mul(e,h).getResult(0)]});ce("hardswish",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(-3,t.type),s=r.full(3,t.type),o=r.full(0,t.type),i=r.full(1,t.type),a=r.full(2,t.type),u=r.full(6,t.type),p=r.builder.compare(t,n,"le").getResult(0),c=r.builder.compare(t,s,"ge").getResult(0),h=r.builder.add(r.builder.mul(a,t).getResult(0),s).getResult(0),f=r.builder.div(h,u).getResult(0),d=r.builder.select(p,o,r.builder.select(c,i,f).getResult(0)).getResult(0);return[r.builder.mul(e,d).getResult(0)]});ce("hardsigmoid",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(-3,t.type),s=r.full(3,t.type),o=r.full(0,t.type),i=r.full(1/6,t.type),a=r.builder.compare(t,n,"le").getResult(0),u=r.builder.compare(t,s,"ge").getResult(0),p=r.builder.select(a,o,r.builder.select(u,o,i).getResult(0)).getResult(0);return[r.builder.mul(e,p).getResult(0)]});ce("embedding",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=n.type.rank,o=r.full(0,t.type);return[r.builder.scatter(o,n,e,{updateWindowDims:[s],insertedWindowDims:[0],scatterDimsToOperandDims:[0],indexVectorDim:s}).getResult(0),null]})});var LM=M(()=>{"use strict";ls();bx("stop_gradient");ce("stop_gradient",()=>[null]);for(let r of["compare","logical_not","argmax","argmin","iota"])bx(r)});var CM=M(()=>{"use strict";ls();vf();Af();xx();y_();SM();NM();BM();RM();MM();IM();EM();LM()});var DM={};nt(DM,{compileWithBackward:()=>b_});function __(r){return typeof r.then=="function"}function b_(r,e,t={}){let n=t.target??Us(),s=t.mode||"separate",o=t.rematPolicy||new Ea(t.remat||{}),i={target:n,verify:!1,...t,backward:void 0,mode:void 0,rematPolicy:void 0,remat:void 0},a=t.dynamic_shapes||null,u=[],p=null,c=null,h=null;function f(v){let O=T_((...B)=>r.forward(...B),v,{name:r.constructor.name||"compiled",dynamicShapes:a}),R=l(B=>{let E=B.graph.functions().next().value,z=s==="joint"?m(E,B,o):d(E,B,o);return z.shapeEnv=B.shapeEnv,z.outputSymShapes=B.outputSymShapes,z},"finish");return __(O)?O.then(R):R(O)}l(f,"_compile");function d(v,O,R){let B=new Bf({rematPolicy:R}),{backwardFunc:E,savedValues:z,gradInputIndices:D}=B.build(v),J=v.getReturnOp(),j=[...J.operands],U=j.length,P=new Map(v.args.map((Se,Qe)=>[Se.id,Qe])),V=new Map(j.map((Se,Qe)=>[Se.id,Qe])),he=[],le=new Map;for(let Se of z)P.has(Se.id)||V.has(Se.id)||le.has(Se.id)||(le.set(Se.id,U+he.length),he.push(Se));he.length>0&&(J.erase(),new An(v).returnOp([...j,...he]),v.outputTypes=Object.freeze([...j.map(Se=>Se.type),...he.map(Se=>Se.type)]));let Ie=z.map(Se=>P.has(Se.id)?{kind:"arg",index:P.get(Se.id)}:V.has(Se.id)?{kind:"output",index:V.get(Se.id)}:{kind:"output",index:le.get(Se.id)}),G=new wn("forward");G.addFunction(v);let K=new Do(i).compile(G),ie=new wn("backward");ie.addFunction(E);let ze=new Do(i).compile(ie);return{mode:"separate",fwdResult:K,bwdResult:ze,forwardFunc:v,backwardFunc:E,savedValues:z,savedSources:Ie,numRealOutputs:U,gradInputIndices:D,capturedParams:O.capturedParams,numUserInputs:O.numUserInputs,outputTypes:O.outputTypes,shapeEnv:O.shapeEnv,outputSymShapes:O.outputSymShapes}}l(d,"_compileSeparate");function m(v,O,R){let B=new Rf({rematPolicy:R}),{jointFunc:E,numForwardOutputs:z,numGradInputs:D}=B.build(v),J=new wn("joint");return J.addFunction(E),{mode:"joint",result:new Do(i).compile(J),jointFunc:E,numForwardOutputs:z,numGradInputs:D,capturedParams:O.capturedParams,numUserInputs:O.numUserInputs,outputTypes:O.outputTypes,inputTypes:v.inputTypes,shapeEnv:O.shapeEnv,outputSymShapes:O.outputSymShapes}}l(m,"_compileJoint");function g(v,O){return v.outputSymShapes&&v.shapeEnv?v.shapeEnv.resolveSymbolicShape(v.outputSymShapes[O]):[...v.outputTypes[O].shape]}l(g,"_resolveOutputShape");function y(v,O,R){let B=v.module||v;return B.executionPlan?B.runPlanAsync(B.executionPlan,R):v.isAsync(O)?v.runAsync(O,...R):(v.run(O,...R),null)}l(y,"_runK");function _(v,O){let B=v.fwdResult.listKernels()[0],E=O.length>0?O[0].device:"cpu",z=O.map(G=>Je(G)),J=v.capturedParams.map(G=>Je(G)),j=v.forwardFunc.outputTypes,U=v.numRealOutputs,P=new Array(j.length),V=new Array(j.length);for(let G=0;G<j.length;G++){let K=G<U?g(v,G):[...j[G].shape],ie=j[G].dtype,ze=cn(K),Se=Dt(ie);P[G]=new Se(Math.max(ze,1)),V[G]=K}let he=[...z,...J,...P],le=l(()=>({results:U===1?tn(P[0],V[0],j[0].dtype,E):Array.from({length:U},(K,ie)=>tn(P[ie],V[ie],j[ie].dtype,E)),inputArrays:z,paramArrays:J,outputArrays:P,device:E}),"build"),Ie=y(v.fwdResult,B,he);return Ie?Ie.then(le):le()}l(_,"_executeSeparateForward");function b(v,O,R){let E=v.bwdResult.listKernels()[0],z=O.map(ie=>Je(ie)),D=v.savedValues,J=v.savedSources,j=[...R.inputArrays,...R.paramArrays],U=new Array(D.length);for(let ie=0;ie<D.length;ie++){let ze=J[ie];U[ie]=ze.kind==="arg"?j[ze.index]:R.outputArrays[ze.index]}let P=v.backwardFunc,V=P.outputTypes.length,he=new Array(V),le=new Array(V);for(let ie=0;ie<V;ie++){let ze=[...P.outputTypes[ie].shape],Se=P.outputTypes[ie].dtype,Qe=cn(ze),It=Dt(Se);he[ie]=new It(Math.max(Qe,1)),le[ie]=ze}let Ie=[...z,...U,...he],G=l(()=>he.map((ie,ze)=>tn(ie,le[ze],P.outputTypes[ze].dtype,R.device)),"build"),K=y(v.bwdResult,E,Ie);return K?K.then(G):G()}l(b,"_executeSeparateBackward");function w(v){for(let O=0;O<u.length;O++){let R=u[O];R.shapeEnv.bindInputShapes(v);let{passed:B}=R.shapeEnv.evaluateGuards();if(B)return R}return null}l(w,"_findCachedEntry");function x(v,O){if(c=v,v.mode==="joint")return k(v,O);let R=_(v,O);return __(R)?R.then(B=>(p=B,B.results)):(p=R,R.results)}l(x,"_forwardWith");function S(v){let O=w(v);if(O)return x(O,v);let R=f(v);return __(R)?R.then(B=>(u.push(B),B.shapeEnv.bindInputShapes(v),x(B,v))):(u.push(R),R.shapeEnv.bindInputShapes(v),x(R,v))}l(S,"_runForward");function T(...v){return h?h.then(()=>S(v)):S(v)}l(T,"compiledForward");function k(v,O){let B=v.result.listKernels()[0],E=O.length>0?O[0].device:"cpu",z=O.map(K=>Je(K)),J=v.capturedParams.map(K=>Je(K)),j=v.jointFunc,U=j.outputTypes.length,P=new Array(U),V=new Array(U);for(let K=0;K<U;K++){let ie=[...j.outputTypes[K].shape],ze=j.outputTypes[K].dtype,Se=cn(ie),Qe=Dt(ze);P[K]=new Qe(Math.max(Se,1)),V[K]=ie}let he=new Array(v.numForwardOutputs);for(let K=0;K<v.numForwardOutputs;K++){let ie=v.outputTypes[K],ze=cn(ie.shape),Se=Dt(ie.dtype);he[K]=new Se(Math.max(ze,1))}p={inputArrays:z,paramArrays:J,gradOutputArrays:he,outputArrays:P,outputShapes:V,device:E,compiled:v};let le=[...z,...J,...he,...P],Ie=l(()=>{let K=[];for(let ie=0;ie<v.numForwardOutputs;ie++)K.push(tn(P[ie],V[ie],j.outputTypes[ie].dtype,E));return K.length===1?K[0]:K},"build"),G=y(v.result,B,le);return G?G.then(Ie):Ie()}l(k,"_executeJointForward");let A=T;A.backward=function(...v){if(!c||!p)throw new Error("Must run forward before backward");return c.mode==="joint"?N(c,v,p):b(c,v,p)};function N(v,O,R){let{inputArrays:B,paramArrays:E,outputArrays:z,outputShapes:D,device:J}=R,j=O.map(K=>Je(K));for(let K=0;K<j.length;K++)R.gradOutputArrays[K].set(j[K]);let U=v.jointFunc,P=U.outputTypes.length,V=new Array(P),he=new Array(P);for(let K=0;K<P;K++){let ie=[...U.outputTypes[K].shape],ze=U.outputTypes[K].dtype,Se=cn(ie),Qe=Dt(ze);V[K]=new Qe(Math.max(Se,1)),he[K]=ie}let le=[...B,...E,...j,...V],Ie=l(()=>{let K=[];for(let ie=v.numForwardOutputs;ie<P;ie++)K.push(tn(V[ie],he[ie],U.outputTypes[ie].dtype,J));return K},"build"),G=y(v.result,v.result.listKernels()[0],le);return G?G.then(Ie):Ie()}if(l(N,"_executeJointBackward"),A.original=r,A.backwardGraph=()=>{if(u.length===0)return null;let v=u[0];return v.mode==="joint"?v.jointFunc:v.backwardFunc},A.forwardGraph=()=>{if(u.length===0)return null;let v=u[0];return v.mode==="joint"?v.jointFunc:v.forwardFunc},A.capturedParams=()=>u.length?u[0].capturedParams:[],e){let v=f(e);__(v)?h=v.then(O=>(u.push(O),h=null,O)):u.push(v)}return A}var w_=M(()=>{"use strict";Sx();u_();Ks();Yo();Af();es();xx();y_();CM();Kn();ts();jr();l(__,"_isThenable");l(b_,"compileWithBackward")});function La(r){return typeof r.then=="function"}function jz(){PM||(mM(),PM=!0)}function qz(r,e){if(!r)return new Array(e.length).fill(null);let t=new Array(e.length);for(let n=0;n<e.length;n++){let s=r[n];if(s===!0){let o=new Set;for(let i=0;i<e[n].shape.length;i++)o.add(i);t[n]=o}else s instanceof Set?t[n]=s:t[n]=null}return t}function T_(r,e,t){jz();let n=t?.name||r.name||"traced",s=new Eg(n),o=qz(t?.dynamicShapes,e);for(let h=0;h<e.length;h++)s.createInput(e[h].shape,e[h].dtype,o[h]);let i=e.length,a=s._initGraph();function u(h){Array.isArray(h)?s.markOutputs(h):s.markOutput(h),s.deactivate();let f=s.getGraphModule(),d=f.functions().next().value;return{graph:f,capturedParams:[...s.capturedParams],numUserInputs:i,outputTypes:d.outputTypes,shapeEnv:s.shapeEnv,outputSymShapes:s.outputSymShapes}}l(u,"_finalize"),s.activate();let p=Pr.fromKey(48),c=LS(p,()=>r(...a));if(La(c))return c.then(h=>u(h),h=>{throw s.deactivate(),h});try{return u(c)}catch(h){throw s.deactivate(),h}}function $M(r,e,t){let n=T_(r,e,t);return La(n)?n.then(s=>s.graph):n.graph}function Uz(r,e,t){let n=r.result.listKernels();if(n.length===0)throw new Error("No kernels compiled");let s=n[0],o=e.length>0?e[0].device:"cpu",i=new Array(e.length);for(let g=0;g<e.length;g++)i[g]=Je(e[g]);let a=r.capturedParams,u=new Array(a.length);for(let g=0;g<a.length;g++)u[g]=Je(a[g]);let p=r.outputTypes,c=r.outputSymShapes,h=new Array(p.length),f=new Array(p.length);for(let g=0;g<p.length;g++){let y=c&&t?t.resolveSymbolicShape(c[g]):p[g].shape,_=p[g].dtype,b=cn(y),w=Dt(_);h[g]=new w(Math.max(b,1)),f[g]=[...y]}let d=new Array(i.length+u.length+h.length),m=0;for(let g=0;g<i.length;g++)d[m++]=new to(i[g],e[g].shape,e[g].dtype);for(let g=0;g<u.length;g++){let y=new to(u[g],a[g].shape,a[g].dtype),_=a[g]._impl;_&&(y.resident={key:_.storage.rawData,version:_.version}),d[m++]=y}for(let g=0;g<h.length;g++)d[m++]=new to(h[g],f[g],p[g].dtype);return{funcName:s,device:o,outputTypes:p,outputArrays:h,outputShapes:f,allArgs:d}}function kx(r,e,t,n){if(e.length===1)return tn(t[0],n[0],e[0].dtype,r);let s=new Array(e.length);for(let o=0;o<e.length;o++)s[o]=tn(t[o],n[o],e[o].dtype,r);return s}function Kz(r,e,t){let{funcName:n,device:s,outputTypes:o,outputArrays:i,outputShapes:a,allArgs:u}=Uz(r,e,t),p=r.result.module.executionPlan;return p?r.result.module.runPlanAsync(p,u,{resident:!0}).then(()=>kx(s,o,i,a)):r.result.isAsync(n)?r.result.runAsync(n,...u).then(()=>kx(s,o,i,a)):(r.result.run(n,...u),kx(s,o,i,a))}function Wz(r,e){let t=[];for(let n of r.result.listKernels()){let s=r.result.getSource(n);s!==null&&t.push(`// ---- compiled entry ${e}, kernel ${n} ----
+`+"  ".repeat(this.indent))}visit(e){if(!e)return;let t="visit"+e.type;this[t]?this[t](e):this.push(`[UnknownNode: ${e.type}]`)}visitPrimFunc(e){this.push(`prim_func ${e.name}(${e.params.map(t=>t.name).join(", ")}) {`),this.indent++,this.newline();for(let[t,n]of e.bufferMap)this.push(`${n.name} = buffer_map(${t.name}, shape=[${n.shape.join(",")}], dtype=${n.dtype})`),this.newline();this.visit(e.body),this.indent--,this.newline(),this.push("}")}visitSeqNode(e){for(let t=0;t<e.stmts.length;t++)this.visit(e.stmts[t]),t<e.stmts.length-1&&this.newline()}visitForNode(e){let t=e.kind==="serial"?"":`@${e.kind} `,n=e.threadTag?`[${e.threadTag}] `:"";this.push(`for ${e.loopVar.name} in 0..`),this.visit(e.extent),this.push(` ${t}${n}{`),this.indent++,this.newline(),this.visit(e.body),this.indent--,this.newline(),this.push("}")}visitBlockNode(e){this.push(`block ${e.name} {`),this.indent++,this.newline();for(let t of e.iterVars)this.push(`bind ${t.iterVar.name} = `),this.visit(t.binding),this.newline();e.reads.length>0&&(this.push("reads(["),this.push(e.reads.map(t=>`${t.buffer.name}[...]`).join(", ")),this.push("])"),this.newline()),e.writes.length>0&&(this.push("writes(["),this.push(e.writes.map(t=>`${t.buffer.name}[...]`).join(", ")),this.push("])"),this.newline()),e.initBody&&(this.push("init {"),this.indent++,this.newline(),this.visit(e.initBody),this.indent--,this.newline(),this.push("}"),this.newline()),this.visit(e.body),this.indent--,this.newline(),this.push("}")}visitBufferStoreNode(e){this.push(`${e.buffer.name}[`);for(let t=0;t<e.indices.length;t++)this.visit(e.indices[t]),t<e.indices.length-1&&this.push(", ");this.push("] = "),this.visit(e.value)}visitBufferLoadNode(e){this.push(`${e.buffer.name}[`);for(let t=0;t<e.indices.length;t++)this.visit(e.indices[t]),t<e.indices.length-1&&this.push(", ");this.push("]")}visitIfThenElseNode(e){this.push("if ("),this.visit(e.condition),this.push(") {"),this.indent++,this.newline(),this.visit(e.thenBody),this.indent--,this.newline(),this.push("}"),e.elseBody&&(this.push(" else {"),this.indent++,this.newline(),this.visit(e.elseBody),this.indent--,this.newline(),this.push("}"))}visitLetStmtNode(e){this.push(`let ${e.variable.name} = `),this.visit(e.value),this.newline(),this.visit(e.body)}visitAllocateNode(e){this.push(`allocate ${e.buffer.name}[${e.buffer.shape.join(", ")}] (${e.scope}) {`),this.indent++,this.newline(),this.visit(e.body),this.indent--,this.newline(),this.push("}")}visitEvaluateNode(e){this.push("evaluate "),this.visit(e.value)}visitMathOpNode(e){this.push("("),this.visit(e.a),e.b&&(this.push(` ${e.op} `),this.visit(e.b)),this.push(")")}visitCompareNode(e){this.push("("),this.visit(e.a),this.push(` ${Ka(e.direction)} `),this.visit(e.b),this.push(")")}visitCallExternNode(e){this.push(`${e.externName}(`);for(let t=0;t<e.args.length;t++)this.visit(e.args[t]),t<e.args.length-1&&this.push(", ");this.push(")")}visitVariableNode(e){this.push(e.name)}visitIntImmNode(e){this.push(e.value.toString())}visitFloatImmNode(e){this.push(e.value.toString())}visitCastNode(e){this.push(`cast<${e.toDtype}>(`),this.visit(e.expr),this.push(")")}};l(jg,"printTensorIR")});var qg,GO=M(()=>{"use strict";Kt();cT();qg=class{static{l(this,"TirPassManager")}constructor(){this.passes=[],this.trace=null,this.verifyHook=null}addPass(e){this.passes.push(e)}setTrace(e){this.trace=e}setVerifyHook(e){this.verifyHook=e}run(e,t){for(let n of this.passes)this._runPass(n,e,t);return e}_runPass(e,t,n){let s=n.trace;e.trace=s,s.phaseStart(e.phase);let o=performance.now();e.begin(n);for(let i=0;i<t.length;i++){let a=t[i];if(!n.failed.has(a.name))try{let u=e.run(a,n);u&&u!==a&&(t[i]=u)}catch(u){if(n.errors.push(new Dn(e.phase,a.name,u.message)),n.failed.add(a.name),s.errorEvent(e.phase,a.name,u.message),!n.resilient)break}}if(e.end(n),s.phaseEnd(e.phase,performance.now()-o),e.snapshotPoint&&s.shouldSnapshot(e.snapshotPoint))for(let i of t)n.failed.has(i.name)||s.irDump(e.snapshotPoint+":"+i.name,jg(i));this.verifyHook&&this._verifyFuncs(t,n),e.trace=null}_verifyFuncs(e,t){for(let n of e){if(t.failed.has(n.name))continue;let s=this.verifyHook(n);if(s&&s.length>0){let o=s.join("; ");if(t.resilient)t.errors.push(new Dn("verification",n.name,o)),t.failed.add(n.name);else throw new Error("TensorIR verification failed for "+n.name+": "+o)}}}}});function jO(r){let e=new Ua;return e.add(new Ja),e.add(new Za),e.add(new Qa(r)),e.add(new Xa),e.add(new eu(r)),e.add(new tu),e.add(new nu),e.add(new bh),e.add(new Ya),e.add(new vh),e.add(new Nh),e.add(new Ah),e.add(new Oh),r&&(e.add(new kh(r)),e.add(new xh(r)),e.add(new Sh(r))),e}var b$,w$,Ug,qO=M(()=>{"use strict";nn();ac();Cw();ru();l(jO,"buildAlgebraicPatterns");b$=jO(!1),w$=jO(!0),Ug=class extends Ge{static{l(this,"AlgebraicSimplificationPass")}constructor(e={}){super("algebraic_simplify"),this.preservedAnalyses=new Set,this.patterns=e.fastMath?w$:b$}run(e,t){return new Gu(this.patterns).applyPatterns(e,10,this.trace)}}});function UO(r){let e=ne.get(r);return!!(e&&e.isConstant)}function T$(r,e){return!mb(e)||typeof r!="number"?!0:Number.isInteger(r)&&Number.isSafeInteger(r)}function KO(r,e,t){if(t.has(r))return t.get(r);let n=x$(r,e,t);return t.set(r,n),n}function x$(r,e,t){let n=r.definingOp;if(!n)return;if(UO(n.opName))return n.getAttr("value");if(e.has(n))return;e.add(n);let s=ne.get(n.opName);if(!s||!s.fold||s.hasSideEffects||n.regions.length>0)return;let o=new Array(n.numOperands),i=new Array(n.numOperands);for(let a=0;a<n.numOperands;a++){let u=KO(n.getOperand(a),e,t);if(u===void 0)return;o[a]=u,i[a]=n.getOperand(a).definingOp}try{return s.fold(o,n.attributes,i)}catch{return}}var Kg,WO=M(()=>{"use strict";nn();es();At();Kt();Oe();l(UO,"isConstantProducer");l(T$,"isFoldResultRepresentable");l(KO,"resolveConstantValue");l(x$,"computeConstantValue");Kg=class extends Ge{static{l(this,"ConstantFoldPass")}constructor(){super("constant_fold")}run(e,t){let n=!1,s=0,o=new An(e),i=new Map;for(let a of[...e.opsRecursive()]){if(!a.parentBlock||UO(a.opName))continue;let u=ne.get(a.opName);if(!u||a.regions.length>0||u.hasSideEffects||u.getMemoryEffects&&u.getMemoryEffects(a).length>0||!u.fold||a.numOperands===0)continue;let p=new Array(a.numOperands),c=new Array(a.numOperands),h=!0;for(let f=0;f<a.numOperands;f++){let d=KO(a.getOperand(f),new Set,i);if(d===void 0){h=!1;break}p[f]=d,c[f]=a.getOperand(f).definingOp}if(h)try{let f=u.fold(p,a.attributes,c);if(f===void 0||!T$(f,a.getResult(0).type.dtype))continue;o.block=a.parentBlock,o.setInsertionPoint(a);let d=o.constant(f,a.getResult(0).type);a.replaceAllResultsWith([d.getResult(0)]),a.erase(),n=!0,s++}catch(f){this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,foldError:a.opName,message:f.message,level:ue.DEBUG})}}return this.trace&&this.trace.level>=ue.DEBUG&&s>0&&this.trace.emit({type:"pass_detail",passName:this.name,foldedCount:s,level:ue.DEBUG}),n?oe.CHANGED:oe.UNCHANGED}}});var Wg,HO=M(()=>{"use strict";nn();At();Kt();Wg=class extends Ge{static{l(this,"CSEPass")}constructor(){super("cse"),this.preservedAnalyses=new Set}run(e,t){let n=!1,s=0,o=typeof e.blocksRecursive=="function"?[...e.blocksRecursive()]:[...e.body];for(let i of o){let a=new Map;for(let u of[...i.ops()]){if(!u.parentBlock||u.regions&&u.regions.length>0)continue;let p=ne.get(u.opName);if(p&&p.hasSideEffects||p&&p.getMemoryEffects&&p.getMemoryEffects(u).length>0)continue;let c=u.structuralHash();if(!a.has(c)){a.set(c,[u]);continue}let h=a.get(c),f=!1;for(let d of h)if(d.parentBlock&&d.structuralEquals(u)){let m=[];for(let g=0;g<d.numResults;g++)m.push(d.getResult(g));u.replaceAllResultsWith(m),u.erase(),n=!0,s++,f=!0;break}f||h.push(u)}}return this.trace&&this.trace.level>=ue.DEBUG&&s>0&&this.trace.emit({type:"pass_detail",passName:this.name,eliminated:s,level:ue.DEBUG}),n?oe.CHANGED:oe.UNCHANGED}}});function fT(r,e){let t=pT[r],n=pT[e];return t===void 0||n===void 0?!1:r===je.REDUCTION?e===je.ELEMENTWISE:e===je.REDUCTION?t<=pT[je.INJECTIVE]:!0}function _l(r){let e=!1,t=!1,n=!1,s=!1;for(let o of r){let i=ne.get(o.opName);if(!i||i.isOpaque){n=!0;continue}i.isReduction?e=!0:i.isInjective?t=!0:i.isBroadcast?s=!0:i.isElementwise||(n=!0)}return n?je.OPAQUE:e?je.REDUCTION:t?je.INJECTIVE:s?je.BROADCAST:je.ELEMENTWISE}function Aa(r){let e=ne.get(r.opName);return!e||e.isOpaque?je.OPAQUE:e.isReduction?je.REDUCTION:e.isInjective?je.INJECTIVE:e.isBroadcast?je.BROADCAST:e.isElementwise?je.ELEMENTWISE:je.OPAQUE}function YO(r,e){if(r.length!==e.length)return!1;for(let t=0;t<r.length;t++)if(!(r[t]===X||e[t]===X)&&r[t]!==e[t])return!1;return!0}var je,pT,gl,yl,Oa=M(()=>{"use strict";Oe();At();Ui();Tr();je=Object.freeze({ELEMENTWISE:"kElementwise",BROADCAST:"kBroadcast",REDUCTION:"kReduction",INJECTIVE:"kInjective",OPAQUE:"kOpaque",HORIZONTAL:"kHorizontal"}),pT={[je.ELEMENTWISE]:0,[je.BROADCAST]:1,[je.INJECTIVE]:2,[je.REDUCTION]:3};l(fT,"canFusePatterns");l(_l,"classifyFusionKind");l(Aa,"classifyOpPattern");gl=Object.freeze({legal:!0}),yl=class{static{l(this,"FusionLegality")}constructor(e={}){this.maxFusionSize=e.maxFusionSize||512,this.maxSharedMemory=e.maxSharedMemory||49152,this.libraryOps=e.libraryOps||new Set,this.allowReductionFusion=e.allowReductionFusion!==!1,this._lowerableCache=new Map}isOpLowerable(e){let t=this._lowerableCache.get(e);return t===void 0&&(t=qt(e)||ko(e)||$b(e),this._lowerableCache.set(e,t)),t}canFuse(e,t){if(!e||!t)return{legal:!1,reason:"null op"};if(e===t)return{legal:!1,reason:"same op"};if(e.regions.length>0&&e.opName!=="fusion"&&e.opName!=="reduce")return{legal:!1,reason:"producer has control flow regions"};if(t.regions.length>0&&t.opName!=="fusion"&&t.opName!=="reduce")return{legal:!1,reason:"consumer has control flow regions"};let n=ne.get(e.opName),s=ne.get(t.opName);if(!n)return{legal:!1,reason:"unknown producer op"};if(!s)return{legal:!1,reason:"unknown consumer op"};if(!this.isOpLowerable(e.opName))return{legal:!1,reason:`producer op '${e.opName}' has no lowering rule`};if(!this.isOpLowerable(t.opName))return{legal:!1,reason:`consumer op '${t.opName}' has no lowering rule`};if(n.isOpaque)return{legal:!1,reason:"producer is opaque (use EpilogueFusionPass for dot/conv epilogues)"};if(s.isOpaque)return{legal:!1,reason:"consumer is opaque"};let o=Aa(e),i=Aa(t);return fT(o,i)?(o===je.REDUCTION||i===je.REDUCTION)&&!this.allowReductionFusion?{legal:!1,reason:"reduction fusion disabled by target"}:o===je.ELEMENTWISE&&i===je.ELEMENTWISE?this._checkElementwisePair(e,t):(o===je.BROADCAST||o===je.REDUCTION)&&i===je.ELEMENTWISE?gl:this._checkProducerConsumerShapes(e,t):{legal:!1,reason:`cannot fuse pattern ${o} -> ${i}`}}canMergeGroups(e,t){if(e.size+t.size>this.maxFusionSize)return{legal:!1,reason:"merged group exceeds max fusion size"};let n=0;for(let s of e.ops){let o=ne.get(s.opName);if(o&&o.isReduction&&n++,o&&o.isOpaque)return{legal:!1,reason:"opaque op in merge (use EpilogueFusionPass)"}}for(let s of t.ops){let o=ne.get(s.opName);if(o&&o.isReduction&&n++,o&&o.isOpaque)return{legal:!1,reason:"opaque op in merge (use EpilogueFusionPass)"}}return n>1?{legal:!1,reason:"merged group would contain multiple reductions"}:gl}_checkElementwisePair(e,t){let n=this._getOutputShape(e),s=this._getOutputShape(t);return n&&s&&!YO(n,s)?{legal:!1,reason:`elementwise shape mismatch: [${n}] vs [${s}]`}:gl}_checkProducerConsumerShapes(e,t){let n=this._getOutputShape(e);if(!n)return gl;for(let s=0;s<t.numOperands;s++)if(t.getOperand(s).definingOp===e){let o=t.getOperand(s).type;if(o instanceof I)return YO(n,o.shape)?gl:{legal:!1,reason:`shape mismatch on data edge: [${n}] vs [${o.shape}]`}}return gl}_getOutputShape(e){for(let t=0;t<e.numResults;t++){let n=e.getResult(t).type;if(n instanceof I)return n.shape}return null}};l(YO,"shapesCompatible")});function Ba(r){return Fh(r,"throw")}function tf(r,{labelOf:e,sameLabel:t=l((i,a)=>i===a,"sameLabel"),canMerge:n=l(()=>!0,"canMerge"),onAttach:s=l(()=>{},"onAttach"),sort:o=Ba}){let i=o(r),a=new Map,u=new Map,p=[],c=0,h=l((m,g)=>{if(m===g)return!0;let y=[g],_=new Set;for(;y.length>0;){let b=y.pop();if(b===m)return!0;if(_.has(b))continue;_.add(b);let w=u.get(b);if(w)for(let x of w)y.push(x)}return!1},"isUpstreamOf"),f=l(m=>{let g=new Set;for(let y=0;y<m.numOperands;y++){let _=m.getOperand(y).definingOp;if(!_)continue;let b=a.get(_);b&&g.add(b)}return g},"operandParts"),d=l((m,g)=>{for(let y of f(m)){if(y===g)continue;let _=u.get(g);_||(_=new Set,u.set(g,_)),_.add(y)}},"recordEdges");for(let m of i){let g=e(m);if(g==null)continue;let y=!1;for(let _=0;_<m.numOperands;_++){let b=m.getOperand(_).definingOp;if(!b)continue;let w=a.get(b);if(!w||!t(w.label,g)||!n(w,m,g))continue;let x=!1;for(let S of f(m))if(S!==w&&h(w,S)){x=!0;break}if(!x){w.ops.push(m),w.opSet.add(m),s(w,m),a.set(m,w),d(m,w),y=!0;break}}if(!y){let _={id:c++,label:g,ops:[m],opSet:new Set([m])};s(_,m),p.push(_),a.set(m,_),d(m,_)}}return{partitions:p,opToPart:a,preds:u}}function _i(r,e,{pullConstants:t=!1,isConstant:n=l(()=>!1,"isConstant")}={}){let s=[],o=new Set,i=[],a=new Set,u=[],p=new Set;for(let c of e){for(let h=0;h<c.numOperands;h++){let f=c.getOperand(h),d=f.definingOp;if(!(d&&r.has(d))){if(t&&d&&n(d)){p.has(d)||(p.add(d),u.push(d));continue}o.has(f)||(o.add(f),s.push(f))}}for(let h=0;h<c.numResults;h++){let f=c.getResult(h);if(a.has(f))continue;let d=!1;for(let m of f.uses())if(!r.has(m.user)){d=!0;break}d&&(a.add(f),i.push(f))}}return{inputs:s,outputs:i,constDefs:u}}function hT(r,e){let t=new Map,n=new Map;for(let a of r)t.set(a,0),n.set(a,[]);for(let a of r){let u=e.get(a);if(u)for(let p of u)n.has(p)&&(n.get(p).push(a),t.set(a,t.get(a)+1))}let s=[];for(let a of r)t.get(a)===0&&s.push(a);let o=[],i=0;for(;i<s.length;){let a=s[i++];o.push(a);for(let u of n.get(a)){let p=t.get(u)-1;t.set(u,p),p===0&&s.push(u)}}return o.length===r.length?o:null}var nf=M(()=>{"use strict";ji();l(Ba,"topoSortOps");l(tf,"buildPartitions");l(_i,"computePartitionIO");l(hT,"topoSortPartitions")});function S$(r){for(let e=0;e<r.numResults;e++){let t=r.getResult(e).type;if(t instanceof I)return t.shape.join(",")}return null}var Eo,Hg,Yg=M(()=>{"use strict";Oe();At();Oa();Ui();nf();Eo=class{static{l(this,"FusionGroup")}constructor(e){this.id=e,this.ops=[],this.opSet=new Set,this.kind=null,this._inputValues=null,this._outputValues=null,this.minTopoPos=1/0,this.maxTopoPos=-1/0}addOp(e,t){this.opSet.has(e)||(this.ops.push(e),this.opSet.add(e),this._inputValues=null,this._outputValues=null,t!==void 0&&(t<this.minTopoPos&&(this.minTopoPos=t),t>this.maxTopoPos&&(this.maxTopoPos=t)))}hasOp(e){return this.opSet.has(e)}merge(e){for(let t of e.ops)this.addOp(t);e.minTopoPos<this.minTopoPos&&(this.minTopoPos=e.minTopoPos),e.maxTopoPos>this.maxTopoPos&&(this.maxTopoPos=e.maxTopoPos)}computeIO(){if(this._inputValues&&this._outputValues)return;let{inputs:e,outputs:t}=_i(this.opSet,this.ops);this._inputValues=e,this._outputValues=t}getInputValues(){return this.computeIO(),this._inputValues}getOutputValues(){return this.computeIO(),this._outputValues}get size(){return this.ops.length}classifyKind(){return this.kind=_l(this.ops),this.kind}allOpsInlineFusable(){for(let e of this.ops){let t=ne.get(e.opName);if(!t)return!1;if(!(t.isReduction||t.isConstant)&&!ko(e.opName))return!1}return!0}};l(S$,"outputShapeKey");Hg=class{static{l(this,"FusionGroupBuilder")}constructor(e){this.legality=e,this._nextId=0,this._topoIndex=null}buildProducerConsumerGroups(e){this._topoIndex=new Map;let t=0;for(let i of e.ops())this._topoIndex.set(i,t++);let n=[],s=new Map;for(let i of e.ops()){let a=ne.get(i.opName);if(!a||a.isConstant||a.isTerminator||a.isReduction)continue;let u=this._topoIndex.get(i);for(let p=0;p<i.numOperands;p++){let c=i.getOperand(p).definingOp;if(!c)continue;let h=ne.get(c.opName);if(!h||h.isConstant||h.isReduction)continue;let f=s.get(i),d=s.get(c);if(f&&f===d||!this.legality.canFuse(c,i).legal)continue;let g=this._topoIndex.get(c);if(!f&&!d){let y=new Eo(this._nextId++);y.addOp(c,g),y.addOp(i,u),s.set(c,y),s.set(i,y)}else if(f&&!d)f.size<this.legality.maxFusionSize&&!this._wouldCreateCycle(f,c)&&(f.addOp(c,g),s.set(c,f));else if(!f&&d)d.size<this.legality.maxFusionSize&&!this._wouldCreateCycle(d,i)&&(d.addOp(i,u),s.set(i,d));else if(this.legality.canMergeGroups(f,d).legal&&!this._mergeWouldCreateCycle(f,d)){f.merge(d);for(let _ of d.ops)s.set(_,f)}}}let o=new Set;for(let i of s.values())o.has(i)||i.size<2||(o.add(i),i.classifyKind(),n.push(i));return n}_wouldCreateCycle(e,t){for(let n=0;n<t.numOperands;n++){let s=t.getOperand(n).definingOp;if(!(!s||e.hasOp(s)||this._topoIndex.get(s)<e.minTopoPos)&&this._dependsOnGroup(s,e))return!0}return!1}_dependsOnOps(e,t,n){let s=new Set,o=[e];for(s.add(e);o.length>0;){let i=o.pop();for(let a=0;a<i.numOperands;a++){let u=i.getOperand(a).definingOp;if(!u||s.has(u))continue;if(t.has(u))return!0;this._topoIndex.get(u)<n||(s.add(u),o.push(u))}}return!1}_dependsOnGroup(e,t){return this._dependsOnOps(e,t.opSet,t.minTopoPos)}_mergeWouldCreateCycle(e,t){let n=Math.min(e.minTopoPos,t.minTopoPos),s=Math.max(e.maxTopoPos,t.maxTopoPos),o=new Set([...e.opSet,...t.opSet]);for(let i of o)for(let a=0;a<i.numOperands;a++){let u=i.getOperand(a).definingOp;if(!u||o.has(u))continue;let p=this._topoIndex.get(u);if(!(p<n||p>s)&&this._dependsOnOps(u,o,n))return!0}return!1}_bucketable(e,t){return t&&!t.isConstant&&!t.isTerminator&&!t.isOpaque&&e.numResults>0&&S$(e)!==null}buildHorizontalGroups(e){let t=[...e.ops()];this._topoIndex=new Map;for(let u=0;u<t.length;u++)this._topoIndex.set(t[u],u);let n=[],s=new Map,o=this.legality.maxFusionSize||t.length,i=new Map,a=0;for(let u=0;u<t.length;u++){let p=t[u];if(s.has(p))continue;let c=ne.get(p.opName);if(!this._bucketable(p,c))continue;let h=p.getResult(0).type;a++,i.set(p,a);let f=new Eo(this._nextId++);f.addOp(p,u),s.set(p,f);let d=Math.min(t.length,u+1+o);for(let m=u+1;m<d;m++){let g=t[m],y=!1;for(let b=0;b<g.numOperands;b++){let w=g.getOperand(b).definingOp;if(w&&i.get(w)===a){y=!0;break}}if(y){i.set(g,a);continue}if(f.size>=o||s.has(g))continue;let _=ne.get(g.opName);this._bucketable(g,_)&&h.equals(g.getResult(0).type)&&(this._sharesInput(p,g)||c.isElementwise&&_.isElementwise)&&(f.addOp(g,m),s.set(g,f),i.set(g,a))}f.size>=2&&(f.kind=je.HORIZONTAL,n.push(f))}return n}buildAllGroups(e){let t=this.buildProducerConsumerGroups(e),n=this.buildHorizontalGroups(e),s=new Map;for(let c of t)for(let h of c.ops)s.set(h,c);let o=new Set(s.keys()),i=[];for(let c of n){let h=!1;for(let f of c.ops)if(o.has(f)){h=!0;break}h||i.push(c)}let a=[...e.ops()],u=[];for(let c of a)for(let h=0;h<c.numOperands;h++){let f=c.getOperand(h).definingOp;f&&u.push(c,f)}for(let c of i)for(let h of c.ops)s.set(h,c);if(!this._condensedHasCycle(a,u,s))return[...t,...i];for(let c of i)for(let h of c.ops)s.delete(h);let p=[...t];for(let c of i){for(let h of c.ops)s.set(h,c);if(this._condensedHasCycle(a,u,s)){for(let h of c.ops)s.delete(h);continue}p.push(c)}return p}_condensedHasCycle(e,t,n){let s=l(h=>n.get(h)||h,"repOf"),o=new Map,i=new Set;for(let h of e)i.add(s(h));for(let h=0;h<t.length;h+=2){let f=s(t[h]),d=s(t[h+1]);if(d===f)continue;i.add(d),i.add(f);let m=o.get(d);m||(m=new Set,o.set(d,m)),m.add(f)}let a=0,u=1,p=2,c=new Map;for(let h of i)c.set(h,a);for(let h of i){if(c.get(h)!==a)continue;let f=[h];for(;f.length>0;){let d=f[f.length-1],m=c.get(d);if(m===a){c.set(d,u);let g=o.get(d);if(g)for(let y of g){let _=c.get(y);if(_===u)return!0;_===a&&f.push(y)}}else m===u&&c.set(d,p),f.pop()}}return!1}_sharesInput(e,t){if(e.numOperands===0||t.numOperands===0)return!1;let n=new Set;for(let s=0;s<e.numOperands;s++)n.add(e.getOperand(s));for(let s=0;s<t.numOperands;s++)if(n.has(t.getOperand(s)))return!0;return!1}_hasDependency(e,t){for(let n=0;n<e.numResults;n++){let s=e.getResult(n);for(let o=0;o<t.numOperands;o++)if(t.getOperand(o)===s)return!0}return!1}}});var k$,bi,Jg=M(()=>{"use strict";Oe();At();k$=Object.freeze({memory:1,launch:1e3}),bi=class{static{l(this,"FusionCostModel")}constructor(e={}){this.memoryBandwidthGBs=e.memoryBandwidthGBs||900,this.computeTFLOPs=e.computeTFLOPs||15,this.launchOverheadUs=e.launchOverheadUs||5,this.minBenefitRatio=e.minBenefitRatio||1.05,this.maxRegistersPerThread=e.maxRegistersPerThread||255,this.maxSharedMemory=e.maxSharedMemory||49152,this.maxCodeSizeOps=e.maxCodeSizeOps||256,this.libraryOps=e.libraryOps||new Set,this.registerBytesPerOp=e.registerBytesPerOp||8,this.policy=e.policy||null,this.benefitWeights={...k$,...e.benefitWeights||{}}}edgeBenefit(e){let t=this.benefitWeights;return t.launch*this.launchOverheadUs+t.memory*e}estimateOpCost(e){let t=this.estimateFLOPs(e),n=this.estimateBytes(e);return{flops:t,bytes:n,arithmeticIntensity:n>0?t/n:0}}estimateFLOPs(e){let t=ne.get(e.opName);if(t&&t.getFlops)return t.getFlops(e);let n=1;for(let s=0;s<e.numResults;s++){let o=e.getResult(s).type;if(o instanceof I){let i=o.numel();i!==X&&(n=i);break}}if(t&&t.isReduction&&e.numOperands>0){let s=e.getOperand(0).type;if(s instanceof I){let o=s.numel();if(o!==X)return o}}return n}estimateBytes(e){let t=0;for(let n=0;n<e.numOperands;n++){let s=e.getOperand(n).type;if(s instanceof I){let o=s.sizeInBytes();o!==X&&(t+=o)}}for(let n=0;n<e.numResults;n++){let s=e.getResult(n).type;if(s instanceof I){let o=s.sizeInBytes();o!==X&&(t+=o)}}return t}estimateGroupCost(e){let t=e.opSet,n=new Map,s=0,o=0,i=0,a=0,u=!1,p=!1,c=0,h=0,f=0,d=0,m=new Set;for(let x of e.ops){let S=this.estimateFLOPs(x);n.set(x,S),s+=S,o+=this.estimateBytes(x),this.libraryOps.has(x.opName)&&a++;let T=ne.get(x.opName);if(T){let k=this._outputElements(x);T.isReduction&&(u=!0,c+=k),T.isElementwise&&(p=!0,h+=k)}for(let k=0;k<x.numResults;k++){let A=x.getResult(k);m.add(A);let N=0,v=!1;for(let O of A.uses())t.has(O.user)?N++:v=!0;N>1&&(i+=(N-1)*S),(N>0||v)&&f++}for(let k=0;k<x.numOperands;k++){let A=x.getOperand(k);if(!m.has(A))continue;let N=!1;for(let v of A.uses())if(t.has(v.user)&&v.user!==x){N=!0;break}N||f--}f>d&&(d=f)}let g=e.getInputValues(),y=e.getOutputValues(),_=0;for(let x of g)if(x.type instanceof I){let S=x.type.sizeInBytes();S!==X&&(_+=S)}for(let x of y)if(x.type instanceof I){let S=x.type.sizeInBytes();S!==X&&(_+=S)}let b=0;for(let x of e.ops)for(let S=0;S<x.numResults;S++){let T=x.getResult(S),k=0;for(let A of T.uses())if(t.has(A.user)&&(k++,k>1))break;if(k>1){let A=T.type;if(A instanceof I){let N=A.sizeInBytes();N!==X&&(b+=N)}}}let w=0;return u&&p&&h>0&&c>0&&(w=Math.abs(h-c)),{unfusedFLOPs:s,unfusedBytes:o,fusedFLOPs:s+i,fusedBytes:_,recomputeCost:i,memorySaved:o-_,launchSaved:(e.size-1)*this.launchOverheadUs,registerPressure:d*this.registerBytesPerOp,sharedMemoryUsage:b,parallelismLoss:w,libraryCallLoss:a}}shouldFuse(e){if(e.size<2)return{fuse:!1,reason:"group too small"};if(this.policy&&typeof this.policy.shouldFuse=="function"){let n=this.policy.shouldFuse(e,this);if(n)return n}let t=this.estimateGroupCost(e);if(t.libraryCallLoss>0)return{fuse:!1,reason:"fusion would lose library call opportunity",cost:t};if(t.registerPressure>this.maxRegistersPerThread)return{fuse:!1,reason:`register pressure ${t.registerPressure} exceeds limit ${this.maxRegistersPerThread}`,cost:t};if(t.sharedMemoryUsage>this.maxSharedMemory)return{fuse:!1,reason:`shared memory ${t.sharedMemoryUsage} exceeds limit ${this.maxSharedMemory}`,cost:t};if(e.size>this.maxCodeSizeOps)return{fuse:!1,reason:`code size ${e.size} exceeds limit ${this.maxCodeSizeOps}`,cost:t};if(t.memorySaved<=0&&t.launchSaved<=0)return{fuse:!1,reason:"no memory or launch benefit",cost:t};if(t.parallelismLoss>0){let n=t.memorySaved+t.launchSaved*1e3;if(t.parallelismLoss>n*this.minBenefitRatio)return{fuse:!1,reason:"parallelism loss outweighs fusion benefit",cost:t}}return{fuse:!0,reason:`saves ${t.memorySaved} bytes, ${t.launchSaved}us launch`,cost:t}}_outputElements(e){for(let t=0;t<e.numResults;t++){let n=e.getResult(t).type;if(n instanceof I){let s=n.numel();if(s!==X)return s}}return 0}}});function bl(r){let e=null;for(let t of r.ops())e=t;return e&&e.opName==="yield"?e:null}function wl(r){let e=0,t=r.regions[0]?.entryBlock;if(!t)return 0;for(let n of t.ops())n.opName!=="yield"&&e++;return e}function Tl(r){let e=0,t=r.regions[0]?.entryBlock;if(!t)return 0;for(let n of t.ops()){let s=ne.get(n.opName);s&&s.isReduction&&e++}return e}function xl(r){let e=r.regions[0]?.entryBlock;if(!e)return!1;for(let t of e.ops())if(t.opName!=="yield"&&!ko(t.opName))return!1;return!0}function v$(r){let e=new Map,t=0;for(let n=r.firstOp;n;n=n._next)e.set(n,t++);return e}function dT(r){let e=v$(r);return(t,n)=>{let s=e.get(t),o=e.get(n);return s!==void 0&&o!==void 0&&s<o}}function Sl(r,e){let t=wo(r.ops,g=>r.hasOp(g),"null");if(t===null||t.length===0)return null;r._inputValues=null,r._outputValues=null;let n=r.getInputValues(),s=r.getOutputValues(),o=n.map(g=>g.type),i=s.map(g=>g.type),a=new Ct,u=new yn(o);a.addBlock(u);let p=new Map;for(let g=0;g<n.length;g++)p.set(n[g],u.arguments[g]);for(let g of t)u.pushOp(g.clone(p));let c=s.map(g=>{let y=p.get(g);if(y===void 0)throw new Error("Fusion materialization: output value not found in valueMap");return y});u.pushOp(new Re("yield",c,[]));let h=new Re("fusion",n,i,{fusion_kind:r.kind||e},[a]),f=t[0].parentBlock;if(!f)return null;let d=dT(f),m=null;for(let g of n){let y=g.definingOp;!y||r.hasOp(y)||(!m||!d(y,m))&&(m=y)}m&&m.parentBlock===f?f.insertAfter(h,m):f.insertBefore(h,t[0]);for(let g=0;g<s.length;g++)s[g].replaceAllUsesWith(h.getResult(g));for(let g of t)g.dropAllOperands(),g.parentBlock&&g.parentBlock.removeOp(g);return h}var kl=M(()=>{"use strict";At();Ui();sr();zi();ji();l(bl,"getYieldOp");l(wl,"countInnerOps");l(Tl,"countReductions");l(xl,"allInnerOpsFusable");l(v$,"blockPositionIndex");l(dT,"makeComesBefore");l(Sl,"materializeFusionGroup")});var Zg,JO=M(()=>{"use strict";nn();Oa();Yg();Jg();kl();Kt();Zg=class extends Ge{static{l(this,"FusionPass")}constructor(e={}){super("FusionPass");let t=e.target||{};this.legality=new yl({maxFusionSize:t.maxFusionSize||e.maxFusionSize,maxSharedMemory:t.sharedMemoryBytes||e.maxSharedMemory,libraryOps:t.libraryOps||e.libraryOps,allowReductionFusion:e.allowReductionFusion}),this.costModel=new bi({memoryBandwidthGBs:t.memoryBandwidthGBs,computeTFLOPs:t.computeTFLOPs,maxRegistersPerThread:t.registersPerThread,maxSharedMemory:t.sharedMemoryBytes,libraryOps:t.libraryOps,policy:t.getAttr?t.getAttr("fusionPolicy"):null,...e.cost}),this.groupBuilder=new Hg(this.legality)}run(e,t){let n=this.groupBuilder.buildAllGroups(e),s=[];for(let i of n)this._createsCycle(e,i)?this._traceDecision(i,!1,"fusing would create a dependency cycle"):s.push(i);let o=[];for(let i of s){if(!i.allOpsInlineFusable()){this._traceDecision(i,!1,"group contains ops without inline fusion support");continue}let a=this.costModel.shouldFuse(i);this._traceDecision(i,a.fuse,a.reason),a.fuse&&o.push(i)}if(o.length===0)return oe.UNCHANGED;for(let i of o)Sl(i,je.ELEMENTWISE);return oe.CHANGED}_traceDecision(e,t,n){if(!this.trace||this.trace.level<ue.DEBUG)return;let s=e.ops.map(o=>o.opName);this.trace.emit({type:"fusion_decision",passName:this.name,groupSize:e.ops.length,ops:s,anchor:s[s.length-1]||null,fuse:t,reason:n||null,level:ue.DEBUG}),this.trace.explain("fusion",s.join("+"),t?"fused":"not-fused",n||null,{groupSize:s.length})}_createsCycle(e,t){let n=t.opSet,s=t.getOutputValues(),o=new Set,i=[];for(let u of s)for(let p of u.uses())n.has(p.user)||(i.push(p.user),o.add(p.user));let a=0;for(;a<i.length;){let u=i[a++];if(n.has(u))return!0;for(let p=0;p<u.numResults;p++)for(let c of u.getResult(p).uses())o.has(c.user)||(o.add(c.user),i.push(c.user))}return!1}}});function Qg(r){return lu(r.opName)||ln(r.opName)}function ZO(r){if(Qg(r))return!0;let e=ne.get(r.opName);return e!==null&&e.isElementwise}function A$(r,e){let t=r.getOperand(0).definingOp;return t&&e.has(t)&&!Qg(t)?r.getOperand(1).definingOp:t}function O$(r,e){if(!r.parentBlock||r.parentBlock!==e.parentBlock)return!1;let t=r.parentBlock.firstOp;for(;t;){if(t===r)return!0;if(t===e)return!1;t=t._next}return!1}function B$(r,e){for(let t of r)if(t!==e){for(let n=0;n<t.numResults;n++)for(let s of t.getResult(n).uses())if(!r.has(s.user))return!0}return!1}function R$(r,e){let t=ne.get(r.opName),n=t&&t.getAttr("epilogueTag");return n?n(r,e):"activation"}function M$(r){let e=[],t=new Set,n=new Set;function s(f){if(!(n.has(f)||f===r)){n.add(f);for(let d=0;d<f.numOperands;d++){let m=f.getOperand(d).definingOp;m&&m!==r&&!n.has(m)&&Qg(m)&&s(m)}e.push(f),t.add(f)}}l(s,"absorb");let o=[],i=r.getResult(0);for(let f of i.uses())ZO(f.user)&&o.push(f.user);for(;o.length>0;){let f=o.pop();if(!n.has(f)){s(f);for(let d=0;d<f.numResults;d++)for(let m of f.getResult(d).uses())!n.has(m.user)&&ZO(m.user)&&o.push(m.user)}}let a=[];for(let f of e)Qg(f)||a.push(R$(f,t));let u=e[e.length-1];for(let f=e.length-1;f>=0;f--){let d=e[f];for(let m=0;m<d.numResults;m++){for(let g of d.getResult(m).uses())if(!t.has(g.user)){u=d,f=-1;break}if(f<0)break}}let p=new Set;for(let f=0;f<r.numOperands;f++)p.add(r.getOperand(f));let c=[],h=new Set;for(let f of e)for(let d=0;d<f.numOperands;d++){let m=f.getOperand(d);if(h.has(m))continue;h.add(m);let g=m.definingOp;g!==r&&(g&&t.has(g)||p.has(m)||c.push(m))}return{chain:e,chainSet:t,tags:a,lastOp:u,extras:c}}function I$(r){let e=null,t=null,n=null,s=null,o=new Set;for(let i=0;i<2;i++){let a=r.getOperand(i),u=a.definingOp;if(!u||u.opName!=="convert")continue;let p=!1;for(let h of a.uses())if(h.user!==r){p=!0;break}if(p)continue;let c=u.getAttr("target_dtype")||u.getResult(0).type.dtype;i===0?(e=c,n=u.getOperand(0)):(t=c,s=u.getOperand(0)),o.add(u)}return{lhsCast:e,rhsCast:t,lhsInput:n,rhsInput:s,removed:o}}var N$,Xg,QO=M(()=>{"use strict";nn();sr();At();Tr();Kt();l(Qg,"isPassthrough");l(ZO,"isEpilogueCandidate");N$=new Map([["add",(r,e)=>e.has(A$(r,e))?"residual_add":"bias"],["sub",()=>"bias"],["mul",()=>"scale"],["maximum",()=>"relu"],["clamp",()=>"clamp"],["neg",()=>"neg"],["exp",()=>"exp"],["tanh",()=>"tanh"],["sqrt",()=>"sqrt"],["abs",()=>"abs"],["log",()=>"log"]]);l(A$,"resolveOtherOperand");for(let[r,e]of N$)ne.has(r)&&ne.registerOpAttr(r,"epilogueTag",e);l(O$,"comesBefore");l(B$,"hasEscapingUse");l(R$,"classifyTag");l(M$,"collectChainAndAnalyze");l(I$,"collectPrologue");Xg=class extends Ge{static{l(this,"EpilogueFusionPass")}constructor(e={}){super("EpilogueFusionPass"),this.maxEpilogueOps=e.maxEpilogueOps||16,this.target=e.target||null}run(e){if(this.target&&!this.target.enableEpilogueFusion)return oe.UNCHANGED;let t=!1,n=[];for(let s of e.ops()){let o=ne.get(s.opName);o&&o.isOutEWiseFusable&&n.push(s)}for(let s of n){let o=M$(s);if(o.chain.length>this.maxEpilogueOps)continue;let i=I$(s),a=o.chain.length>0,u=i.lhsCast!==null||i.rhsCast!==null;if(!a&&!u)continue;let{chain:p,tags:c,extras:h}=o,f=0;for(let k of c)k==="bias"||k==="residual_add"||k==="scale"?f++:k==="clamp"&&(f+=2);if(f!==h.length)continue;let d=i.lhsInput||s.getOperand(0),m=i.rhsInput||s.getOperand(1),g=[d,m,...h],y=a?o.lastOp:s,_=y.getResult(0).type,b=new Map(s.attributes);b.set("epilogue_ops",p.map(k=>k.opName)),b.set("epilogue_tags",c),b.set("num_dot_operands",2),b.set("num_extra_inputs",h.length),i.lhsCast&&b.set("lhs_prologue_cast",i.lhsCast),i.rhsCast&&b.set("rhs_prologue_cast",i.rhsCast);let w=new Re("fused_dot_epilogue",g,[_],b),x=s.parentBlock;if(!x)continue;let S=new Set(p);S.add(s);for(let k of i.removed)S.add(k);if(B$(S,y))continue;let T=null;for(let k of g){let A=k.definingOp;!A||S.has(A)||(!T||!O$(A,T))&&(T=A)}T&&T.parentBlock===x?x.insertAfter(w,T):x.insertBefore(w,s),y.getResult(0).replaceAllUsesWith(w.getResult(0));for(let k=p.length-1;k>=0;k--)p[k].dropAllOperands(),p[k].parentBlock&&p[k].parentBlock.removeOp(p[k]);s.dropAllOperands(),s.parentBlock&&s.parentBlock.removeOp(s);for(let k of i.removed)k.dropAllOperands(),k.parentBlock&&k.parentBlock.removeOp(k);t=!0}return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,dotsFound:n.length,changed:t,level:ue.DEBUG}),t?oe.CHANGED:oe.UNCHANGED}}});var ey,XO=M(()=>{"use strict";nn();sr();zi();Oe();At();Kt();Oa();kl();ey=class extends Ge{static{l(this,"FusionMergerPass")}constructor(e={}){super("FusionMergerPass"),this.maxFusionSize=e.maxFusionSize||512,this.maxReductions=e.maxReductions||1,this.launchOverheadUs=e.launchOverheadUs||5,this.minMemorySavings=e.minMemorySavings||0}run(e){let t=[];for(let u of e.ops())u.opName==="fusion"&&t.push(u);if(t.length<2)return oe.UNCHANGED;let n=new Set(t),s=this._buildProducerConsumerEdges(t,n);if(s.length===0)return oe.UNCHANGED;let o=!1,i=0,a=new Set;for(let{producer:u,consumer:p,sharedResults:c}of s)a.has(u)||a.has(p)||this._canMerge(u,p)&&this._shouldMerge(u,p,c)&&(this._mergeCreatesCycle(u,p,n)||(this._merge(u,p,c),a.add(u),a.add(p),o=!0,i++));return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,fusionOps:t.length,edges:s.length,mergeCount:i,level:ue.DEBUG}),o?oe.CHANGED:oe.UNCHANGED}_buildProducerConsumerEdges(e,t){let n=[],s=new Map;for(let o of e)for(let i=0;i<o.numResults;i++){let a=o.getResult(i);for(let u of a.uses()){if(!t.has(u.user))continue;let p=u.user;if(p===o)continue;let c=`${o.id}|${p.id}`,h=s.get(c);h||(h={producer:o,consumer:p,sharedResults:new Map},s.set(c,h),n.push(h)),h.sharedResults.set(i,u.operandIndex)}}return n}_canMerge(e,t){let n=wl(e),s=wl(t);if(n+s>this.maxFusionSize)return!1;let o=0;return o+=Tl(e),o+=Tl(t),!(o>this.maxReductions||!xl(e)||!xl(t))}_shouldMerge(e,t,n){let s=0;for(let[u]of n){let p=e.getResult(u);if(p.type instanceof I){let c=p.type.sizeInBytes();c!==X&&(s+=c)}}let o=!0;for(let u=0;u<e.numResults;u++){let p=e.getResult(u);for(let c of p.uses())if(c.user!==t){o=!1;break}if(!o)break}if(o)return!0;let i=this._estimateRecomputeCost(e);return s+this.launchOverheadUs*1e3>i}_mergeCreatesCycle(e,t,n){let s=new Set,o=[];for(let a=0;a<t.numOperands;a++){let p=t.getOperand(a).definingOp;p&&p!==e&&p.opName!=="constant"&&(s.has(p)||(s.add(p),o.push(p)))}let i=0;for(;i<o.length;){let a=o[i++];if(a===e)return!0;for(let u=0;u<a.numOperands;u++){let p=a.getOperand(u).definingOp;p&&!s.has(p)&&(s.add(p),o.push(p))}}return!1}_merge(e,t,n){let s=e.regions[0].entryBlock,o=t.regions[0].entryBlock,i=bl(s),a=bl(o);if(!i||!a)return;let u=new Map;for(let[N,v]of n)u.set(v,N);let p=[],c=new Map;for(let N=0;N<e.numOperands;N++)c.set(N,p.length),p.push(e.getOperand(N));let h=new Map;for(let N=0;N<t.numOperands;N++){if(u.has(N))continue;let v=t.getOperand(N),O=-1;for(let R=0;R<p.length;R++)if(p[R]===v){O=R;break}O>=0?h.set(N,O):(h.set(N,p.length),p.push(v))}let f=p.map(N=>N.type),d=new Ct,m=new yn(f);d.addBlock(m);let g=new Map;for(let N=0;N<s.arguments.length;N++)g.set(s.arguments[N],m.arguments[c.get(N)]);for(let N of s.ops())N.opName!=="yield"&&m.pushOp(N.clone(g));for(let N=0;N<i.numOperands;N++){let v=i.getOperand(N),O=g.get(v)||v;for(let[R,B]of u)B===N&&g.set(o.arguments[R],O)}for(let N=0;N<o.arguments.length;N++){if(u.has(N))continue;let v=h.get(N);v!==void 0&&g.set(o.arguments[N],m.arguments[v])}for(let N of o.ops())N.opName!=="yield"&&m.pushOp(N.clone(g));let y=[],_=[],b=[];for(let N=0;N<e.numResults;N++){let v=!1;for(let O of e.getResult(N).uses())if(O.user!==t){v=!0;break}if(v){let O=i.getOperand(N),R=g.get(O)||O;b.push({resultIdx:N,value:R}),y.push(R),_.push(e.getResult(N).type)}}for(let N=0;N<t.numResults;N++){let v=a.getOperand(N),O=g.get(v)||v;y.push(O),_.push(t.getResult(N).type)}let w=new Re("yield",y,[]);m.pushOp(w);let x=[];for(let N of m.ops())N.opName!=="yield"&&x.push(N);let S=_l(x),T=new Re("fusion",p,_,{fusion_kind:S},[d]),k=t;if(!k.parentBlock)return;k.parentBlock.insertBefore(T,k);let A=0;for(let{resultIdx:N}of b)e.getResult(N).replaceAllUsesWith(T.getResult(A)),A++;for(let N=0;N<t.numResults;N++)t.getResult(N).replaceAllUsesWith(T.getResult(A)),A++;e.dropAllOperands(),e.parentBlock&&e.parentBlock.removeOp(e),t.dropAllOperands(),t.parentBlock&&t.parentBlock.removeOp(t)}_estimateRecomputeCost(e){let t=0,n=e.regions[0]?.entryBlock;if(!n)return 0;for(let s of n.ops()){if(s.opName==="yield")continue;let o=ne.get(s.opName);if(o&&o.getFlops)t+=o.getFlops(s);else for(let i=0;i<s.numResults;i++){let a=s.getResult(i).type;if(a instanceof I){let u=a.numel();u!==X&&(t+=u);break}}}return t}}});function E$(r,e){let t=Math.min(r.id,e.id),n=Math.max(r.id,e.id);return`${t}|${n}`}var rf,eB=M(()=>{"use strict";nn();sr();zi();Oe();Oa();Kt();kl();rf=class extends Ge{static{l(this,"MultiOutputFusionPass")}constructor(e={}){super("MultiOutputFusionPass"),this.maxFusionSize=e.maxFusionSize||512,this.maxReductions=e.maxReductions||1,this.minSharedInputBytes=e.minSharedInputBytes||0,this.maxOutputs=e.maxOutputs||8,this.maxConsumersPerInput=e.maxConsumersPerInput||64}run(e){let t=[];for(let a of e.ops())a.opName==="fusion"&&t.push(a);if(t.length<2)return oe.UNCHANGED;let n=this._findCandidates(t);if(n.length===0)return oe.UNCHANGED;let s=!1,o=0,i=new Set;for(let{left:a,right:u,sharedInputs:p,sharedBytes:c}of n)i.has(a)||i.has(u)||this._canMerge(a,u)&&(this._mergeCreatesCycle(a,u)||(this._mergeMultiOutput(a,u,p),i.add(a),i.add(u),s=!0,o++));return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,fusionOps:t.length,candidates:n.length,mergeCount:o,level:ue.DEBUG}),s?oe.CHANGED:oe.UNCHANGED}_findCandidates(e){let t=new Map;for(let o of e)for(let i=0;i<o.numOperands;i++){let u=o.getOperand(i).id,p=t.get(u);p||(p=[],t.set(u,p)),p.push(o)}let n=new Map;for(let[o,i]of t){if(i.length<2)continue;let a=Math.min(i.length,this.maxConsumersPerInput);for(let u=0;u<a;u++)for(let p=u+1;p<a;p++){let c=i[u],h=i[p];if(c===h)continue;let f=E$(c,h),d=n.get(f);d||(d={left:c,right:h,sharedInputs:new Set,sharedBytes:0},n.set(f,d)),d.sharedInputs.add(o)}}let s=[];for(let o of n.values()){let i=0;for(let a of o.sharedInputs){let u=t.get(a);if(!u||u.length===0)continue;let p=u[0];for(let c=0;c<p.numOperands;c++){let h=p.getOperand(c);if(h.id===a&&h.type instanceof I){let f=h.type.sizeInBytes();f!==X&&(i+=f);break}}}o.sharedBytes=i,i>=this.minSharedInputBytes&&s.push(o)}return s.sort((o,i)=>i.sharedBytes-o.sharedBytes),s}_canMerge(e,t){if(this._hasProducerConsumerEdge(e,t)||this._hasProducerConsumerEdge(t,e))return!1;let n=wl(e),s=wl(t);if(n+s>this.maxFusionSize||e.numResults+t.numResults>this.maxOutputs)return!1;let i=0;return i+=Tl(e),i+=Tl(t),!(i>this.maxReductions||!xl(e)||!xl(t))}_mergeCreatesCycle(e,t){let n=new Set,s=[];for(let i=0;i<e.numResults;i++){let a=e.getResult(i);for(let u of a.uses())u.user!==t&&!n.has(u.user)&&(n.add(u.user),s.push(u.user))}let o=0;for(;o<s.length;){let i=s[o++];if(i===t)return!0;for(let a=0;a<i.numResults;a++)for(let u of i.getResult(a).uses())n.has(u.user)||(n.add(u.user),s.push(u.user))}n.clear(),s.length=0;for(let i=0;i<t.numResults;i++){let a=t.getResult(i);for(let u of a.uses())u.user!==e&&!n.has(u.user)&&(n.add(u.user),s.push(u.user))}for(o=0;o<s.length;){let i=s[o++];if(i===e)return!0;for(let a=0;a<i.numResults;a++)for(let u of i.getResult(a).uses())n.has(u.user)||(n.add(u.user),s.push(u.user))}return!1}_mergeMultiOutput(e,t,n){let s=e.regions[0].entryBlock,o=t.regions[0].entryBlock,i=bl(s),a=bl(o);if(!i||!a)return;let u=[],p=new Map,c=new Map;for(let v=0;v<e.numOperands;v++){let O=e.getOperand(v),R=O.id,B=p.get(R);B===void 0&&(B=u.length,p.set(R,B),u.push(O)),c.set(v,B)}let h=new Map;for(let v=0;v<t.numOperands;v++){let O=t.getOperand(v),R=O.id,B=p.get(R);B===void 0&&(B=u.length,p.set(R,B),u.push(O)),h.set(v,B)}let f=u.map(v=>v.type),d=new Ct,m=new yn(f);d.addBlock(m);let g=new Map;for(let v=0;v<s.arguments.length;v++)g.set(s.arguments[v],m.arguments[c.get(v)]);for(let v of s.ops())v.opName!=="yield"&&m.pushOp(v.clone(g));for(let v=0;v<o.arguments.length;v++)g.set(o.arguments[v],m.arguments[h.get(v)]);for(let v of o.ops())v.opName!=="yield"&&m.pushOp(v.clone(g));let y=[],_=[];for(let v=0;v<i.numOperands;v++){let O=i.getOperand(v);y.push(g.get(O)||O),_.push(e.getResult(v).type)}for(let v=0;v<a.numOperands;v++){let O=a.getOperand(v);y.push(g.get(O)||O),_.push(t.getResult(v).type)}let b=new Re("yield",y,[]);m.pushOp(b);let w=[];for(let v of m.ops())v.opName!=="yield"&&w.push(v);let x=_l(w),S=new Re("fusion",u,_,{fusion_kind:x},[d]),T=e.parentBlock;if(!T)return;let k=dT(T),A=null;for(let v of u){let O=v.definingOp;!O||O===e||O===t||(!A||!k(O,A))&&(A=O)}A&&A.parentBlock===T?T.insertAfter(S,A):T.insertBefore(S,e);for(let v=0;v<e.numResults;v++)e.getResult(v).replaceAllUsesWith(S.getResult(v));let N=e.numResults;for(let v=0;v<t.numResults;v++)t.getResult(v).replaceAllUsesWith(S.getResult(N+v));e.dropAllOperands(),e.parentBlock&&e.parentBlock.removeOp(e),t.dropAllOperands(),t.parentBlock&&t.parentBlock.removeOp(t)}_hasProducerConsumerEdge(e,t){for(let n=0;n<e.numResults;n++)for(let s of e.getResult(n).uses())if(s.user===t)return!0;return!1}};l(E$,"pairKey")});var mT,at,Es=M(()=>{"use strict";mT=class{static{l(this,"UseDefResult")}constructor(e,t,n,s,o){this.topologicalOrder=e,this.valueToOp=t,this.opUsers=n,this.depth=s,this.height=o}},at=class{static get name(){return"use_def"}static get depKey(){return"useDef"}static get dependencies(){return[]}static compute(e){let t=[],n=new Map,s=new Map,o=new Set,i=new Set,a=l(c=>{if(o.has(c))return;i.add(c);let h=[{op:c,i:0}];for(;h.length>0;){let f=h[h.length-1],d=f.op;if(f.i<d.numOperands){let m=d.getOperand(f.i).definingOp;if(f.i++,m&&!o.has(m)){if(i.has(m))throw new Error("Cycle detected in UseDefAnalysis");i.add(m),h.push({op:m,i:0})}continue}i.delete(d),o.add(d),t.push(d),h.pop()}},"visit");for(let c of e.ops()){for(let h=0;h<c.numResults;h++)n.set(c.getResult(h),c);s.set(c,new Set)}let u=new Map,p=new Map;for(let c of e.ops())a(c);for(let c of t){let h=0;for(let f=0;f<c.numOperands;f++){let d=c.getOperand(f).definingOp;if(d){s.get(d).add(c);let m=u.get(d)||0;m+1>h&&(h=m+1)}}u.set(c,h)}for(let c=t.length-1;c>=0;c--){let h=t[c],f=0,d=s.get(h)||new Set;for(let m of d){let g=p.get(m)||0;g+1>f&&(f=g+1)}p.set(h,f)}return new mT(t,n,s,u,p)}}});var gT,ty,tB=M(()=>{"use strict";Es();Tr();gT=class{static{l(this,"DominanceResult")}constructor(e){this.idom=e}postDominates(e,t){let n=t;for(;n;){if(n===e)return!0;n=this.idom.get(n)}return!1}immediatePDom(e){return this.idom.get(e)||null}pathToPDom(e){let t=[],n=this.idom.get(e);for(;n&&(t.push(n),n!==this.idom.get(n));)n=this.idom.get(n);return t}},ty=class{static get name(){return"post_dominance"}static get depKey(){return"postDominance"}static get dependencies(){return[at]}static compute(e,t={}){let s=(t.useDef||at.compute(e)).topologicalOrder,o=new Map;for(let m of s)o.set(m,[]);for(let m of s)for(let g=0;g<m.numResults;g++){let y=m.getResult(g);for(let _ of y.uses()){let b=_.user;o.has(b)&&o.get(m).push(b)}}let i=new Set;for(let m of s)qt(m.opName)&&i.add(m);let a={opName:"__pdom_root__"},u=Math.max(1,Math.ceil(Math.log2(s.length+2))+1),p=new Map,c=new Map,h=new Map;c.set(a,0),h.set(a,new Array(u).fill(a));let f=l((m,g)=>{p.set(m,g),c.set(m,c.get(g)+1);let y=new Array(u);y[0]=g;for(let _=1;_<u;_++)y[_]=h.get(y[_-1])[_-1];h.set(m,y)},"link"),d=l((m,g)=>{if(c.get(m)<c.get(g)){let _=m;m=g,g=_}let y=c.get(m)-c.get(g);for(let _=0;_<u;_++)y>>_&1&&(m=h.get(m)[_]);if(m===g)return m;for(let _=u-1;_>=0;_--)h.get(m)[_]!==h.get(g)[_]&&(m=h.get(m)[_],g=h.get(g)[_]);return h.get(m)[0]},"lca");for(let m=s.length-1;m>=0;m--){let g=s[m];if(i.has(g)){f(g,a);continue}let y=null;for(let _ of o.get(g))p.has(_)&&(y=y===null?_:d(y,_));y!==null&&f(g,y)}for(let[m,g]of p)g===a&&p.delete(m);return new gT(p,s)}}});function nB(r){return qt(r)||ln(r)}var ny,rB=M(()=>{"use strict";nn();At();Oa();Yg();Jg();tB();Kt();Es();kl();Tr();l(nB,"isSkipOp");ny=class extends Ge{static{l(this,"DominatorFusionPass")}constructor(e={}){super("DominatorFusionPass"),this.requiredAnalyses=[at];let t=e.target||{};this.maxFusionSize=t.maxFusionSize||e.maxFusionSize||512,this.maxReductions=e.maxReductions||1,this.libraryOps=t.libraryOps||e.libraryOps||new Set,this.costModel=new bi({memoryBandwidthGBs:t.memoryBandwidthGBs,computeTFLOPs:t.computeTFLOPs,maxRegistersPerThread:t.registersPerThread,maxSharedMemory:t.sharedMemoryBytes,libraryOps:t.libraryOps,...e.cost})}run(e,t){let n=t?t.getAnalysis(at,e):at.compute(e),s=ty.compute(e,{useDef:n}),o=n.topologicalOrder,i=this._buildGroups(o,s);if(i.length===0)return oe.UNCHANGED;let a=[];for(let u of i){if(!u.allOpsInlineFusable()){this._explain(u,!1,"group contains ops without inline fusion support");continue}let p=this.costModel.shouldFuse(u);this._explain(u,p.fuse,p.reason),p.fuse&&a.push(u)}if(a.length===0)return oe.UNCHANGED;for(let u of a)Sl(u,je.ELEMENTWISE);return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,groupsBuilt:i.length,groupsFused:a.length,level:ue.DEBUG}),oe.CHANGED}_explain(e,t,n){if(!this.trace||!this.trace.explainsEnabled)return;let s=e.ops.map(o=>o.opName);this.trace.explain("fusion",s.join("+"),t?"fused":"not-fused",n||null,{groupSize:s.length,strategy:"dominator"})}_buildGroups(e,t){let n=new Map,s=[],o=0;for(let p=e.length-1;p>=0;p--){let c=e[p];if(nB(c.opName))continue;let h=Aa(c);if(h===je.OPAQUE||this.libraryOps.has(c.opName))continue;let f=t.immediatePDom(c);if(!f||nB(f.opName))continue;let d=Aa(f);if(d===je.OPAQUE||this.libraryOps.has(f.opName)||!this._canFusePatterns(h,d)||!this._pathAllFusable(c,f,e,t))continue;let m=n.get(f);if(m){if(m.size+1>this.maxFusionSize||!this._checkReductionLimit(m,c))continue;m.addOp(c),n.set(c,m),this._absorbIntermediates(c,f,m,n,e,t)}else{let g=n.get(c);if(g){if(g.size+1>this.maxFusionSize||!this._checkReductionLimit(g,f))continue;g.addOp(f),n.set(f,g),this._absorbIntermediates(c,f,g,n,e,t)}else{let y=new Eo(o++);if(y.addOp(c),y.addOp(f),!this._checkGroupReductions(y))continue;n.set(c,y),n.set(f,y),s.push(y),this._absorbIntermediates(c,f,y,n,e,t)}}}let i=new Set,a=[];for(let p of s)i.has(p)||p.size<2||(i.add(p),p.classifyKind(),a.push(p));for(let p of n.values())i.has(p)||p.size<2||(i.add(p),p.classifyKind(),a.push(p));let u=[];for(let p of a){if(!this._checkGroupReductions(p)){this._explain(p,!1,`group exceeds the ${this.maxReductions}-reduction limit`);continue}if(this._createsCycle(p)){this._explain(p,!1,"fusing would create a dependency cycle");continue}u.push(p)}return u}_createsCycle(e){let t=new Set;for(let o of e.getInputValues())o.definingOp&&!e.hasOp(o.definingOp)&&t.add(o.definingOp);if(t.size===0)return!1;let n=new Set,s=[];for(let o of e.getOutputValues())for(let i of o.uses())e.hasOp(i.user)||s.push(i.user);for(;s.length>0;){let o=s.pop();if(!n.has(o)){if(n.add(o),t.has(o))return!0;for(let i=0;i<o.numResults;i++)for(let a of o.getResult(i).uses())e.hasOp(a.user)||s.push(a.user)}}return!1}_canFusePatterns(e,t){return fT(e,t)}_pathAllFusable(e,t,n,s){let o=new Set,i=[e];for(o.add(e),o.add(t);i.length>0;){let a=i.pop();for(let u=0;u<a.numResults;u++)for(let p of a.getResult(u).uses()){let c=p.user;if(c===t||o.has(c)||!s.postDominates(t,c))continue;if(o.add(c),Aa(c)===je.OPAQUE||this.libraryOps.has(c.opName))return!1;i.push(c)}}return!0}_absorbIntermediates(e,t,n,s,o,i){let a=new Set,u=[e];for(a.add(e),a.add(t);u.length>0;){let p=u.pop();for(let c=0;c<p.numResults;c++)for(let h of p.getResult(c).uses()){let f=h.user;f===t||a.has(f)||!i.postDominates(t,f)||(a.add(f),Aa(f)===je.OPAQUE||this.libraryOps.has(f.opName))||(n.size<this.maxFusionSize&&this._checkReductionLimit(n,f)&&(n.addOp(f),s.set(f,n)),u.push(f))}}}_checkGroupReductions(e){let t=0;for(let n of e.ops){let s=ne.get(n.opName);s&&s.isReduction&&t++}return t<=this.maxReductions}_checkReductionLimit(e,t){let n=ne.get(t.opName);if(!n||!n.isReduction)return!0;let s=0;for(let o of e.ops){let i=ne.get(o.opName);i&&i.isReduction&&s++}return s<this.maxReductions}}});var ry,sB=M(()=>{"use strict";ry=class{static{l(this,"GraphCycles")}constructor(e,t){this._n=e,this._parent=new Int32Array(e),this._rank=new Int32Array(e),this._nodeAtRank=new Int32Array(e),this._out=new Array(e),this._in=new Array(e);for(let n=0;n<e;n++)this._parent[n]=n,this._rank[n]=n,this._nodeAtRank[n]=n,this._out[n]=new Set,this._in[n]=new Set;if(t)for(let[n,s]of t)n!==s&&(this._out[n].add(s),this._in[s].add(n))}find(e){let t=this._parent,n=e;for(;t[n]!==n;)n=t[n];for(;t[e]!==n;){let s=t[e];t[e]=n,e=s}return n}wouldCreateCycle(e,t){let n=this.find(e),s=this.find(t);if(n===s)return!1;let o=this._rank[n]<this._rank[s]?n:s,i=o===n?s:n;return this._hasIntermediatePath(o,i)}_hasIntermediatePath(e,t){let n=this._rank[t],s=new Set([e]),o=[e];for(;o.length>0;){let i=o.pop();for(let a of this._out[i]){let u=this.find(a);if(!(u===i||u===e)){if(u===t){if(i!==e)return!0;continue}this._rank[u]>=n||s.has(u)||(s.add(u),o.push(u))}}}return!1}merge(e,t){let n=this.find(e),s=this.find(t);if(n===s)return n;let o=this._rank[n]<this._rank[s]?n:s,i=o===n?s:n,a=this._rank[o],u=this._rank[i],p=this._out[n].size+this._in[n].size,c=this._out[s].size+this._in[s].size,h=p>=c?n:s,f=h===n?s:n,d=this._out[h],m=this._in[h];for(let g of this._in[f]){let y=this.find(g);y===h||y===f||(this._out[y].delete(f),this._out[y].add(h),m.add(y))}for(let g of this._out[f]){let y=this.find(g);y===h||y===f||(this._in[y].delete(f),this._in[y].add(h),d.add(y))}return d.delete(f),m.delete(f),d.delete(h),m.delete(h),this._parent[f]=h,this._nodeAtRank[this._rank[f]]=-1,this._reorder(a,u),h}_reorder(e,t){let n=[],s=[];for(let c=e;c<=t;c++){let h=this._nodeAtRank[c];if(!(h<0)){if(this.find(h)!==h){this._nodeAtRank[c]=-1;continue}n.push(c),s.push(h)}}if(s.length<=1){for(let c=0;c<s.length;c++)this._rank[s[c]]=n[c],this._nodeAtRank[n[c]]=s[c];return}let o=new Set(s),i=new Map;for(let c of s)i.set(c,0);for(let c of s)for(let h of this._out[c]){let f=this.find(h);f!==c&&o.has(f)&&i.set(f,i.get(f)+1)}let a=[];for(let c of s)i.get(c)===0&&a.push(c);let u=[],p=0;for(;p<a.length;){let c=a[p++];u.push(c);for(let h of this._out[c]){let f=this.find(h);if(f!==c&&o.has(f)){let d=i.get(f)-1;i.set(f,d),d===0&&a.push(f)}}}for(let c=0;c<u.length;c++){let h=u[c],f=n[c];this._rank[h]=f,this._nodeAtRank[f]=h}}}});var sy,oB=M(()=>{"use strict";sy=class{static{l(this,"MaxHeap")}constructor(){this._items=[]}get size(){return this._items.length}isEmpty(){return this._items.length===0}push(e,t){let n=this._items;n.push({priority:e,value:t});let s=n.length-1;for(;s>0;){let o=s-1>>1;if(n[o].priority>=n[s].priority)break;let i=n[o];n[o]=n[s],n[s]=i,s=o}}pop(){let e=this._items,t=e.length;if(t===0)return;let n=e[0],s=e.pop();if(t>1){e[0]=s;let o=e.length,i=0;for(;;){let a=i*2+1,u=a+1,p=i;if(a<o&&e[a].priority>e[p].priority&&(p=a),u<o&&e[u].priority>e[p].priority&&(p=u),p===i)break;let c=e[p];e[p]=e[i],e[i]=c,i=p}}return n.value}}});function L$(r){let e=ne.get(r.opName);return!e||e.isConstant||e.isTerminator||e.isOpaque?!1:e.isReduction?!0:ko(r.opName)}var oy,iB=M(()=>{"use strict";nn();At();Oe();Oa();Yg();Jg();Es();sB();oB();kl();Ui();Kt();l(L$,"isFusibleOp");oy=class extends Ge{static{l(this,"PriorityFusionPass")}constructor(e={}){super("PriorityFusionPass"),this.requiredAnalyses=[at];let t=e.target||{};this.maxReductions=e.maxReductions??1,this.legality=new yl({maxFusionSize:t.maxFusionSize||e.maxFusionSize,maxSharedMemory:t.sharedMemoryBytes||e.maxSharedMemory,libraryOps:t.libraryOps||e.libraryOps,allowReductionFusion:e.allowReductionFusion});let n=t.getAttr&&t.getAttr("fusionBenefitWeights")||e.benefitWeights;this.costModel=new bi({memoryBandwidthGBs:t.memoryBandwidthGBs,computeTFLOPs:t.computeTFLOPs,maxRegistersPerThread:t.registersPerThread,maxSharedMemory:t.sharedMemoryBytes,libraryOps:t.libraryOps,policy:t.getAttr?t.getAttr("fusionPolicy"):null,benefitWeights:n,...e.cost})}run(e,t){let s=(t?t.getAnalysis(at,e):at.compute(e)).topologicalOrder,o=s.length;if(o===0)return oe.UNCHANGED;let i=new Map;for(let T=0;T<o;T++)i.set(s[T],T);let a=[];for(let T=0;T<o;T++){let k=s[T];for(let A=0;A<k.numOperands;A++){let N=k.getOperand(A).definingOp;N&&i.has(N)&&a.push([i.get(N),T])}}let u=new ry(o,a),p=new Int32Array(o),c=new Map;for(let T=0;T<o;T++)if(L$(s[T])){let k=new Eo(T);k.addOp(s[T]),c.set(T,k)}let h=new sy,f=l((T,k)=>{let A=new Eo(-1);for(let N of T.ops)A.addOp(N);for(let N of k.ops)A.addOp(N);return A},"mergedGroup"),d=l(T=>{if(T.size<2||!T.allOpsInlineFusable())return!1;let k=0;for(let A of T.ops){let N=ne.get(A.opName);N&&N.isReduction&&k++}return k>this.maxReductions||(T.classifyKind(),T.kind===je.OPAQUE)?!1:this.costModel.shouldFuse(T).fuse===!0},"legalMerge"),m=l((T,k)=>{let A=0;for(let N=0;N<k.numOperands;N++){let v=k.getOperand(N);if(v.definingOp===T&&v.type instanceof I){let O=v.type.sizeInBytes();O!==X&&(A+=O)}}return A},"edgeBytes"),g=l((T,k)=>{let A=u.find(i.get(T)),N=u.find(i.get(k));if(A===N||!c.has(A)||!c.has(N)||!this.legality.canFuse(T,k).legal)return;let v=this.costModel.edgeBenefit(m(T,k));h.push(v,{a:A,b:N,va:p[A],vb:p[N],prodOp:T,consOp:k})},"pushCandidate"),y=l((T,k)=>{let A=new Set;for(let N of T.ops){for(let v=0;v<N.numResults;v++)for(let O of N.getResult(v).uses()){let R=i.get(O.user);if(R===void 0)continue;let B=u.find(R);if(B===k||!c.has(B))continue;let E=k*o+B;A.has(E)||(A.add(E),g(N,O.user))}for(let v=0;v<N.numOperands;v++){let O=N.getOperand(v).definingOp;if(!O)continue;let R=i.get(O);if(R===void 0)continue;let B=u.find(R);if(B===k||!c.has(B))continue;let E=B*o+k;A.has(E)||(A.add(E),g(O,N))}}},"reEval"),_=new Set;for(let T=0;T<o;T++){if(!c.has(T))continue;let k=s[T];for(let A=0;A<k.numOperands;A++){let N=k.getOperand(A).definingOp;if(!N)continue;let v=i.get(N);if(v===void 0||!c.has(v))continue;let O=v*o+T;_.has(O)||(_.add(O),g(N,k))}}let b=0;for(;!h.isEmpty();){let T=h.pop(),k=u.find(T.a),A=u.find(T.b);if(k===A||p[k]!==T.va||p[A]!==T.vb)continue;let N=c.get(k),v=c.get(A);if(!N||!v||!this.legality.canFuse(T.prodOp,T.consOp).legal)continue;let O=f(N,v);if(!d(O)||u.wouldCreateCycle(k,A))continue;let R=u.merge(k,A),B=N.size>=v.size?N:v,E=B===N?v:N;B.merge(E),c.delete(k),c.delete(A),c.set(R,B),p[R]=(p[k]>p[A]?p[k]:p[A])+1,b++,y(B,R)}if(b===0)return oe.UNCHANGED;let w=[],x=new Set;for(let T of c.values())x.has(T)||T.size<2||(x.add(T),w.push(T));let S=0;for(let T of w)T.classifyKind(),this._explain(T),Sl(T,je.ELEMENTWISE)&&S++;return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,ops:o,merges:b,fusions:S,level:ue.DEBUG}),S>0?oe.CHANGED:oe.UNCHANGED}_explain(e){if(!this.trace||!this.trace.explainsEnabled)return;let t=e.ops.map(n=>n.opName);this.trace.explain("fusion",t.join("+"),"fused",null,{groupSize:t.length,strategy:"priority"})}}});function C$(r,e){return r===e?!0:!r||!e?!1:typeof r.equals=="function"?r.equals(e):!1}var wi,iy,aB=M(()=>{"use strict";Oe();wi=class{static{l(this,"LayoutPreference")}constructor(e,t,n=0){this.inputs=e,this.outputs=t,this.cost=n}},iy=class{static{l(this,"LayoutPolicy")}constructor(e){this.target=e,this._rules=new Map,this._initDefaultRules()}registerRule(e,t){this._rules.set(e,t)}getPreference(e){let t=this._rules.get(e.opName);return t?t(e,this.target):null}estimateConversionCost(e,t,n){if(!(n instanceof I)||C$(e,t))return 0;let s=n.numel();return s<0?1024:s*2}estimateBenefit(e,t,n){if(!(t instanceof I))return 0;let s=t.numel();if(s<0)return 0;let o=e.opName;if(o==="dot"||o==="conv"||o==="matmul")return s*4*n;if(o==="reduce")return s*2*n;let i=this.target.cacheLineBytes||64;return s*4<=i*4?0:Math.floor(s*.5)}_initDefaultRules(){this._rules.set("conv",(e,t)=>{let s=e.getOperand(0).type?.rank||4;if(t.preferredConvLayout)return new wi([t.preferredConvLayout,null],[t.preferredConvLayout]);if(t.isGPU()&&s===4){let o=new Et([0,2,3,1]);return new wi([o,null],[o])}if(t.isCPU()&&s===4){let o=new Et([0,2,3,1]);return new wi([o,null],[o])}return null}),this._rules.set("dot",(e,t)=>{let n=e.getOperand(0).type,s=e.getOperand(1).type;if(!n||!s)return null;let o=Et.rowMajor(n.rank);if(t.isCPU()&&s.rank===2){let a=Et.columnMajor(s.rank);return new wi([o,a],[o])}let i=Et.rowMajor(s.rank);return new wi([o,i],[o])}),this._rules.set("reduce",(e,t)=>{let n=e.getResult(0).type;return n?new wi([null],[Et.rowMajor(n.rank)]):null})}};l(C$,"layoutEquals")});function uB(r,e){let t=new Map,n=null,s=0;for(let i=0;i<r.numOperands;i++){let a=r.getOperand(i),u=e.get(a);if(!u)continue;let p=u.hash?u.hash():0,c=(t.get(p)||0)+1;t.set(p,c),c>s&&(s=c,n=u)}if(n)return n;let o=r.getResult(0);return o&&o.type instanceof I?Et.rowMajor(o.type.rank):Et.rowMajor(1)}function D$(r,e){return r===e?!0:!r||!e?!1:typeof r.equals=="function"?r.equals(e):!1}function lB(r){if(r instanceof Et)return r;if(r&&typeof r.toLayout=="function")try{return r.toLayout()}catch{}return r&&r.baseOrder&&!r.isBlocked?.()?new Et(r.baseOrder):r&&r.order?new Et(r.order):null}var yT,ay,cB=M(()=>{"use strict";Es();Oe();At();_n();Tr();yT=class{static{l(this,"LayoutAnalysisResult")}constructor(e,t,n){this.assignments=e,this.conversions=t,this.totalCost=n}},ay=class{static get name(){return"layout"}static get depKey(){return"layout"}static get dependencies(){return[at]}static compute(e,t,n){let s=t.useDef,o=new Map,i=s.topologicalOrder;for(let p of e.args)p.type instanceof I&&o.set(p,p.type.layout||Et.rowMajor(p.type.rank));for(let p=0;p<i.length;p++){let c=i[p];if(qt(c.opName))continue;let h=n?n.getPreference(c):null;if(h&&h.outputs.length>0){for(let m=0;m<c.numResults;m++){let g=c.getResult(m);if(!(g.type instanceof I))continue;let y=h.outputs[m]||null;y?o.set(g,y):o.set(g,uB(c,o))}continue}let f=ne.get(c.opName),d=f&&f.hasTrait(H.ELEMENTWISE);for(let m=0;m<c.numResults;m++){let g=c.getResult(m);g.type instanceof I&&(d?o.set(g,uB(c,o)):o.set(g,Et.rowMajor(g.type.rank)))}}let a=[],u=0;for(let p=0;p<i.length;p++){let c=i[p],h=n?n.getPreference(c):null;if(h)for(let f=0;f<c.numOperands;f++){let d=c.getOperand(f);if(!(d.type instanceof I))continue;let m=o.get(d);if(!m)continue;let g=h.inputs[f];if(!g||D$(m,g))continue;let y=lB(m),_=lB(g);!y||!_||y.equals(_)||(a.push({value:d,consumer:c,operandIdx:f,from:y,to:_}),n&&(u+=n.estimateConversionCost(y,_,d.type)))}}return new yT(o,a,u)}};l(uB,"resolveFromInputs");l(D$,"layoutEquals");l(lB,"toPlainLayout")});function P$(r,e,t){let n=r.id,s=e.hash?e.hash():0,o=t.hash?t.hash():0;return`${n}:${s}:${o}`}var uy,pB=M(()=>{"use strict";nn();sr();Oe();aB();cB();Es();Kt();uy=class extends Ge{static{l(this,"LayoutTransformPass")}constructor(e={}){super("LayoutTransformPass"),this.target=e.target||null,this._policy=null}run(e,t){if(!this.target)return oe.UNCHANGED;this._policy||(this._policy=new iy(this.target));let n=t?t.getAnalysis(at,e):at.compute(e),s=ay.compute(e,{useDef:n},this._policy);if(s.conversions.length===0)return oe.UNCHANGED;let o=new Map;for(let p of s.conversions){let{value:c,consumer:h,operandIdx:f,from:d,to:m}=p,g=P$(c,d,m),y=o.get(g);y||(y={value:c,from:d,to:m,consumers:[],cost:this._policy.estimateConversionCost(d,m,c.type),benefit:0},o.set(g,y)),y.consumers.push({consumer:h,operandIdx:f});let _=this.target.layoutAwareOps&&this.target.layoutAwareOps.has(h.opName);y.benefit+=_?this._policy.estimateBenefit(h,c.type,1):0}let i=0,a=0,u=[];for(let p of o.values())p.benefit<p.cost||(u.push(p),i+=p.cost,a+=p.benefit);if(u.length===0||i>a)return oe.UNCHANGED;for(let p of u){let c=p.from instanceof Et?p.from.order:Array.from({length:p.value.type.rank},(y,_)=>_),h=p.to instanceof Et?p.to.order:Array.from({length:p.value.type.rank},(y,_)=>_),f=new I(p.value.type.shape,p.value.type.dtype,p.to),d=new Re("layout_transform",[p.value],[f],{src_layout:[...c],dst_layout:[...h]}),m=p.value.definingOp;m&&m.parentBlock?m.parentBlock.insertAfter(d,m):p.consumers[0].consumer.parentBlock&&p.consumers[0].consumer.parentBlock.insertBefore(d,p.consumers[0].consumer);let g=d.getResult(0);for(let y of p.consumers)y.consumer.replaceOperand(y.operandIdx,g)}return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,conversions:s.conversions.length,uniqueTransforms:u.length,level:ue.DEBUG}),oe.CHANGED}};l(P$,"valueLayoutKey")});function fB(r,e){if(r===e)return!0;if(typeof r=="number"&&typeof e=="number")return r===e;if(typeof r!=typeof e||r.length!==e.length)return!1;for(let t=0;t<r.length;t++)if(r[t]!==e[t])return!1;return!0}function hB(r){let e=0;for(let t=0;t<r.length;t++)e=(e<<5)-e+r.charCodeAt(t)&2147483647;return e}function z$(r){if(typeof r=="number"){let t=new Float64Array([r]),n=new Uint32Array(t.buffer);return(n[0]^n[1])&2147483647}let e=r.length;for(let t=0;t<Math.min(r.length,8);t++){let n=new Float64Array([r[t]]),s=new Uint32Array(n.buffer);e=(e^(s[0]^s[1]))*16777619&2147483647}return e}var kr,$$,F$,hr,_T=M(()=>{"use strict";Oe();kr=Object.freeze({PER_TENSOR_SYMMETRIC:"per_tensor_symmetric",PER_TENSOR_ASYMMETRIC:"per_tensor_asymmetric",PER_CHANNEL:"per_channel",PER_GROUP:"per_group"}),$$=new Set(Object.values(kr)),F$=new Set(["i8","ui8"]),hr=class r{static{l(this,"QuantizationParams")}constructor(e){this.scheme=e.scheme,this.scale=e.scale,this.zeroPoint=e.zeroPoint,this.axis=e.axis??null,this.groupSize=e.groupSize??null,this.dtype=e.dtype||"i8",this.numBits=e.numBits||Gs(this.dtype)*8,this._hash=null}clampRange(){if(this.isSymmetric()){let n=2**(this.numBits-1)-1;return[-n,n]}if(this.dtype==="ui8")return[0,2**this.numBits-1];let e=-(2**(this.numBits-1)),t=2**(this.numBits-1)-1;return[e,t]}quantize(e){let[t,n]=this.clampRange(),s=Math.round(e/this.getScalarScale()+this.getScalarZeroPoint());return Math.max(t,Math.min(n,s))}dequantize(e){return(e-this.getScalarZeroPoint())*this.getScalarScale()}quantizeArray(e){let[t,n]=this.clampRange(),s=new Array(e.length);if(this.isPerChannel())throw new Error("Use quantizeArrayPerChannel(floatArr, shape) for per-channel quantization");let o=this.getScalarScale(),i=this.getScalarZeroPoint();for(let a=0;a<e.length;a++)s[a]=Math.max(t,Math.min(n,Math.round(e[a]/o+i)));return s}dequantizeArray(e){let t=new Array(e.length);if(this.isPerChannel())throw new Error("Use dequantizeArrayPerChannel(intArr, shape) for per-channel dequantization");let n=this.getScalarScale(),s=this.getScalarZeroPoint();for(let o=0;o<e.length;o++)t[o]=(e[o]-s)*n;return t}_channelStride(e){let t=1;for(let n=this.axis+1;n<e.length;n++)t*=e[n];return t}quantizeArrayPerChannel(e,t){let[n,s]=this.clampRange(),o=this._channelStride(t),i=t[this.axis],a=new Array(e.length);for(let u=0;u<e.length;u++){let p=Math.floor(u/o)%i,c=this.getScaleForChannel(p),h=this.getZeroPointForChannel(p);a[u]=Math.max(n,Math.min(s,Math.round(e[u]/c+h)))}return a}dequantizeArrayPerChannel(e,t){let n=this._channelStride(t),s=t[this.axis],o=new Array(e.length);for(let i=0;i<e.length;i++){let a=Math.floor(i/n)%s;o[i]=(e[i]-this.getZeroPointForChannel(a))*this.getScaleForChannel(a)}return o}quantizeArrayPerGroup(e){let[t,n]=this.clampRange(),s=new Array(e.length);for(let o=0;o<e.length;o++){let i=Math.floor(o/this.groupSize),a=this.scale[i],u=this.zeroPoint[i];s[o]=Math.max(t,Math.min(n,Math.round(e[o]/a+u)))}return s}dequantizeArrayPerGroup(e){let t=new Array(e.length);for(let n=0;n<e.length;n++){let s=Math.floor(n/this.groupSize);t[n]=(e[n]-this.zeroPoint[s])*this.scale[s]}return t}getScaleForGroup(e){return this.scale[e]}getZeroPointForGroup(e){return this.zeroPoint[e]}getScaleForChannel(e){return this.isPerChannel()?this.scale[e]:this.getScalarScale()}getZeroPointForChannel(e){return this.isPerChannel()?this.zeroPoint[e]:this.getScalarZeroPoint()}getScalarScale(){return typeof this.scale=="number"?this.scale:this.scale[0]}getScalarZeroPoint(){return typeof this.zeroPoint=="number"?this.zeroPoint:this.zeroPoint[0]}numChannels(){return this.isPerChannel()?typeof this.scale=="number"?1:this.scale.length:1}isPerChannel(){return this.scheme===kr.PER_CHANNEL}isPerGroup(){return this.scheme===kr.PER_GROUP}isSymmetric(){return this.scheme===kr.PER_TENSOR_SYMMETRIC}equals(e){return this===e?!0:!(!(e instanceof r)||this.scheme!==e.scheme||this.dtype!==e.dtype||this.numBits!==e.numBits||this.axis!==e.axis||this.groupSize!==e.groupSize||!fB(this.scale,e.scale)||!fB(this.zeroPoint,e.zeroPoint))}hash(){if(this._hash!==null)return this._hash;let e=2166136261;return e=(e^hB(this.scheme))*16777619&2147483647,e=(e^hB(this.dtype))*16777619&2147483647,e=(e^this.numBits)*16777619&2147483647,e=(e^z$(this.scale))*16777619&2147483647,this._hash=e,e}serialize(){let e=this.isPerChannel()||this.isPerGroup();return{scheme:this.scheme,scale:e?[...this.scale]:this.scale,zeroPoint:e?[...this.zeroPoint]:this.zeroPoint,axis:this.axis,groupSize:this.groupSize,dtype:this.dtype,numBits:this.numBits}}static deserialize(e){return new r(e)}static fromRange(e,t,n,s="i8",o=8){if(n===kr.PER_TENSOR_SYMMETRIC){let f=Math.max(Math.abs(e),Math.abs(t)),d=2**(o-1)-1,m=f/d||1e-10;return new r({scheme:n,scale:m,zeroPoint:0,dtype:s,numBits:o})}let[i,a]=s==="ui8"?[0,2**o-1]:[-(2**(o-1)),2**(o-1)-1],p=(t-e||1e-10)/(a-i),c=Math.round(i-e/p),h=Math.max(i,Math.min(a,c));return new r({scheme:n,scale:p,zeroPoint:h,dtype:s,numBits:o})}static fromRangePerChannel(e,t,n,s="i8",o=8){let i=e.length,a=new Float64Array(i),u=new Int32Array(i),p=2**(o-1)-1;for(let c=0;c<i;c++){let h=Math.max(Math.abs(e[c]),Math.abs(t[c]));a[c]=h/p||1e-10,u[c]=0}return new r({scheme:kr.PER_CHANNEL,scale:a,zeroPoint:u,axis:n,dtype:s,numBits:o})}static defaultForActivation(e,t="i8",n=8){return r.fromRange(-6,6,e,t,n)}static fromConstantArray(e,t,n="i8",s=8){let o=1/0,i=-1/0;for(let a=0;a<e.length;a++)e[a]<o&&(o=e[a]),e[a]>i&&(i=e[a]);return isFinite(o)||(o=-1),isFinite(i)||(i=1),o===i&&(o-=.5,i+=.5),r.fromRange(o,i,t,n,s)}static fromConstantArrayPerChannel(e,t,n,s="i8",o=8){let i=t[n],a=1;for(let c=n+1;c<t.length;c++)a*=t[c];let u=new Array(i).fill(1/0),p=new Array(i).fill(-1/0);for(let c=0;c<e.length;c++){let h=Math.floor(c/a)%i,f=e[c];f<u[h]&&(u[h]=f),f>p[h]&&(p[h]=f)}for(let c=0;c<i;c++)isFinite(u[c])||(u[c]=-1),isFinite(p[c])||(p[c]=1),u[c]===p[c]&&(u[c]-=.5,p[c]+=.5);return r.fromRangePerChannel(u,p,n,s,o)}static fromConstantArrayPerGroup(e,t,n="i8",s=4){let o=Math.ceil(e.length/t),i=new Float64Array(o),a=new Int32Array(o),u=2**(s-1)-1;for(let p=0;p<o;p++){let c=p*t,h=Math.min(c+t,e.length),f=0;for(let d=c;d<h;d++){let m=Math.abs(e[d]);m>f&&(f=m)}i[p]=f/u||1e-10,a[p]=0}return new r({scheme:kr.PER_GROUP,scale:i,zeroPoint:a,groupSize:t,dtype:n,numBits:s})}static isQuantizableDtype(e){return F$.has(e)}static isValidScheme(e){return $$.has(e)}};l(fB,"scaleEquals");l(hB,"hashStr");l(z$,"hashScaleValue")});function q$(r){for(let e=0;e<r.numOperands;e++){let t=r.getOperand(e).definingOp;if(t&&t.opName==="constant")return!0}return!1}function U$(r,e,t){for(let n=0;n<r.numOperands;n++){let s=r.getOperand(n);if(!e.has(s)&&(!(s.type instanceof I)||!Lt(s.type.dtype)))return!1}return r.numOperands>0}var V$,G$,j$,ly,cy,dB=M(()=>{"use strict";nn();sr();Oe();At();Es();_T();Kt();Tr();V$=new Set(["softmax","sqrt","div","rsqrt","log","exp","tanh"]),G$=new Set(["dot","conv","add","mul","sub"]),j$=new Map([["dot","quantized_dot"],["conv","quantized_conv"]]);for(let[r,e]of j$)ne.has(r)&&ne.registerOpAttr(r,"quantizedVariant",e);ly=class{static{l(this,"QuantizationConfig")}constructor(e={}){this.scheme=e.scheme||kr.PER_TENSOR_SYMMETRIC,this.calibration=e.calibration||null,this.targetDtype=e.targetDtype||"i8",this.excludeOps=e.excludeOps||V$,this.quantizableOps=e.quantizableOps||G$,this.sensitivityThreshold=e.sensitivityThreshold||0,this.sensitivityResult=e.sensitivityResult||null,this.weightOnly=e.weightOnly||!1,this.target=e.target||null}},cy=class extends Ge{static{l(this,"QuantizationPass")}constructor(e={}){super("QuantizationPass"),this.requiredAnalyses=[at],this.config=e instanceof ly?e:new ly(e)}run(e,t){let s=(t?t.getAnalysis(at,e):at.compute(e)).topologicalOrder,o=new Set;this._paramsByValue=new Map;let i=this.config,a=!1;if(i.target&&!i.target.supportsInt8)return oe.UNCHANGED;for(let p=0;p<s.length;p++){let c=s[p];if(qt(c.opName))continue;if(i.excludeOps.has(c.opName)||!i.quantizableOps.has(c.opName)){for(let d=0;d<c.numOperands;d++)o.has(c.getOperand(d))&&(a=this._insertDequantBefore(c,d,c.getOperand(d),i)||a);continue}if(i.sensitivityResult&&i.sensitivityThreshold>0&&i.sensitivityResult.isSensitive(c,i.sensitivityThreshold)||i.weightOnly&&!q$(c))continue;let h=ne.get(c.opName),f=h?h.getAttr("quantizedVariant"):null;if(f&&U$(c,o,i)){i.scheme===kr.PER_CHANNEL&&this._canPerChannelDot(c,o)?a=this._replacePerChannelDot(c,i)||a:a=this._replaceWithNativeQuantized(c,f,o,i)||a;continue}a=this._insertDequantQuantBoundary(c,o,i)||a}let u=e.getReturnOp();if(u)for(let p=0;p<u.numOperands;p++){let c=u.getOperand(p);o.has(c)&&(a=this._insertDequantBefore(u,p,c,i)||a)}return this.trace&&this.trace.level>=ue.DEBUG&&a&&this.trace.emit({type:"pass_detail",passName:this.name,opsProcessed:s.length,changed:a,level:ue.DEBUG}),a?oe.CHANGED:oe.UNCHANGED}_getQuantParams(e,t){let n=Gs(t.targetDtype)*8;if(t.calibration&&t.calibration.hasData(e))return t.calibration.getQuantParams(e,t.scheme,t.targetDtype);let s=e.definingOp;if(s&&s.opName==="constant"){let o=s.getAttr("value");if(typeof o=="number")return hr.fromRange(-Math.abs(o)||-1,Math.abs(o)||1,t.scheme,t.targetDtype,n);if(o&&typeof o.length=="number")return hr.fromConstantArray(o,t.scheme,t.targetDtype,n)}return e.type instanceof I&&Lt(e.type.dtype)?hr.defaultForActivation(t.scheme,t.targetDtype,n):null}_resolveQuantParams(e,t){let n=this._paramsByValue.get(e);if(n)return n;let s=e.definingOp;return s&&s.opName==="quantize"?new hr({scheme:s.getAttr("scheme")||t.scheme,scale:s.getAttr("scale"),zeroPoint:s.getAttr("zero_point")||0,dtype:s.getAttr("target_dtype")||t.targetDtype}):this._getQuantParams(e,t)}_insertQuantizeAfter(e,t,n){let s=e.getResult(t),o=this._getQuantParams(s,n);if(!o)return null;let i=new I(s.type.shape,n.targetDtype),a=new Re("quantize",[s],[i],{scale:o.getScalarScale(),zero_point:o.getScalarZeroPoint(),scheme:n.scheme,target_dtype:n.targetDtype,...o.axis!==null?{axis:o.axis}:{}});e.parentBlock&&e.parentBlock.insertAfter(a,e);let u=a.getResult(0);return this._paramsByValue.set(u,o),u}_insertDequantBefore(e,t,n,s){let o=this._resolveQuantParams(n,s);if(!o)return!1;let i="f32",a=new I(n.type.shape,i),u=new Re("dequantize",[n],[a],{scale:o.getScalarScale(),zero_point:o.getScalarZeroPoint(),scheme:s.scheme,target_dtype:i});return e.parentBlock&&(e.parentBlock.insertBefore(u,e),e.replaceOperand(t,u.getResult(0))),!0}_insertDequantQuantBoundary(e,t,n){let s=!1;for(let o=0;o<e.numOperands;o++){let i=e.getOperand(o);t.has(i)&&(s=this._insertDequantBefore(e,o,i,n)||s)}for(let o=0;o<e.numResults;o++){let i=e.getResult(o);if(!(i.type instanceof I)||!Lt(i.type.dtype))continue;let a=this._insertQuantizeAfter(e,o,n);if(a){let u=[...i.uses()];for(let p of u)p.user!==a.definingOp&&p.user.replaceOperand(p.operandIndex,a);t.add(a),s=!0}}return s}_replaceWithNativeQuantized(e,t,n,s){let o=[],i={};for(let[h,f]of e.attributes||[])i[h]=f;for(let h=0;h<e.numOperands;h++){let f=e.getOperand(h);if(n.has(f)){o.push(f);let _=this._resolveQuantParams(f,s);if(_){let b=h===0?"lhs":"rhs";if(t==="quantized_dot")i[`${b}_scale`]=_.getScalarScale(),i[`${b}_zero_point`]=_.getScalarZeroPoint();else{let w=h===0?"input":"kernel";i[`${w}_scale`]=_.getScalarScale(),i[`${w}_zero_point`]=_.getScalarZeroPoint()}}continue}let d=this._getQuantParams(f,s);if(!d)return!1;let m=new I(f.type.shape,s.targetDtype),g=new Re("quantize",[f],[m],{scale:d.getScalarScale(),zero_point:d.getScalarZeroPoint(),scheme:s.scheme,target_dtype:s.targetDtype});e.parentBlock&&e.parentBlock.insertBefore(g,e),o.push(g.getResult(0));let y=h===0?t==="quantized_dot"?"lhs":"input":t==="quantized_dot"?"rhs":"kernel";i[`${y}_scale`]=d.getScalarScale(),i[`${y}_zero_point`]=d.getScalarZeroPoint()}let a=i.lhs_scale||i.input_scale||1,u=i.rhs_scale||i.kernel_scale||1;i.output_scale=a*u,i.output_zero_point=0;let p=new I(e.getResult(0).type.shape,"i32"),c=new Re(t,o,[p],i);if(e.parentBlock){e.parentBlock.insertBefore(c,e);let h=new I(e.getResult(0).type.shape,e.getResult(0).type.dtype),f=i.output_scale||1,d=new Re("dequantize",[c.getResult(0)],[h],{scale:f,zero_point:i.output_zero_point||0,scheme:s.scheme,target_dtype:e.getResult(0).type.dtype});e.parentBlock.insertBefore(d,e),e.replaceAllResultsWith([d.getResult(0)]),e.erase()}return!0}_canPerChannelDot(e,t){if(e.opName!=="dot")return!1;let n=e.getOperand(0),s=e.getOperand(1);if(t.has(n)||t.has(s)||!(n.type instanceof I)||n.type.shape.length!==2||!(s.type instanceof I)||s.type.shape.length!==2)return!1;let o=s.definingOp;if(!o||o.opName!=="constant")return!1;let i=o.getAttr("value");if(!i||typeof i=="number"||typeof i.length!="number")return!1;let a=e.getAttr("rhs_contracting")||[],u=e.getAttr("lhs_contracting")||[];return!(a.length!==1||u.length!==1||(e.getAttr("rhs_batch")||[]).length!==0||(e.getAttr("lhs_batch")||[]).length!==0)}_activationParams(e,t){let n=Gs(t.targetDtype)*8,s=kr.PER_TENSOR_SYMMETRIC;return t.calibration&&t.calibration.hasData(e)?t.calibration.getQuantParams(e,s,t.targetDtype):hr.defaultForActivation(s,t.targetDtype,n)}_replacePerChannelDot(e,t){let n=e.getOperand(0),s=e.getOperand(1),o=s.type.shape,i=s.definingOp.getAttr("value"),a=Gs(t.targetDtype)*8,p=e.getAttr("rhs_contracting")[0]===0?1:0,c=hr.fromConstantArrayPerChannel([...i],o,p,t.targetDtype,a),h=c.quantizeArrayPerChannel([...i],o),f=this._activationParams(n,t),d=f.getScalarScale(),m=f.getScalarZeroPoint(),g=e.parentBlock;if(!g)return!1;let y=new I(o,t.targetDtype),_=new Re("constant",[],[y],{value:h,tensor_type:y});g.insertBefore(_,e);let b=new I(n.type.shape,t.targetDtype),w=new Re("quantize",[n],[b],{scale:d,zero_point:m,scheme:kr.PER_TENSOR_SYMMETRIC,target_dtype:t.targetDtype});g.insertBefore(w,e);let x=e.getResult(0).type.shape,S={};for(let[U,P]of e.attributes||[])S[U]=P;S.lhs_scale=d,S.lhs_zero_point=m,S.rhs_scale=1,S.rhs_zero_point=0,S.output_scale=1,S.output_zero_point=0;let T=new I(x,"i32"),k=new Re("quantized_dot",[w.getResult(0),_.getResult(0)],[T],S);g.insertBefore(k,e);let A=new I(x,"f32"),N=new Re("convert",[k.getResult(0)],[A],{target_dtype:"f32"});g.insertBefore(N,e);let v=o[p],O=new Array(v);for(let U=0;U<v;U++)O[U]=d*c.getScaleForChannel(U);let R=new Set(e.getAttr("lhs_contracting")||[]),B=0;for(let U=0;U<n.type.shape.length;U++)R.has(U)||B++;let E=B,z=new I([v],"f32"),D=new Re("constant",[],[z],{value:O,tensor_type:z});g.insertBefore(D,e);let J=new Re("broadcast_in_dim",[D.getResult(0)],[A],{broadcast_dimensions:[E],result_shape:x});g.insertBefore(J,e);let j=new Re("mul",[N.getResult(0),J.getResult(0)],[A],{});return g.insertBefore(j,e),e.replaceAllResultsWith([j.getResult(0)]),e.erase(),!0}};l(q$,"hasConstantOperand");l(U$,"allOperandsCanQuantize")});var bT,py,mB=M(()=>{"use strict";Es();Oe();bT=class{static{l(this,"LivenessResult")}constructor(e,t,n,s,o,i,a){this.liveIn=e,this.liveOut=t,this.intervals=n,this.opIndex=s,this.peakPressure=o,this.peakOp=i,this.pressureAtOp=a}interfere(e,t){if(e===t)return!0;let n=this.intervals.get(e),s=this.intervals.get(t);return n&&s?n.start<=s.end&&s.start<=n.end:!1}liveAtOp(e){return this.liveIn.get(e)||new Set}intervalOf(e){return this.intervals.get(e)||null}},py=class r{static get name(){return"liveness"}static get depKey(){return"liveness"}static get dependencies(){return[at]}static buildIntervals(e,t){let n=new Map;for(let o=0;o<t.length;o++)n.set(t[o],o);let s=new Map;for(let o of e.args)s.set(o,{start:-1,end:-1});for(let o=0;o<t.length;o++){let i=t[o];for(let a=0;a<i.numResults;a++)s.set(i.getResult(a),{start:o,end:o})}for(let o=0;o<t.length;o++){let i=t[o];for(let a=0;a<i.numOperands;a++){let u=i.getOperand(a),p=s.get(u);p&&p.end<o&&(p.end=o)}}return{intervals:s,opIndex:n}}static compute(e,t={}){let n=t.useDef||at.compute(e),s=n.topologicalOrder,{intervals:o,opIndex:i}=r.buildIntervals(e,s),a=new Map,u=new Map;for(let f of s)a.set(f,new Set),u.set(f,new Set);for(let f=s.length-1;f>=0;f--){let d=s[f],m=u.get(d),g=n.opUsers.get(d);if(g)for(let _ of g){let b=a.get(_);if(b)for(let w of b)m.add(w)}let y=a.get(d);for(let _ of m)y.add(_);for(let _=0;_<d.numResults;_++)y.delete(d.getResult(_));for(let _=0;_<d.numOperands;_++)y.add(d.getOperand(_))}for(let[f,d]of i){let m=a.get(f);for(let g of m){let y=o.get(g);y&&y.end<d&&(y.end=d)}}let p=0,c=null,h=new Map;for(let f=0;f<s.length;f++){let d=s[f],m=a.get(d),g=0;for(let y of m)if(y.type instanceof I){let _=y.type.sizeInBytes();_!==X&&(g+=_)}h.set(d,g),g>p&&(p=g,c=d)}return new bT(a,u,o,i,p,c,h)}}});var fy,hy,gB=M(()=>{"use strict";nn();sr();Oe();At();Kt();Es();mB();Tr();fy=class{static{l(this,"RematerializationConfig")}constructor(e={}){this.memoryBudget=e.memoryBudget||1/0,this.maxIterations=e.maxIterations||100,this.maxRecomputeCost=e.maxRecomputeCost||1/0,this.excludeOps=e.excludeOps||new Set}},hy=class extends Ge{static{l(this,"RematerializationPass")}constructor(e={}){super("RematerializationPass"),this.config=e instanceof fy?e:new fy(e)}run(e,t){if(this.config.memoryBudget===1/0)return oe.UNCHANGED;let n=!1,s=0,o=0;for(;s<this.config.maxIterations;){let i=at.compute(e),{peakPressure:a,candidates:u,opIndex:p}=this._analyzeIntervalPressure(e,i);if(o=a,a<=this.config.memoryBudget||u.length===0)break;u.sort((c,h)=>h.score-c.score),this._rematerialize(e,u[0],p),n=!0,s++}return this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,iterations:s,peakPressure:o,budget:this.config.memoryBudget,changed:n,level:ue.DEBUG}),n?oe.CHANGED:oe.UNCHANGED}_analyzeIntervalPressure(e,t){let n=t.topologicalOrder,{intervals:s,opIndex:o}=py.buildIntervals(e,n),i=[];for(let[m,g]of s){if(!(m.type instanceof I))continue;let y=m.type.sizeInBytes();y===X||y<=0||(i.push({idx:g.start,delta:y,value:m}),i.push({idx:g.end+1,delta:-y,value:m}))}i.sort((m,g)=>m.idx-g.idx||m.delta-g.delta);let a=0,u=0,p=0,c=new Set,h=new Set,f=0;for(let m=-1;m<=n.length;m++){for(;f<i.length&&i[f].idx<=m;)a+=i[f].delta,i[f].delta>0&&i[f].value&&h.add(i[f].value),i[f].delta<0&&i[f].value&&h.delete(i[f].value),f++;if(a>u){u=a,p=m,c.clear();for(let g of h)c.add(g)}}let d=[];for(let m of c){if(!this._canRematerialize(m))continue;let g=m.definingOp,y=this._computeMemorySaved(m);if(y<=0)continue;let _=this._estimateRecomputeCost(g);_>=this.config.maxRecomputeCost||_!==0&&d.push({value:m,definingOp:g,memorySaved:y,recomputeCost:_,score:y/_})}return{peakPressure:u,peakIdx:p,candidates:d,opIndex:o}}_canRematerialize(e){if(e.isBlockArgument())return!1;let t=e.definingOp;if(!t||qt(t.opName)||ln(t.opName)||this.config.excludeOps.has(t.opName)||t.regions.length>0||t.hasSideEffects()||e.useCount<=1)return!1;for(let n=0;n<t.numOperands;n++){let s=t.getOperand(n);if(s.definingOp&&s.definingOp.hasSideEffects())return!1}return!0}_computeMemorySaved(e){if(!(e.type instanceof I))return 0;let t=e.type.sizeInBytes();return t===X?0:t}_operandExtensionCost(e,t,n){let s=0;for(let o=0;o<e.numOperands;o++){let i=e.getOperand(o),a=0;for(let u of i.uses()){let p=n.get(u.user)||0;p>a&&(a=p)}if(t>a&&i.type instanceof I){let u=i.type.sizeInBytes();u!==X&&(s+=u)}}return s}_estimateRecomputeCost(e){let t=ne.get(e.opName);if(!t||t.isOpaque)return 1/0;if(t.getFlops){let n=t.getFlops(e);if(n>0)return n}if(t.isElementwise||t.isBroadcast||t.isView){let n=0;for(let s=0;s<e.numResults;s++){let o=e.getResult(s).type;if(o instanceof I){let i=o.numel();if(i!==X){n=i;break}}}return n||1}if(t.isReduction)for(let n=0;n<e.numOperands;n++){let s=e.getOperand(n).type;if(s instanceof I){let o=s.numel();if(o!==X)return o}}return 1/0}_rematerialize(e,t,n){let{value:s,definingOp:o}=t,i=[];for(let c of s.uses())i.push({user:c.user,operandIndex:c.operandIndex});if(i.length<=1)return;i.sort((c,h)=>(n.get(c.user)||0)-(n.get(h.user)||0));let a=n.get(i[i.length-1].user)||0,u=this._operandExtensionCost(o,a,n),p=this._computeMemorySaved(s);if(!(u>=p))for(let c=1;c<i.length;c++){let h=i[c],f=h.user;if(!f.parentBlock)continue;let d=new Re(o.opName,[...o.operands],o.results.map(m=>m.type),new Map(o.attributes));f.parentBlock.insertBefore(d,f),f.replaceOperand(h.operandIndex,d.getResult(s.resultIndex))}}}});function oo(r){return ln(r.opName)}function yB(r){let e=0;for(let t=0;t<r.numResults;t++){let n=r.getResult(t).type;if(!n||!n.isFullyStatic)continue;let s=n.sizeInBytes();s>e&&(e=s)}return e}function W$(r){let e=r.definingOp;if(e&&e.opName==="broadcast"){let t=e.getOperand(0);e=t&&t.definingOp}if(e&&oo(e)){let t=e.getAttr("value");if(typeof t=="number")return t}return 0}function wT(r){if(r.opName!=="dot"&&r.opName!=="cublas_gemm")return null;let e=r.getOperand(0).type,t=r.getOperand(1).type,n=r.getResult(0).type;if(!e||!t||!n||e.dtype!=="f32"||t.dtype!=="f32"||n.dtype!=="f32")return null;let s=r.getAttr("lhs_batch")||[],o=r.getAttr("rhs_batch")||[];if(s.length>0||o.length>0)return null;let i=r.getAttr("lhs_contracting")||[],a=r.getAttr("rhs_contracting")||[];if(i.length!==1||a.length!==1||t.rank!==2||i[0]!==e.rank-1||a[0]!==0&&a[0]!==1||!e.isFullyStatic||!t.isFullyStatic||!n.isFullyStatic)return null;let u=r.getOperand(0).definingOp,p=r.getOperand(1).definingOp;if(u&&oo(u)||p&&oo(p))return null;let c=a[0]===1,h=e.shape[e.rank-1];if((c?t.shape[1]:t.shape[0])!==h)return null;let d=1;for(let g=0;g<e.rank-1;g++)d*=e.shape[g];let m=c?t.shape[0]:t.shape[1];return d<=0||m<=0||h<=0?null:{M:d,N:m,K:h,transB:c}}function _B(r){return{labelOf:l(e=>r.get(e),"labelOf"),canMerge:l((e,t)=>Math.max(e.maxBuf||0,yB(t))<=K$,"canMerge"),onAttach:l((e,t)=>{e.maxBuf=Math.max(e.maxBuf||0,yB(t))},"onAttach")}}function dy(r,e,t){let n=r.opSet,s=Ba(r.ops),{inputs:o,outputs:i,constDefs:a}=_i(n,s,{pullConstants:!0,isConstant:oo});for(let f of o)if(!f.type||!f.type.isFullyStatic)return null;for(let f of i)if(!f.type||!f.type.isFullyStatic)return null;let u=new un(e,o.map(f=>f.type),i.map(f=>f.type)),p=new Map;for(let f=0;f<o.length;f++)p.set(o[f],u.args[f]);for(let f of a)u.entryBlock.pushOp(f.clone(p));for(let f of s)u.entryBlock.pushOp(f.clone(p));let c=i.map(f=>p.get(f));if(c.some(f=>f===void 0))return null;u.entryBlock.pushOp(new Re("return",c,[]));let h=r.ops.length===1&&t.has(r.ops[0])?r.ops[0]:null;return{part:r,subFunc:u,inputs:o,outputs:i,dotOp:h}}function sf(r){if(H$.has(r.opName))return!0;if(r.regions)for(let e of r.regions){let t=e.entryBlock;if(t){for(let n of t.ops())if(sf(n))return!0}}return!1}function Y$(r){let e=0;for(let t=0;t<r.numResults;t++){let n=r.getResult(t).type;if(!n||!n.isFullyStatic)continue;let s=1;for(let o of n.shape)s*=o;s>e&&(e=s)}return e}function bB(r,e=256){if(r.functionCount!==1)return!1;let t=r.functions().next().value,n=new Map,s=l(o=>{if(!o)return 0;let i=n.get(o);if(i!==void 0)return i;n.set(o,0);let a=sf(o)?Y$(o):0;for(let u=0;u<o.numOperands;u++){let p=s(o.getOperand(u).definingOp);p>a&&(a=p)}return n.set(o,a),a},"maxBoundaryInSubtree");for(let o of t.ops())if(!(qt(o.opName)||!sf(o))){for(let i=0;i<o.numOperands;i++)if(s(o.getOperand(i).definingOp)>e)return!0}return!1}function wB(r,e,t){let n=new Map,s=0,o=l(d=>{let m=n.get(d);return m===void 0&&(m=s++,n.set(d,m)),m},"getSlot");for(let d of r.args)o(d);for(let d of t)for(let m of d.outputs)o(m);let i=[];for(let d of r.args)i.push(o(d));let a=[],u=new Set;for(let d=0;d<e.numOperands;d++){let m=e.getOperand(d),g=i.length,y=m.isBlockArgument&&m.isBlockArgument(),_=m.definingOp&&oo(m.definingOp);if(!y&&!_&&n.has(m)){let b=n.get(m);if(!u.has(b)){u.add(b),i.push(b);continue}i.push(s++),a.push({pos:g,kind:"copy",srcSlot:b});continue}if(y){i.push(s++),a.push({pos:g,kind:"copy",srcSlot:o(m)});continue}if(_){i.push(s++),a.push({pos:g,kind:"const",value:W$(m)});continue}return null}let p=[];for(let d of t){let m=[];for(let y of d.inputs){let _=n.get(y);if(_===void 0)return null;m.push(_)}let g=d.outputs.map(y=>n.get(y));p.push({name:d.subFunc.name,inputSlots:m,outputSlots:g})}let c=new Set(i),h=[],f=new Set;for(let[d,m]of n)if(!(c.has(m)||f.has(m))){if(f.add(m),!d.type||!d.type.isFullyStatic)return null;h.push({slot:m,shape:[...d.type.shape],dtype:d.type.dtype})}return{plan:{numSlots:s,argSlots:i,intermediates:h,steps:p,returnFixups:a}}}function Ti(r,e=2){if(r.functionCount!==1)return null;let t=r.functions().next().value,n=t.getReturnOp();if(!n)return null;let s=[],o=new Map,i=0;for(let g of t.ops())qt(g.opName)||oo(g)||(sf(g)?o.set(g,"boundary#"+i++):o.set(g,"native"),s.push(g));if(i<e||s.length===0)return null;let{partitions:a,preds:u}=tf(s,_B(o));if(a.length<2)return null;let p=hT(a,u);if(!p)return null;let c=t.name,h=[],f=new Map,d=0;for(let g of p){let y=dy(g,c+"_p"+d++,f);if(!y)return null;h.push(y)}let m=wB(t,n,h);if(!m)return null;r.removeFunction(t.name);for(let g of h)r.addFunction(g.subFunc);return{plan:m.plan}}function TB(r){if(r.functionCount!==1)return null;let e=r.functions().next().value,t=e.getReturnOp();if(!t)return null;let n=[],s=new Map,o=new Map,i=0,a=0;for(let y of e.ops()){if(qt(y.opName)||oo(y))continue;let _=wT(y);_?(s.set(y,"cublas#"+i),o.set(y,_),i++):sf(y)?s.set(y,"boundary#"+a++):s.set(y,"native"),n.push(y)}if(i+a===0||n.length===0)return null;let{partitions:u,preds:p}=tf(n,_B(s));if(u.length<2)return null;let c=hT(u,p);if(!c)return null;let h=e.name,f=[],d=0;for(let y of c){let _=dy(y,h+"_p"+d++,o);if(!_)return null;f.push(_)}let m=wB(e,t,f);if(!m)return null;let g=new Map;for(let y of f){if(!y.dotOp)continue;let _=o.get(y.dotOp),b=y.inputs.indexOf(y.dotOp.getOperand(0)),w=y.inputs.indexOf(y.dotOp.getOperand(1)),x=y.outputs.indexOf(y.dotOp.getResult(0));b<0||w<0||x<0||g.set(y.subFunc.name,{M:_.M,N:_.N,K:_.K,transB:_.transB,aIdx:b,bIdx:w,cIdx:y.inputs.length+x})}if(g.size===0)return null;r.removeFunction(e.name);for(let y of f)r.addFunction(y.subFunc);return{plan:m.plan,cublasInfos:g}}var K$,H$,my=M(()=>{"use strict";xs();sr();nf();Tr();l(oo,"isConstantOp");K$=32*1024;l(yB,"maxResultBytes");l(W$,"constScalarOf");l(wT,"cublasDotInfo");l(_B,"bufferLimitedConfig");l(dy,"materializePartition");H$=new Set(["dot","fused_dot_epilogue","cublas_gemm","reduce","conv","quantized_conv","scaled_dot_product_attention"]);l(sf,"containsBoundaryOp");l(Y$,"maxBoundaryResultBytes");l(bB,"hasDependentBoundaries");l(wB,"buildExecutionPlan");l(Ti,"splitGraphForNative");l(TB,"splitGraphForCublas")});function SB(r){return xB.push(r),r}var xB,gy,kB=M(()=>{"use strict";nn();xB=[];l(SB,"registerBackendOpRewrite");gy=class extends Ge{static{l(this,"BackendOpRewritePass")}constructor(e={}){super(e.name||"BackendOpRewritePass"),this.config=e,this.rewrites=e.rewrites||xB}run(e){let t=!1;for(let n of[...e.ops()]){let s=n.parentBlock;if(s){for(let o of this.rewrites)if(o.match(n,this.config)){o.build(n,s,this.config),t=!0;break}}}return t?oe.CHANGED:oe.UNCHANGED}}});var J$,yy,vB=M(()=>{"use strict";sr();my();kB();J$=SB({name:"dot->cublas_gemm",match:l(r=>r.opName==="dot"&&r.numOperands===2&&wT(r)!==null,"match"),build:l((r,e)=>{let t=r.results.map(s=>s.type),n=new Re("cublas_gemm",[r.getOperand(0),r.getOperand(1)],t,new Map(r.attributes),null);e.insertBefore(n,r),r.replaceAllResultsWith(n.results),r.erase()},"build")}),yy=class extends gy{static{l(this,"CublasRewritePass")}constructor(e={}){super({...e,name:"CublasRewritePass",rewrites:[J$]})}}});function NB(r,e,{cudaMatmulChain:t=!1,context:n=null}={}){let s=n?a=>n.passesForPhase(a,r,e):a=>Vg(a,r,e),o=[];for(let a of s("pre"))o.push(a);if(o.push(new Vu(e)),o.push(new zu("canonicalize",[new ia,new Ug({fastMath:r.optimization.fastMath}),new Kg,new Wg,new no],r.optimization.maxSimplifyIterations)),r.optimization.layout&&e&&(o.push(new uy({target:e})),o.push(new no)),r.quantization.enabled&&(o.push(new cy({...r.quantization,target:e})),o.push(new ia),o.push(new no)),r.fusion.enabled&&r.matmulBackend!=="cublas"&&e&&e.enableEpilogueFusion&&(o.push(new Xg({target:e})),o.push(new no)),r.fusion.enabled){let a=r.fusion,u=a.launchOverheadUs??Z$;a.strategy==="dominator"?o.push(new ny({target:e,...a})):a.strategy==="priority"?(o.push(new oy({target:e,cost:{launchOverheadUs:u},...a})),o.push(new rf({maxFusionSize:e?.maxFusionSize,...a}))):(o.push(new Zg({target:e,cost:{launchOverheadUs:u},...a})),o.push(new ey({maxFusionSize:e?.maxFusionSize,...a})),o.push(new rf({maxFusionSize:e?.maxFusionSize,...a}))),o.push(new no)}if(r.matmulBackend==="cublas"&&o.push(new yy),r.optimization.rematerialization){let a={...r.optimization.rematConfig};a.memoryBudget===void 0&&e&&e.memoryBudgetBytes>0&&(a.memoryBudget=e.memoryBudgetBytes),o.push(new hy(a))}for(let a of s("post"))o.push(a);return o}var Z$,AB=M(()=>{"use strict";am();Dw();qO();WO();HO();$w();JO();QO();XO();eB();rB();iB();pB();dB();Lw();gB();vB();uT();Z$=5;l(NB,"buildGraphPipeline")});var Wn,xi=M(()=>{"use strict";Wn=class{static{l(this,"PrimFuncPass")}constructor(e,t=null){this.name=e,this.phase=t||e,this.snapshotPoint=null,this.trace=null}begin(e){}run(e,t){throw new Error("PrimFuncPass.run not implemented")}end(e){}}});var _y,TT,Ra,xT=M(()=>{"use strict";De();_y=["iterCount","depth","parallelLoops","vectorizedLoops","unrolledLoops","threadBoundLoops","serialLoops","threadBlockSize","gridSize","underReduction","numMathOps","numExternCalls","numReads","numWrites","stride1Accesses","stridedAccesses","reuseCount","touchedBytes","arithmeticIntensity","vectorized","parallelized","innermostExtent"],TT=class{static{l(this,"ScheduleFeatures")}constructor(e){this.numLoops=e.numLoops||0,this.numBlocks=e.numBlocks||0,this.totalIterations=e.totalIterations||0,this.maxLoopDepth=e.maxLoopDepth||0,this.numParallelLoops=e.numParallelLoops||0,this.numVectorizedLoops=e.numVectorizedLoops||0,this.numUnrolledLoops=e.numUnrolledLoops||0,this.numThreadBound=e.numThreadBound||0,this.numSerialLoops=e.numSerialLoops||0,this.totalBufferBytes=e.totalBufferBytes||0,this.numBufferReads=e.numBufferReads||0,this.numBufferWrites=e.numBufferWrites||0,this.numMathOps=e.numMathOps||0,this.numExternCalls=e.numExternCalls||0,this.arithmeticIntensity=e.arithmeticIntensity||0,this.innermostExtent=e.innermostExtent||0,this.outermostExtent=e.outermostExtent||0,this.hasReduction=e.hasReduction||!1,this.reductionDepth=e.reductionDepth||0,this.threadBlockSize=e.threadBlockSize||0,this.gridSize=e.gridSize||0,this.strideOneAccesses=e.strideOneAccesses||0,this.nonStrideOneAccesses=e.nonStrideOneAccesses||0}},Ra=class r{static{l(this,"FeatureExtractor")}static extract(e){let t={numLoops:0,numBlocks:0,totalIterations:1,maxLoopDepth:0,currentDepth:0,numParallelLoops:0,numVectorizedLoops:0,numUnrolledLoops:0,numThreadBound:0,numSerialLoops:0,totalBufferBytes:0,buffersSeen:new Set,numBufferReads:0,numBufferWrites:0,numMathOps:0,numExternCalls:0,hasReduction:!1,reductionDepth:0,threadBlockSize:1,gridSize:1,innermostExtent:0,outermostExtent:0,loopExtents:[],strideOneAccesses:0,nonStrideOneAccesses:0};r._visitIterative(e.body,t);let n=t.totalBufferBytes,s=t.numMathOps+t.numExternCalls;return t.arithmeticIntensity=n>0?s/n:0,new TT(t)}static extractStatements(e){let t=[],n=[],s=0,o=[{node:e.body,action:"enter"}];for(;o.length>0;){let{node:i,action:a}=o.pop();if(i){if(a==="leaveFor"){n.pop();continue}if(a==="leaveBlock"){s--;continue}switch(i.type){case"ForNode":n.push(i),o.push({node:null,action:"leaveFor"}),o.push({node:i.body,action:"enter"});break;case"BlockNode":i.initBody&&(s++,o.push({node:null,action:"leaveBlock"}),o.push({node:i.initBody,action:"enter"})),o.push({node:i.body,action:"enter"});break;case"SeqNode":for(let u=i.stmts.length-1;u>=0;u--)o.push({node:i.stmts[u],action:"enter"});break;case"AllocateNode":case"LetStmtNode":o.push({node:i.body,action:"enter"});break;case"IfThenElseNode":i.elseBody&&o.push({node:i.elseBody,action:"enter"}),o.push({node:i.thenBody,action:"enter"});break;case"BufferStoreNode":t.push(r._statementVector(i,n,s));break}}}return t}static _statementVector(e,t,n){let s=1,o=0,i=0,a=0,u=0,p=0,c=1,h=1;for(let T of t){let k=T.extent&&T.extent.type==="IntImmNode"?T.extent.value:1;switch(s*=k,T.kind){case F.PARALLEL:o++;break;case F.VECTORIZED:i++;break;case F.UNROLLED:a++;break;case F.THREAD_BINDING:u++,T.threadTag&&T.threadTag.startsWith("threadIdx")?c*=k:T.threadTag&&T.threadTag.startsWith("blockIdx")&&(h*=k);break;default:p++;break}}let f={math:0,extern:0};r._countExpr(e.value,f);let d=t.map(T=>T.loopVar.name),m=[{buffer:e.buffer,indices:e.indices}];r._collectLoads(e.value,m);let g=0,y=0,_=0,b=0;for(let T of m){let k=T.buffer&&T.buffer.sizeInBytes?T.buffer.sizeInBytes():0;k>0&&(b+=k);let A=T.indices&&T.indices.length>0?T.indices[T.indices.length-1]:null;A&&A.type==="VariableNode"?g++:y++;let N=new Set;if(T.indices)for(let v of T.indices)r._collectVars(v,N);for(let v of d)N.has(v)||_++}let w=t.length>0?t[t.length-1]:null,x=w&&w.extent&&w.extent.type==="IntImmNode"?w.extent.value:0,S={iterCount:s,depth:t.length,parallelLoops:o,vectorizedLoops:i,unrolledLoops:a,threadBoundLoops:u,serialLoops:p,threadBlockSize:c,gridSize:h,underReduction:n>0?1:0,numMathOps:f.math,numExternCalls:f.extern,numReads:m.length-1,numWrites:1,stride1Accesses:g,stridedAccesses:y,reuseCount:_,touchedBytes:b,arithmeticIntensity:b>0?(f.math+f.extern)/b:0,vectorized:i>0?1:0,parallelized:o+u>0?1:0,innermostExtent:x};return _y.map(T=>S[T]||0)}static _countExpr(e,t){if(!(!e||typeof e!="object"))switch(e.type){case"MathOpNode":t.math++,r._countExpr(e.a,t),e.b&&r._countExpr(e.b,t);break;case"CompareNode":t.math++,r._countExpr(e.a,t),r._countExpr(e.b,t);break;case"CallExternNode":t.extern++;for(let n of e.args)r._countExpr(n,t);break;default:break}}static _collectLoads(e,t){if(!(!e||typeof e!="object")){if(e.type==="BufferLoadNode"){t.push({buffer:e.buffer,indices:e.indices});return}if(e.a&&r._collectLoads(e.a,t),e.b&&r._collectLoads(e.b,t),e.args)for(let n of e.args)r._collectLoads(n,t)}}static _collectVars(e,t){if(!(!e||typeof e!="object")){if(e.type==="VariableNode"){t.add(e.name);return}if(e.a&&r._collectVars(e.a,t),e.b&&r._collectVars(e.b,t),e.args)for(let n of e.args)r._collectVars(n,t);if(e.indices)for(let n of e.indices)r._collectVars(n,t)}}static _visitIterative(e,t){let n=[{node:e,action:"enter"}];for(;n.length>0;){let{node:s,action:o}=n.pop();if(s){if(o==="leave_for"){t.currentDepth--;continue}switch(s.type){case"ForNode":{t.numLoops++,t.currentDepth++,t.currentDepth>t.maxLoopDepth&&(t.maxLoopDepth=t.currentDepth);let i=s.extent.type==="IntImmNode"?s.extent.value:1;switch(t.loopExtents.push(i),t.numLoops===1&&(t.outermostExtent=i),t.innermostExtent=i,s.kind){case F.PARALLEL:t.numParallelLoops++;break;case F.VECTORIZED:t.numVectorizedLoops++;break;case F.UNROLLED:t.numUnrolledLoops++;break;case F.THREAD_BINDING:t.numThreadBound++,s.threadTag&&s.threadTag.startsWith("threadIdx")?t.threadBlockSize*=i:s.threadTag&&s.threadTag.startsWith("blockIdx")&&(t.gridSize*=i);break;default:t.numSerialLoops++;break}t.totalIterations*=i,n.push({node:null,action:"leave_for"}),n.push({node:s.body,action:"enter"});break}case"BlockNode":t.numBlocks++,s.initBody&&(t.hasReduction=!0,t.reductionDepth=t.currentDepth);for(let i of s.reads)r._visitBuffer(i.buffer,t);for(let i of s.writes)r._visitBuffer(i.buffer,t);n.push({node:s.body,action:"enter"}),s.initBody&&n.push({node:s.initBody,action:"enter"});break;case"SeqNode":for(let i=s.stmts.length-1;i>=0;i--)n.push({node:s.stmts[i],action:"enter"});break;case"AllocateNode":r._visitBuffer(s.buffer,t),n.push({node:s.body,action:"enter"});break;case"IfThenElseNode":s.elseBody&&n.push({node:s.elseBody,action:"enter"}),n.push({node:s.thenBody,action:"enter"});break;case"LetStmtNode":n.push({node:s.body,action:"enter"});break;case"BufferStoreNode":t.numBufferWrites++,r._visitBuffer(s.buffer,t),r._checkStride(s.buffer,s.indices,t),r._visitExpr(s.value,t);break}}}}static _visitBuffer(e,t){if(!e||t.buffersSeen.has(e))return;t.buffersSeen.add(e);let n=e.sizeInBytes();n>0&&(t.totalBufferBytes+=n)}static _visitExpr(e,t){if(e)switch(e.type){case"MathOpNode":t.numMathOps++,r._visitExpr(e.a,t),e.b&&r._visitExpr(e.b,t);break;case"CallExternNode":t.numExternCalls++;for(let n of e.args)r._visitExpr(n,t);break;case"BufferLoadNode":t.numBufferReads++,r._checkStride(e.buffer,e.indices,t);break;case"CompareNode":t.numMathOps++,r._visitExpr(e.a,t),r._visitExpr(e.b,t);break;default:break}}static _checkStride(e,t,n){if(!e||!t||t.length===0)return;let s=t[t.length-1];for(;s&&s.type==="MathOpNode"&&s.b&&s.b.type==="IntImmNode";)if(s.op==="+"&&s.b.value===0)s=s.a;else if(s.op==="*"&&s.b.value===1)s=s.a;else break;s&&s.type==="VariableNode"?n.strideOneAccesses++:n.nonStrideOneAccesses++}}});function OB(r,e){if(r.length===0)return 0;let t=0;for(let n of r)t+=e[n];return t/r.length}function ST(r,e,t,n,s,o,i){let a=t.length;if(n>=s||a<o*2)return{leaf:OB(t,e)};let u=r[0].length,p=0,c=0,h=new Set(t);for(let g of t){let y=e[g];p+=y,c+=y*y}let f=null,d=null;for(let g=0;g<u;g++){let y=i[g].filter(w=>h.has(w)),_=0,b=0;for(let w=0;w<y.length-1;w++){let x=e[y[w]];_+=x,b+=x*x;let S=r[y[w]][g],T=r[y[w+1]][g];if(S===T)continue;let k=w+1,A=a-k;if(k<o||A<o)continue;let N=p-_,v=c-b,O=b-_*_/k+(v-N*N/A);(!f||O<f.cost)&&(f={f:g,thr:(S+T)/2,sIdx:w,cost:O},d=y)}}if(!f)return{leaf:OB(t,e)};let m=d;return{f:f.f,thr:f.thr,left:ST(r,e,m.slice(0,f.sIdx+1),n+1,s,o,i),right:ST(r,e,m.slice(f.sIdx+1),n+1,s,o,i)}}function Q$(r){let e=r.length,t=e>0?r[0].length:0,n=new Array(t),s=[];for(let o=0;o<e;o++)s.push(o);for(let o=0;o<t;o++)n[o]=s.slice().sort((i,a)=>r[i][o]-r[a][o]);return n}function BB(r,e){for(;r&&r.leaf===void 0;)r=e[r.f]<=r.thr?r.left:r.right;return r?r.leaf:0}var of,RB=M(()=>{"use strict";l(OB,"leafValue");l(ST,"buildTree");l(Q$,"buildFeatureOrder");l(BB,"predictTree");of=class r{static{l(this,"GradientBoostedTrees")}constructor(e={}){this.numTrees=e.numTrees??60,this.maxDepth=e.maxDepth??3,this.lr=e.lr??.1,this.minSamples=e.minSamples??1,this.trees=null,this.base=0}fit(e,t){let n=e.length;if(n===0)return;let s=0;for(let p of t)s+=p;s/=n;let o=new Array(n).fill(s),i=[];for(let p=0;p<n;p++)i.push(p);let a=Q$(e),u=[];for(let p=0;p<this.numTrees;p++){let c=new Array(n);for(let f=0;f<n;f++)c[f]=t[f]-o[f];let h=ST(e,c,i,0,this.maxDepth,this.minSamples,a);for(let f=0;f<n;f++)o[f]+=this.lr*BB(h,e[f]);u.push(h)}this.trees=u,this.base=s}predict(e){if(!this.trees)return 0;let t=this.base;for(let n of this.trees)t+=this.lr*BB(n,e);return t}serialize(){return{trees:this.trees,base:this.base,lr:this.lr}}static deserialize(e){let t=new r({lr:e.lr});return t.trees=e.trees,t.base=e.base,t}}});function MB(r){let e=r[0].length,t=new Array(e+1).fill(0);for(let n of r)for(let s=0;s<e;s++){let o=n[s]||0;tF.has(s)?o>t[s]&&(t[s]=o):t[s]+=o}for(let n of nF)n<e&&(t[n]/=r.length);return t[e]=r.length,t}var X$,eF,tF,nF,kT,rF,by,vl,Nl,vT=M(()=>{"use strict";xT();RB();X$=new Set(["depth","threadBlockSize","gridSize","underReduction","vectorized","parallelized","innermostExtent"]),eF=new Set(["arithmeticIntensity"]),tF=new Set(_y.map((r,e)=>X$.has(r)?e:-1).filter(r=>r>=0)),nF=new Set(_y.map((r,e)=>eF.has(r)?e:-1).filter(r=>r>=0));l(MB,"aggregateStatements");kT=class{static{l(this,"CostEstimate")}constructor(e,t){this.score=e,this.breakdown=t}},rF={parallelism:2,vectorization:1.5,memoryCoalescing:2,occupancy:1,arithmeticIntensity:1,loopOverhead:-.5,codeSize:-.3},by=class{static{l(this,"AnalyticalCostModel")}constructor(e,t={}){this.target=e,this._weights={...rF,...e&&e.costModelWeights?e.costModelWeights:{},...t.weights||{}}}estimate(e){let t=Ra.extract(e);return this.estimateFromFeatures(t)}score(e){return this.estimate(e).score}estimateFromFeatures(e){let t=this._scoreParallelism(e),n=this._scoreVectorization(e),s=this._scoreMemoryAccess(e),o=this._scoreOccupancy(e),i=this._scoreIntensity(e),a=this._scoreOverhead(e),u=this._scoreCodeSize(e),p=this._weights,c=p.parallelism*t+p.vectorization*n+p.memoryCoalescing*s+p.occupancy*o+p.arithmeticIntensity*i+p.loopOverhead*a+p.codeSize*u;return new kT(c,{parallelism:t,vectorization:n,memoryCoalescing:s,occupancy:o,arithmeticIntensity:i,loopOverhead:a,codeSize:u})}_scoreParallelism(e){if(this.target.isGPU()){let n=e.threadBlockSize*e.gridSize,s=this.target.maxParallelism();return Math.min(1,n/Math.max(s*.1,1))}return e.numParallelLoops/Math.max(e.numLoops,1)}_scoreVectorization(e){return e.numLoops===0?0:this.target.isGPU()?e.strideOneAccesses/Math.max(e.strideOneAccesses+e.nonStrideOneAccesses,1):e.numVectorizedLoops>0?Math.min(1,e.innermostExtent/this.target.vectorWidth):0}_scoreMemoryAccess(e){let t=e.strideOneAccesses+e.nonStrideOneAccesses;return t===0?1:e.strideOneAccesses/t}_scoreOccupancy(e){if(!this.target.isGPU())return 1;if(e.threadBlockSize===0)return 0;let t=this.target.warpSize,n=Math.ceil(e.threadBlockSize/t),s=Math.floor(this.target.maxThreadsPerBlock/t);return Math.min(1,n/s)}_scoreIntensity(e){let t=this.target.supportsFloat16?1.5:1;return Math.min(1,e.arithmeticIntensity*10*t)}_scoreOverhead(e){return e.numSerialLoops/Math.max(e.numLoops,1)}_scoreCodeSize(e){return Math.min(1,(e.numMathOps+e.numExternCalls)/256)}compare(e,t){return this.estimate(e).score-this.estimate(t).score}},vl=class r{static{l(this,"LearnedCostModel")}constructor(e=null,t={}){this.opts={numTrees:t.numTrees??60,maxDepth:t.maxDepth??3,lr:t.lr??.1,minSamples:t.minSamples??1},this._gbt=e?of.deserialize(e):null,this._X=[],this._Y=[]}addSample(e,t){!e||e.length===0||Number.isFinite(t)&&(this._X.push(MB(e)),this._Y.push(t))}train(){if(this._X.length===0)return;let e=new of(this.opts);e.fit(this._X,this._Y),this._gbt=e}predict(e){return!this._gbt||!e||e.length===0?0:this._gbt.predict(MB(e))}get trained(){return this._gbt!==null}get sampleCount(){return this._X.length}serialize(){return{gbt:this._gbt?this._gbt.serialize():null,numSamples:this._X.length}}static deserialize(e){return new r(e&&e.gbt?e.gbt:null)}},Nl=class{static{l(this,"GuidedCostModel")}constructor(e,t,n={}){this.analytical=e,this.learned=t,this.confidenceSamples=n.confidenceSamples??8}_learnedConfident(){return this.learned&&this.learned.trained&&this.learned.sampleCount>=this.confidenceSamples}score(e){return this._learnedConfident()?this.learned.predict(Ra.extractStatements(e)):this.analytical.score(e)}}});function*IB(r){let e=[r];for(;e.length>0;){let t=e.pop();if(t){if(yield t,t.body&&e.push(t.body),t.stmts)for(let n of t.stmts)e.push(n);t.thenBody&&e.push(t.thenBody),t.elseBody&&e.push(t.elseBody),t.initBody&&e.push(t.initBody)}}}function af(r,e){for(let t of IB(r))if(t.type==="BlockNode"&&t.name===e)return t;return null}function io(r){let e=[];for(let t of IB(r))t.type==="BlockNode"&&e.push(t.name);return e}function uf(r,e){let t=dt(r,e);if(!t)return{spatial:0,reduction:0,reads:0,hasReduction:!1};let n=0,s=0;for(let o of t.loops)t.reductionLoopVars.has(o.loopVar.name)?s++:n++;return{spatial:n,reduction:s,reads:t.readBuffers.length,hasReduction:t.hasReduction}}var Al=M(()=>{"use strict";ro();l(IB,"walkStmts");l(af,"findBlock");l(io,"collectAllBlockNames");l(uf,"analyzeBlockStructure")});function LB(r){return r.isGPU()?oF:sF}function NT(r){let e=0,t=0;for(let[n,s]of r.order)n==="S"?e=Math.max(e,s+1):t=Math.max(t,s+1);return{spatialLevels:e,reductionLevels:t}}var sF,oF,EB,AT=M(()=>{"use strict";sF={name:"mlt_cpu",order:[["S",0],["S",1],["S",2],["S",3],["R",0]],roles:{S0:"parallelize",S3:"vectorize"}},oF={name:"mlt_gpu",order:[["S",0],["S",1],["S",2],["R",0]],roles:{S0:"blockIdx",S1:"threadIdx",S2:"unroll"}},EB={name:"ssrsrs_cpu",order:[["S",0],["S",1],["R",0],["S",2],["R",1],["S",3]],roles:{S0:"parallelize",S3:"vectorize",R1:"unroll"}};l(LB,"getTileStructure");l(NT,"levelCounts")});var os,vr,wy=M(()=>{"use strict";os=class{static{l(this,"SearchVariable")}constructor(e,t){this.name=e,this.candidates=t}sample(e){return this.candidates[e(this.candidates.length)]}},vr=class{static{l(this,"ScheduleSketch")}constructor(e,t,n){this.name=e,this.variables=t,this._apply=n}instantiate(e){return(t,n,s)=>{this._apply(t,n,s,e)}}sampleParams(e){let t={};for(let n of this.variables)t[n.name]=n.sample(e);return t}}});function iF(r){let e=[];for(let t=1;t*t<=r;t++){if(r%t!==0)continue;e.push(t);let n=r/t;n!==t&&e.push(n)}return e.sort((t,n)=>t-n),e}function aF(r,e){let t=Math.min(r.length,e.length);for(let n=0;n<t;n++)if(r[n]!==e[n])return r[n]-e[n];return r.length-e.length}function uF(r,e){let t=new Set,n=[];r.sort(aF);for(let i of r){let a=i.join(",");t.has(a)||(t.add(a),n.push(i))}if(n.length<=e)return n;let s=new Set,o=[];for(let i=0;i<e;i++){let a=Math.floor(i*(n.length-1)/(e-1));s.has(a)||(s.add(a),o.push(n[a]))}return o}function OT(r,e,t=48){if(!Number.isFinite(r)||r<1||e<=1)return[[r]];let n=[],s=t*8,o=l((i,a,u)=>{if(!(n.length>=s)){if(a===e-1){n.push([...u,i]);return}for(let p of iF(i))if(u.push(p),o(i/p,a+1,u),u.pop(),n.length>=s)return}},"rec");return o(r,0,[]),uF(n,t)}var CB=M(()=>{"use strict";l(iF,"divisorsOf");l(aF,"cmpTuple");l(uF,"selectDiverse");l(OT,"enumerateFactorizations")});function Ty(r){return r.extent&&r.extent.type==="IntImmNode"?r.extent.value:null}function PB(r,e,t,n){let s=[],o=t;for(let i=0;i<n.length-1;i++){let a=r.getLoops(e).find(h=>h.loopVar.name===o);if(!a)return null;let u=1;for(let h=i+1;h<n.length;h++)u*=n[h];let[p,c]=r.split(a,u);s.push(p.loopVar.name),o=c.loopVar.name}return s.push(o),s}function lF(r,e,t,n,s){let o=l(i=>r.getLoops(e).find(a=>a.loopVar.name===i),"find");for(let[i,a]of t.order){let u=t.roles[i+a];if(!u)continue;let p=i==="S"?n:s;if(u==="parallelize"){let c=p[0]&&a<p[0].length?o(p[0][a]):null;c&&r.parallelize(c)}else if(u==="vectorize"){let c=p[p.length-1],h=c&&a<c.length?o(c[a]):null;h&&r.vectorize(h)}else if(u==="unroll")for(let c of p){let h=a<c.length?o(c[a]):null;h&&r.unroll(h)}else(u==="blockIdx"||u==="threadIdx")&&p.forEach((c,h)=>{if(h>=DB.length)return;let f=a<c.length?o(c[a]):null;f&&r.bindThread(f,`${u}.${DB[h]}`)})}}function $B(r,e,t,n,s,o){let i=[];for(let h=0;h<n.length;h++){let f=PB(r,e,n[h],o[`s${h}`]);if(!f)return;i.push(f)}let a=[];for(let h=0;h<s.length;h++){let f=PB(r,e,s[h],o[`r${h}`]);if(!f)return;a.push(f)}let u=[];for(let[h,f]of t.order){let d=h==="S"?i:a;for(let m of d)f<m.length&&u.push(m[f])}let p=new Map(r.getLoops(e).map(h=>[h.loopVar.name,h])),c=u.map(h=>p.get(h)).filter(Boolean);c.length>=2&&r.reorder(...c),lF(r,e,t,i,a)}function FB(r){let e=r.loops.filter(n=>!r.reductionLoopVars.has(n.loopVar.name)),t=r.loops.filter(n=>r.reductionLoopVars.has(n.loopVar.name));return{spatialLoops:e,reductionLoops:t}}function zB(r,e,t,n){let s=[];return r.forEach((o,i)=>s.push(new os(`s${i}`,OT(Ty(o),t)))),e.forEach((o,i)=>s.push(new os(`r${i}`,OT(Ty(o),n)))),s}function VB(r,e){let{spatialLevels:t,reductionLevels:n}=NT(e),{spatialLoops:s,reductionLoops:o}=FB(r);if(s.length===0)return null;for(let p of[...s,...o])if(Ty(p)===null)return null;let i=zB(s,o,t,n),a=s.map(p=>p.loopVar.name),u=o.map(p=>p.loopVar.name);return new vr(e.name,i,(p,c,h,f)=>{$B(p,c,e,a,u,f)})}function GB(r,e){let{spatialLevels:t,reductionLevels:n}=NT(e),{spatialLoops:s,reductionLoops:o}=FB(r);if(s.length===0||o.length===0)return null;for(let p of[...s,...o])if(Ty(p)===null)return null;let i=zB(s,o,t,n),a=s.map(p=>p.loopVar.name),u=o.map(p=>p.loopVar.name);return new vr(e.name,i,(p,c,h,f)=>{p.decomposeReduction(c),$B(p,`${c}_upd`,e,a,u,f)})}var DB,jB=M(()=>{"use strict";wy();CB();AT();DB=["x","y","z"];l(Ty,"staticExtent");l(PB,"multiLevelSplit");l(lF,"applyRoles");l($B,"tileBlock");l(FB,"splitLoops");l(zB,"tilingVariables");l(VB,"createMultiLevelTilingSketch");l(GB,"createSSRSRSTilingSketch")});function UB(r){return Math.min(r&&r.maxThreadsPerBlock||256,256)}function pF(r){let e=new Set;for(let t=2;t*t<=r;t++){if(r%t!==0)continue;e.add(t);let n=r/t;n>1&&n<r&&e.add(n)}return[...e].sort((t,n)=>t-n)}function KB(r){return new vr("fused",[],(e,t,n,s)=>{e.fuseConsumer(t,r);let o=e.getLoops(t);o.length>0&&e.parallelize(o[0])})}function WB(r){let e=r.loops.filter(i=>r.reductionLoopVars.has(i.loopVar.name));if(e.length!==1)return null;let t=e[0],n=t.extent&&t.extent.type==="IntImmNode"?t.extent.value:null;if(n===null)return null;let s=pF(n);if(s.length===0)return null;let o=t.loopVar.name;return new vr("rfactor",[new os("rf_factor",s)],(i,a,u,p)=>{i.rfactor(a,o,p.rf_factor);let c=i.getLoops(`${a}_rf_p`);c.length>0&&i.parallelize(c[0])})}function HB(){return new vr("elementwise_cpu",[new os("vector_width",cF)],(r,e,t,n)=>{let s=r.getLoops(e);if(s.length===0)return;if(s.length===1){let a=s[0].extent;if(a.type==="IntImmNode"&&a.value>=n.vector_width*2){let[u,p]=r.split(s[0],n.vector_width);r.parallelize(u),r.vectorize(p)}else r.parallelize(s[0]);return}r.parallelize(s[0]);let o=s[s.length-1],i=o.extent;if(i.type==="IntImmNode"&&i.value>=n.vector_width){let[,a]=r.split(o,n.vector_width);r.vectorize(a)}})}function YB(){return new vr("elementwise_gpu",[new os("block_size",qB)],(r,e,t,n)=>{let s=r.getLoops(e);if(s.length===0)return;let o=s[0];for(let c=1;c<s.length;c++){let f=r.getLoops(e).find(d=>d.loopVar.name===s[c].loopVar.name);f&&o.body===f&&(o=r.fuseLoops(o,f))}let i=o.extent;if(i.type!=="IntImmNode"){r.bindThread(o,"threadIdx.x");return}let a=i.value,u=Math.min(t.maxThreadsPerBlock,1024);if(o0(r.func)&&a<=u){r.bindThread(o,"threadIdx.x");return}let p=Math.min(n.block_size,UB(t));if(a>p){let[c,h]=r.split(o,p);r.bindThread(c,"blockIdx.x"),r.bindThread(h,"threadIdx.x")}else r.bindThread(o,"threadIdx.x")})}function JB(){return new vr("reduction_cpu",[],(r,e,t,n)=>{let s=r.getLoops(e);s.length>0&&r.parallelize(s[0])})}function ZB(){return new vr("reduction_gpu",[new os("block_size",qB)],(r,e,t,n)=>{let s=r.getLoops(e);if(s.length===0)return;let o=dt(r.func,e),i=s.filter(c=>!o||!Ju(c,o));if(i.length===0)return;let a=i[0];for(let c=1;c<i.length;c++){let f=r.getLoops(e).find(d=>d.loopVar.name===i[c].loopVar.name);f&&a.body===f&&(a=r.fuseLoops(a,f))}let u=a.extent;if(u.type!=="IntImmNode"){r.bindThread(a,"threadIdx.x");return}let p=Math.min(n.block_size,UB(t));if(u.value>p){let[c,h]=r.split(a,p);r.bindThread(c,"blockIdx.x"),r.bindThread(h,"threadIdx.x")}else r.bindThread(a,"threadIdx.x")})}var qB,cF,QB=M(()=>{"use strict";ro();wy();qB=[32,64,128,256,512,1024],cF=[1,2,4,8,16];l(UB,"gpuThreadCap");l(pF,"reductionDivisors");l(KB,"createFusedTilingSketch");l(WB,"createRfactorSketch");l(HB,"createElementwiseCPUSketch");l(YB,"createElementwiseGPUSketch");l(JB,"createReductionCPUSketch");l(ZB,"createReductionGPUSketch")});function xy(r){return gm(r,xy,(e,t,n)=>{switch(e.type){case"PrimFunc":return t.name=e.name,t.params=e.params,t.body=n(e.body),t.bufferMap=new Map(e.bufferMap),t.shapeParams=e.shapeParams,t.shapeParamMap=e.shapeParamMap instanceof Map?new Map(e.shapeParamMap):e.shapeParamMap,t._setChild("body",t.body),t;case"AllocateNode":return t.buffer=e.buffer,t.scope=e.scope,t.body=n(e.body),t._setChild("body",t.body),t;case"LetStmtNode":return t.variable=e.variable,t.value=n(e.value),t.body=n(e.body),t._setChild("body",t.body),t;case"WhileNode":return t.condVar=e.condVar,t.condBody=n(e.condBody),t.loopBody=n(e.loopBody),t._setChild("condBody",t.condBody),t._setChild("loopBody",t.loopBody),t;default:for(let s of Object.keys(e)){if(s==="_parent"||s==="_parentKey"||s==="_parentIdx")continue;let o=e[s];o instanceof Map?t[s]=new Map(o):Array.isArray(o)?t[s]=o.map(n):typeof o=="object"&&o!==null&&o.type?t[s]=n(o):t[s]=o}return t}})}function Lo(r){return xy(r)}function Ma(r){return xy(r)}function BT(r){return xy(r)}function XB(r,e,t){let n=t.get(e);if(!n)return null;let s=[],o=n._parent;for(;o&&o!==r;)o.type==="ForNode"&&s.push(o),o=o._parent;s.reverse();let i=BT(n);for(let p=s.length-1;p>=0;p--){let c=s[p],h=new de(c.loopVar,BT(c.min),BT(c.extent),c.kind,i,c.threadTag);h._setChild("body",i),i=h}let a=new Map;for(let p of n.reads)a.set(p.buffer.name,p.buffer);for(let p of n.writes)a.set(p.buffer.name,p.buffer);let u=[];for(let p of r.params)a.has(p.name)&&u.push(p);return new Jo("__tune_"+e,u,i,a,[])}var lf=M(()=>{"use strict";De();Ww();l(xy,"deepCloneIR");l(Lo,"clonePrimFunc");l(Ma,"cloneTensorIR");l(BT,"cloneBlockSubtree");l(XB,"extractBlockMini")});function ET(r,e){let t=[];for(let n=1;n<=e;n*=2)n>=r&&t.push(n);return t}function ff(r){if(!r)return null;let e=[];for(let t of r){if(!t||t.type!=="VariableNode")return null;e.push(t.name)}return e}function fF(r,e){let t=[r];for(;t.length>0;){let n=t.pop();if(!(!n||typeof n!="object")){if(n.type==="BufferStoreNode"&&n.buffer&&n.buffer.name===e&&n.value&&n.value.type==="MathOpNode"&&n.value.op==="+")return n;if(n.body&&t.push(n.body),n.stmts)for(let s of n.stmts)t.push(s);n.thenBody&&t.push(n.thenBody),n.elseBody&&t.push(n.elseBody)}}return null}function pf(r){if(!r||r.broadcastDims||r.offset&&r.offset!==0)return!1;let e=r.shape,t=r.strides;if(!t||t.length!==e.length)return!1;let n=1;for(let s=e.length-1;s>=0;s--){if(typeof e[s]!="number"||e[s]<=0||t[s]!==n)return!1;n*=e[s]}return!0}function LT(r,e){if(!dt(r,e))return null;let n=af(r.body,e);if(!n||n.reads.length<2||n.writes.length<1)return null;let s=n.writes[0].buffer;if(!s||s.shape.length<2)return null;let o=s.shape.length,i=fF(n.body,s.name);if(!i)return null;let a=ff(i.indices);if(!a||a.length!==o)return null;let u=i.value,p=l(T=>T&&T.type==="BufferLoadNode"&&T.buffer&&T.buffer.name===s.name,"isCLoad"),c=p(u.a)?u.b:p(u.b)?u.a:null;if(!c||c.type!=="MathOpNode"||c.op!=="*")return null;let h=[c.a,c.b];if(!h.every(T=>T&&T.type==="BufferLoadNode"&&T.buffer))return null;let f=a[o-1],d=a.slice(0,o-1),m=null,g=null;for(let T of h){let k=ff(T.indices);if(!k||k.length!==o)continue;let A=!0;for(let N=0;N<o-1;N++)if(k[N]!==d[N]){A=!1;break}if(A){m=T,g=k[o-1];break}}if(!m||g==null)return null;let y=h[0]===m?h[1]:h[0],_=ff(y.indices);if(!_)return null;let b=m.buffer,w=y.buffer;if(b.shape.length!==o||b.dtype!=="f32"||w.dtype!=="f32"||s.dtype!=="f32")return null;for(let T=0;T<o-1;T++)if(b.shape[T]!==s.shape[T])return null;let x=s.shape[o-1],S=b.shape[o-1];if(![x,S].every(T=>typeof T=="number"&&T>0))return null;if(_.length===2&&w.shape.length===2){let T;if(_[0]===g&&_[1]===f)T=!1;else if(_[0]===f&&_[1]===g)T=!0;else return null;if(T){if(w.shape[0]!==x||w.shape[1]!==S)return null}else if(w.shape[0]!==S||w.shape[1]!==x)return null;let k=1;for(let A=0;A<o-1;A++){let N=s.shape[A];if(typeof N!="number"||N<=0)return null;k*=N}return o===2?{A:b,B:w,C:s,M:k,N:x,K:S,transB:T,batch:1}:!pf(b)||!pf(s)?null:{A:new me(b.name,[k,S],b.dtype,b.scope),B:w,C:new me(s.name,[k,x],s.dtype,s.scope),M:k,N:x,K:S,transB:T,batch:1}}if(o>=3&&_.length===o&&w.shape.length===o){let T=a.slice(0,o-2);for(let B=0;B<o-2;B++)if(_[B]!==T[B])return null;let k;if(_[o-2]===g&&_[o-1]===f)k=!1;else if(_[o-2]===f&&_[o-1]===g)k=!0;else return null;for(let B=0;B<o-2;B++)if(w.shape[B]!==s.shape[B])return null;if(k){if(w.shape[o-2]!==x||w.shape[o-1]!==S)return null}else if(w.shape[o-2]!==S||w.shape[o-1]!==x)return null;let A=s.shape[o-2];if(typeof A!="number"||A<=0)return null;let N=1;for(let B=0;B<o-2;B++){let E=s.shape[B];if(typeof E!="number"||E<=0)return null;N*=E}if(!pf(b)||!pf(w)||!pf(s))return null;let v=new me(b.name,[N,A,S],b.dtype,b.scope),O=new me(w.name,k?[N,x,S]:[N,S,x],w.dtype,w.scope),R=new me(s.name,[N,A,x],s.dtype,s.scope);return{A:v,B:O,C:R,M:A,N:x,K:S,transB:k,batch:N}}return null}function sR(r,e,t=32){let n=r.maxThreadsPerBlock||1024,s=r.warpSize||32,o=r.sharedMemoryBytes||49152,i=r.registersPerThread||255,a=4,{M:u,N:p,K:c}=e,h=ET(2,8),f=ET(2,s/2),d=ET(s/4,s/2),m=2*s,g=Math.min(n,8*s),y=4*s,_=new Set,b=[];for(let w of h)for(let x of h)if(!(w*x>y)&&!(w*x+w+x+s>i))for(let S of f)for(let T of f){let k=T*S;if(k<m||k>g||k%s!==0)continue;let A=w*S,N=x*T;if(!(A>u||N>p||A>y||N>y))for(let v of d){if(v>c||(A*v+v*N)*a>o)continue;let R=`${A}_${N}_${v}_${w}_${x}`;_.has(R)||(_.add(R),b.push({BM:A,BN:N,BK:v,TM:w,TN:x,threads:k}))}}return b.sort((w,x)=>nR(x,s)-nR(w,s)),b.slice(0,t)}function nR(r,e){let t=r.TM*r.TN,n=-Math.abs(r.TM-r.TN),s=-Math.abs(Math.log2(r.BM)-Math.log2(r.BN)),o=r.threads>=4*e&&r.threads<=8*e?1:0,i=r.BK===e/4?1:0;return o*100+t*4+n*6+s*4+i}function Sy(r,e){if(!e)return null;let t=sR(r,e,256);return t.find(s=>s.BM===64&&s.BN===64&&s.BK===8&&s.TM===4&&s.TN===4)||(t.length>0?t[0]:null)}function hF(r,e,t,n){let s=Ma(r.storeValue);return ti(s,o=>o.type==="BufferLoadNode"&&o.buffer?o.buffer.name===r.inputName?e:r.scalarConsts.has(o.buffer.name)?Ma(r.scalarConsts.get(o.buffer.name)):o:o.type==="VariableNode"?o.name===r.iv0?Ma(t):o.name===r.iv1?Ma(n):o:o)}function CT(r,e,t=null){let{A:n,B:s,C:o,M:i,N:a,K:u,transB:p}=r,c=r.batch||1,{BM:h,BN:f,BK:d,TM:m,TN:g}=e,y=f/g,_=h/m,b=y*_,w=Math.ceil(u/d),x=h*d,S=d*f,T=Math.ceil(x/b),k=Math.ceil(S/b),A=i%h!==0,N=a%f!==0,v=u%d!==0,O=x%b!==0,R=S%b!==0,B=new me("rb_As",[x],"f32","shared"),E=new me("rb_Bs",[S],"f32","shared"),z=new me("rb_acc",[m*g],"f32","local"),D=new me("rb_af",[m],"f32","local"),J=new me("rb_bf",[g],"f32","local"),j=Vt("rb_bx"),U=Vt("rb_by"),P=Vt("rb_tx"),V=Vt("rb_ty"),he=Vt("rb_tid"),le=Vt("rb_brow"),Ie=Vt("rb_bcol"),G=Vt("rb_k0"),K=Vt("rb_bz"),ie=c>1,ze=l((wt,Fs)=>ie?[K,wt,Fs]:[wt,Fs],"gIdx"),Se=l((wt,Fs)=>Pn(Nr(wt,Ze(g)),Fs),"accIdx"),Qe=Vt("rb_im"),It=Vt("rb_in"),Jt=Co(Qe,m,Co(It,g,new Y(z,[Se(Qe,It)],RT()))),Gt=Vt("rb_la"),tt=Vt("rb_aidx"),ye=Pn(le,eR(tt,Ze(d))),He=Pn(G,tR(tt,Ze(d))),gn=new q(n,ze(ye,He));if(A||v){let wt=A?Ls(ye,Ze(i)):null;v&&(wt=wt?MT(wt,Ls(He,Ze(u))):Ls(He,Ze(u))),gn=new xe(wt,gn,RT())}let jt=new Y(B,[tt],gn);O&&(jt=new xe(Ls(tt,Ze(x)),jt));let Zt=IT(Gt,T,new Ye(tt,Pn(he,Nr(Gt,Ze(b))),jt)),Qt=Vt("rb_lb"),on=Vt("rb_bidx"),Jn=Pn(G,eR(on,Ze(f))),Bn=Pn(Ie,tR(on,Ze(f))),yr=new q(s,p?ze(Bn,Jn):ze(Jn,Bn));if(v||N){let wt=v?Ls(Jn,Ze(u)):null;N&&(wt=wt?MT(wt,Ls(Bn,Ze(a))):Ls(Bn,Ze(a))),yr=new xe(wt,yr,RT())}let Zn=new Y(E,[on],yr);R&&(Zn=new xe(Ls(on,Ze(S)),Zn));let pt=IT(Qt,k,new Ye(on,Pn(he,Nr(Qt,Ze(b))),Zn)),Qn=Vt("rb_kk"),mt=Vt("rb_fi"),Or=Vt("rb_fj"),hs=Co(mt,m,new Y(D,[mt],new q(B,[Pn(Nr(Pn(Nr(V,Ze(m)),mt),Ze(d)),Qn)]))),$s=Co(Or,g,new Y(J,[Or],new q(E,[Pn(Nr(Qn,Ze(f)),Pn(Nr(P,Ze(g)),Or))]))),Br=Vt("rb_mi"),Xn=Vt("rb_ni"),_r=Co(Br,m,Co(Xn,g,new Y(z,[Se(Br,Xn)],Pn(new q(z,[Se(Br,Xn)]),Nr(new q(D,[Br]),new q(J,[Xn])))))),fo=Co(Qn,d,new te([hs,$s,_r])),Rr=Vt("rb_kt"),Mr=new Ye(G,Nr(Rr,Ze(d)),new te([Zt,pt,new On,fo,new On])),ds=IT(Rr,w,Mr),Qr=Vt("rb_wm"),Mi=Vt("rb_wn"),Vo=Pn(Pn(le,Nr(V,Ze(m))),Qr),Go=Pn(Pn(Ie,Nr(P,Ze(g))),Mi),Ii=new q(z,[Se(Qr,Mi)]),Xl=t?t.outBuffer:o,Fa=t?hF(t,Ii,Vo,Go):Ii,Ei=new Y(Xl,ze(Vo,Go),Fa);if(A||N){let wt=A?Ls(Vo,Ze(i)):null;N&&(wt=wt?MT(wt,Ls(Go,Ze(a))):Ls(Go,Ze(a))),Ei=new xe(wt,Ei)}let Li=Co(Qr,m,Co(Mi,g,Ei)),ho=new te([Jt,ds,Li]),Ci=new Ve(z,"local",new Ve(D,"local",new Ve(J,"local",ho))),jo=new Ye(he,Pn(Nr(V,Ze(y)),P),new Ye(le,Nr(U,Ze(h)),new Ye(Ie,Nr(j,Ze(f)),Ci))),mo=Math.ceil(a/f),qo=Math.ceil(i/h),Di=cf(U,"blockIdx.y",qo,cf(j,"blockIdx.x",mo,cf(V,"threadIdx.y",_,cf(P,"threadIdx.x",y,jo))));return ie&&(Di=cf(K,"blockIdx.z",c,Di)),new Ve(B,"shared",new Ve(E,"shared",Di))}function dF(r){let e=new os("config_index",r.map((n,s)=>s)),t=new vr("matmul_register_block_gpu",[e],(n,s,o,i)=>{let a=LT(n.func,s);if(!a)return;let u=r[i.config_index];if(!u)return;let p=CT(a,u);n.func.body=p,n.func._setChild&&n.func._setChild("body",p),n.func.gpuRegisterBlocked=!0});return t.configs=r,t.enumerate=()=>r.map((n,s)=>({config_index:s})),t}function mF(r){let e=io(r.body),t=null;for(let s of e){let o=uf(r,s);if(o.hasReduction&&o.spatial>=2&&o.reads>=2){if(t)return null;t=s}}if(!t)return null;let n=LT(r,t);if(!n)return null;for(let s of e){if(s===t)continue;let o=dt(r,s);if(!o||o.hasReduction||o.readBuffers.length>0)return null;for(let i of o.writeBuffers)if(i!==n.C.name)return null}return{reductionBlock:t,dims:n}}function gF(r){let e=new Map,t=new Map;Ss(r,n=>{if(n.type==="BufferStoreNode"&&n.buffer){let s=n.buffer.name;t.set(s,(t.get(s)||0)+1),(!n.indices||n.indices.length===0)&&n.value&&(n.value.type==="FloatImmNode"||n.value.type==="IntImmNode")&&e.set(s,n.value)}});for(let n of[...e.keys()])(t.get(n)||0)!==1&&e.delete(n);return e}function yF(r,e){let t=null;return Ss(r,n=>{if(n.type==="BufferStoreNode"&&n.buffer&&n.buffer.name===e)return t=n,id}),t}function _F(r,e){let t=null,n=0;return Ss(r,s=>{s.type==="BufferLoadNode"&&s.buffer&&s.buffer.name===e&&(n++,t=s)}),n===1?t:null}function oR(r){let e=io(r.body),t=null;for(let y of e){let _=uf(r,y);if(_.hasReduction&&_.spatial>=2&&_.reads>=2){if(t)return null;t=y}}if(!t)return null;let n=LT(r,t);if(!n)return null;let s=n.C.name,o=new Set;for(let y of e){let _=dt(r,y);if(!_)return null;for(let b of _.writeBuffers)o.add(b)}let i=[];for(let y of e){if(y===t)continue;let _=dt(r,y);if(_.hasReduction)return null;if(_.readBuffers.length===0){if(_.writeBuffers.every(b=>b===s))continue;return null}i.push({name:y,info:_})}if(i.length===0)return{reductionBlock:t,dims:n,epilogue:null};if(i.length!==1||(n.batch||1)!==1)return null;let a=i[0];if(a.info.writeBuffers.length!==1)return null;let u=a.info.writeBuffers[0],p=gF(r.body),c=0;for(let y of a.info.readBuffers){if(y===s){c++;continue}if(o.has(y)&&!p.has(y))return null}if(c!==1)return null;let h=af(r.body,a.name);if(!h)return null;let f=yF(h.body,u);if(!f)return null;let d=ff(f.indices);if(!d||d.length!==2)return null;let m=_F(f.value,s);if(!m)return null;let g=ff(m.indices);return!g||g.length!==d.length||g.some((y,_)=>y!==d[_])?null:{reductionBlock:t,dims:n,epilogue:{outBuffer:f.buffer,storeValue:f.value,inputName:s,iv0:d[0],iv1:d[1],scalarConsts:p}}}function iR(r,e,t){let n=mF(r);if(!n)return null;let s=rR.get(r);if(s===void 0){let o=sR(t,n.dims);s=o.length>0?dF(o):null,rR.set(r,s)}return s?e===n.reductionBlock?[s]:[]:null}var Ze,RT,Vt,Pn,Nr,eR,tR,Ls,MT,IT,Co,cf,rR,ky=M(()=>{"use strict";De();xo();ro();wy();Al();Gr();lf();Ze=l(r=>new L(r),"I"),RT=l(()=>new fe(0),"FZERO"),Vt=l(r=>new Ne(r,"i32"),"IV"),Pn=l((r,e)=>In("+",r,e),"ADD"),Nr=l((r,e)=>In("*",r,e),"MUL"),eR=l((r,e)=>In("//",r,e),"DIV"),tR=l((r,e)=>In("%",r,e),"MOD"),Ls=l((r,e)=>new qe("lt",r,e),"LT"),MT=l((r,e)=>new C("&&",r,e),"AND"),IT=l((r,e,t)=>new de(r,Ze(0),Ze(e),F.SERIAL,t),"forS"),Co=l((r,e,t)=>new de(r,Ze(0),Ze(e),F.UNROLLED,t),"forU"),cf=l((r,e,t,n)=>new de(r,Ze(0),Ze(t),F.THREAD_BINDING,n,e),"forT");l(ET,"pow2Range");l(ff,"plainVars");l(fF,"findAccStore");l(pf,"isContiguousRowMajor");l(LT,"matmulTileDims");l(sR,"enumerateRegisterBlockConfigs");l(nR,"goodness");l(Sy,"pickFixedConfig");l(hF,"foldEpilogue");l(CT,"buildRegisterBlockedMatmul");l(dF,"createMatmulRegisterBlockGPUSketch");l(mF,"analyzePureMatmul");l(gF,"collectScalarConstBuffers");l(yF,"findEpilogueStore");l(_F,"singleBufferLoad");l(oR,"analyzeMatmulEpilogue");rR=new WeakMap;l(iR,"richMatmulSketches")});function bF(r,e){let t=[r];for(;t.length;){let n=t.pop();if(!(!n||typeof n!="object")&&(n.type==="BlockNode"&&e(n),n.body&&t.push(n.body),n.initBody&&t.push(n.initBody),n.thenBody&&t.push(n.thenBody),n.elseBody&&t.push(n.elseBody),Array.isArray(n.stmts)))for(let s of n.stmts)t.push(s)}}function df(r){let e=[];bF(r.body,n=>{e.push({name:n.name,node:n,reads:(n.reads||[]).map(s=>s.buffer&&s.buffer.name).filter(Boolean),writes:(n.writes||[]).map(s=>s.buffer&&s.buffer.name).filter(Boolean)})});let t=new Map;for(let n of e)for(let s of n.reads)t.has(s)||t.set(s,[]),t.get(s).push(n.name);return{blocks:e,consumersByBuf:t,byName:new Map(e.map(n=>[n.name,n]))}}function DT(r){let e=[];for(let t of r){if(!t||t.type!=="VariableNode")return null;e.push(t.name)}return e}function hf(r,e,t){if(!r||typeof r!="object")return t;r.type==="BufferLoadNode"&&r.buffer&&r.buffer.name===e&&t.push(r);for(let n of["a","b","expr","value","condition","thenBody","elseBody","body","initBody"])r[n]&&hf(r[n],e,t);if(Array.isArray(r.args))for(let n of r.args)hf(n,e,t);if(Array.isArray(r.indices))for(let n of r.indices)hf(n,e,t);if(Array.isArray(r.stmts))for(let n of r.stmts)hf(n,e,t);return t}function Ol(r,e){if(!r||typeof r!="object")return e;r.type==="BufferLoadNode"&&r.buffer&&r.buffer.name&&e.add(r.buffer.name);for(let t of["a","b","expr","value","condition","thenBody","elseBody","body","initBody"])r[t]&&Ol(r[t],e);if(Array.isArray(r.args))for(let t of r.args)Ol(t,e);if(Array.isArray(r.indices))for(let t of r.indices)Ol(t,e);if(Array.isArray(r.stmts))for(let t of r.stmts)Ol(t,e);return e}function vy(r,e,t,n){let s=e.byName.get(t);if(!s||s.writes.length!==1)return null;let o=s.writes[0],i=[...new Set((e.consumersByBuf.get(o)||[]).filter(w=>w!==t))];if(i.length!==1)return null;let a=i[0],u=n(r,t),p=n(r,a);if(!u||!p||p.hasReduction)return null;let c=e.byName.get(a);if(!c||c.writes.length!==1||c.writes[0]===o)return null;let h=new Set(s.reads);if(Ol(s.node.body,h),Ol(s.node.initBody,h),h.has(c.writes[0]))return null;let f=s.node.body&&s.node.body.type==="BufferStoreNode"?s.node.body:null,d=c.node.body&&c.node.body.type==="BufferStoreNode"?c.node.body:null;if(!f||!d)return null;let m=u.loops.filter(w=>!u.reductionLoopVars.has(w.loopVar.name)).map(w=>w.loopVar.name),g=DT(f.indices);if(!g||g.join(",")!==m.join(","))return null;let y=p.loops.map(w=>w.loopVar.name);if(y.length!==m.length)return null;let _=DT(d.indices);if(!_||_.join(",")!==y.join(","))return null;let b=hf(d.value,o,[]);if(b.length===0)return null;for(let w of b){let x=DT(w.indices);if(!x||x.join(",")!==_.join(","))return null}return a}var PT=M(()=>{"use strict";l(bF,"walkBlocks");l(df,"buildBlockDAG");l(DT,"indexVarNames");l(hf,"collectLoads");l(Ol,"collectLoadedBuffers");l(vy,"findFusibleConsumer")});function $T(r){return r.isGPU()?ZB():JB()}function wF(r){return r.isGPU()?YB():HB()}function TF(r,e,t,n){let s=dt(r,e);if(!s)return[$T(t)];let o=[],i=VB(s,LB(t));if(i&&o.push(i),t.kind===Ot.CPU){let a=GB(s,EB);a&&o.push(a);let u=WB(s);u&&o.push(u);let p=n?vy(r,n,e,dt):null;p&&o.push(KB(p))}return o.push($T(t)),o}function FT(r,{priority:e=100}={}){Ny.some(t=>t.derive===r.derive&&t.matches===r.matches)||(Ny.push({matches:r.matches,derive:r.derive,priority:e}),Ny.sort((t,n)=>t.priority-n.priority))}function aR(r,e,t,n={}){if(n.richGpu&&t.isGPU()){let i=iR(r,e,t);if(i!==null)return i}if(t.kind!==Ot.CPU&&!t.isGPU())return[];let s=uf(r,e),o=n.dag||df(r);for(let i of Ny)if(i.matches(s,t))return i.derive(r,e,t,o);return[]}var Ny,uR=M(()=>{"use strict";Ks();ro();Al();AT();jB();QB();ky();PT();l($T,"reductionSketch");l(wF,"elementwiseSketch");l(TF,"deriveMultiLevel");Ny=[];l(FT,"registerSketchRule");FT({matches:l(r=>r.hasReduction&&r.spatial>=1&&r.reads>=2,"matches"),derive:TF},{priority:10});FT({matches:l(r=>r.hasReduction,"matches"),derive:l((r,e,t)=>[$T(t)],"derive")},{priority:20});FT({matches:l(()=>!0,"matches"),derive:l((r,e,t)=>[wF(t)],"derive")},{priority:30});l(aR,"deriveSketches")});function zT(r,e,t,n,s={}){return aR(r,e,t,s)}var lR=M(()=>{"use strict";uR();l(zT,"getSketchesForBlock")});function Ay(r){let e=new Map,t=[r];for(;t.length>0;){let n=t.pop();if(n){if(n.type==="BlockNode"&&e.set(n.name,n),n.body&&t.push(n.body),n.stmts)for(let s of n.stmts)t.push(s);n.thenBody&&t.push(n.thenBody),n.elseBody&&t.push(n.elseBody),n.initBody&&t.push(n.initBody)}}return e}function Oy(r,e,t,n=null){let s=n||Ay(r.body),o=s.get(e)||null,i=[];if(o){let a=[];for(let p of o.reads)a.push(`${p.buffer.shape.join("x")}:${p.buffer.dtype}`);for(let p of o.writes)a.push(`${p.buffer.shape.join("x")}:${p.buffer.dtype}`);i.push(a.join(","));let u=[];if(Yt(o.body,u),o.initBody&&Yt(o.initBody,u),i.push(u.join(";")),o.writes.length===1){let p=o.writes[0].buffer.name,c=[];for(let h of xF(s).get(p)||[]){if(h===o)continue;let f=[];Yt(h.body,f),h.initBody&&Yt(h.initBody,f),c.push(f.join(";"))}c.length>0&&(c.sort(),i.push(`consumers:${c.join("|")}`))}}return i.push(t.name),i.push(t.kind),SF(i.join("|"))}function xF(r){if(r.__readersByBuffer)return r.__readersByBuffer;let e=new Map;for(let t of r.values()){if(!t.reads)continue;let n=new Set;for(let s of t.reads)s.buffer&&n.add(s.buffer.name);for(let s of n){let o=e.get(s);o||(o=[],e.set(s,o)),o.push(t)}}return r.__readersByBuffer=e,e}function Yt(r,e){if(!(!r||typeof r!="object"))switch(r.type){case"BufferStoreNode":if(e.push("store"),r.indices)for(let t of r.indices)Yt(t,e);Yt(r.value,e);return;case"BufferLoadNode":if(e.push(`load:${r.buffer.shape.join("x")}:${r.buffer.dtype}`),r.indices)for(let t of r.indices)Yt(t,e);return;case"MathOpNode":e.push(`math:${r.op}`),Yt(r.a,e),r.b&&Yt(r.b,e);return;case"CallExternNode":e.push(`call:${r.externName}`);for(let t of r.args)Yt(t,e);return;case"CompareNode":e.push(`cmp:${r.direction}`),Yt(r.a,e),Yt(r.b,e);return;case"CastNode":e.push(`cast:${r.fromDtype}->${r.toDtype}`),Yt(r.expr,e);return;case"IfThenElseNode":Yt(r.condition,e),Yt(r.thenBody,e),r.elseBody&&Yt(r.elseBody,e);return;case"SeqNode":for(let t of r.stmts)Yt(t,e);return;case"ForNode":Yt(r.body,e);return;case"BlockNode":r.initBody&&Yt(r.initBody,e),Yt(r.body,e);return;case"LetStmtNode":Yt(r.value,e),Yt(r.body,e);return;default:return}}function SF(r){let e=2166136261;for(let t=0;t<r.length;t++)e^=r.charCodeAt(t),e=e*16777619>>>0;return e.toString(16).padStart(8,"0")}var VT=M(()=>{"use strict";l(Ay,"buildBlockMap");l(Oy,"computeWorkloadKey");l(xF,"readersByBuffer");l(Yt,"collectBlockOps");l(SF,"fnv1a")});function kF(r,e){let t=r.medianMs!=null?1:0,n=e.medianMs!=null?1:0;return t!==n?n-t:t===1?r.medianMs-e.medianMs:e.score-r.score}var cR,mf,gf,pR=M(()=>{"use strict";VT();cR="mlfw-codegen-1",mf=class{static{l(this,"TuningRecord")}constructor(e,t,n,s,o,i){this.workloadKey=e,this.sketchName=t,this.params=n,this.score=s,this.traceData=o,this.version=i,this.timestamp=Date.now(),this.medianMs=null,this.minMs=null}};l(kF,"rankRecords");gf=class r{static{l(this,"TuningDatabase")}constructor(e=1){this.version=e,this._records=new Map}computeWorkloadKey(e,t,n,s=null){return Oy(e,t,n,s)}store(e,t){let n=this._records.get(e);n||(n=[],this._records.set(e,n)),n.push(t),n.sort(kF),n.length>10&&(n.length=10)}lookup(e){let t=this._records.get(e);return!t||t.length===0?null:t[0]}lookupTopK(e,t=5){let n=this._records.get(e);return n?n.slice(0,t):[]}has(e){return this._records.has(e)&&this._records.get(e).length>0}get size(){let e=0;for(let[,t]of this._records)e+=t.length;return e}serialize(){let e=[];for(let[t,n]of this._records)for(let s of n)e.push({workloadKey:s.workloadKey,sketchName:s.sketchName,params:s.params,score:s.score,traceData:s.traceData,version:s.version,timestamp:s.timestamp,medianMs:s.medianMs,minMs:s.minMs});return{version:this.version,codegenVersion:cR,entries:e}}static deserialize(e){let t=new r(e.version);if(e.codegenVersion!==void 0&&e.codegenVersion!==cR)return t;for(let n of e.entries){let s=new mf(n.workloadKey,n.sketchName,n.params,n.score,n.traceData,n.version);s.timestamp=n.timestamp,s.medianMs=n.medianMs??null,s.minMs=n.minMs??null,t.store(n.workloadKey,s)}return t}saveToFile(e,t){return t.writeFile(e,JSON.stringify(this.serialize())),e}static loadFromFile(e,t){return t.exists(e)?r.deserialize(JSON.parse(t.readFile(e))):new r}clear(){this._records.clear()}}});function Si(r,e){if(!r)return 0;if(r.type==="ForNode"){let t=r.extent&&r.extent.type==="IntImmNode"?r.extent.value:1,n=r.threadTag?e:e*t;return Si(r.body,n)}if(r.type==="BlockNode")return Math.max(e,Si(r.body,e),r.initBody?Si(r.initBody,e):0);if(r.type==="SeqNode"){let t=e;for(let n of r.stmts)t=Math.max(t,Si(n,e));return t}return r.type==="IfThenElseNode"?Math.max(Si(r.thenBody,e),r.elseBody?Si(r.elseBody,e):e):r.type==="AllocateNode"||r.type==="LetStmtNode"?Si(r.body,e):e}function fR(r,e=.1){let t=r.slice().sort((g,y)=>g-y),n=t.length;if(n===0)return{median:0,min:0,trimmedMean:0,cv:0};let s=t[n>>1],o=t[0],i=Math.floor(n*e),a=i,u=n-i>a?n-i:n,p=0,c=0;for(let g=a;g<u;g++)p+=t[g],c++;let h=c>0?p/c:s,f=0;for(let g=a;g<u;g++){let y=t[g]-h;f+=y*y}let d=c>1?Math.sqrt(f/(c-1)):0,m=h>0?d/h:0;return{median:s,min:o,trimmedMean:h,cv:m}}var vF,By,Ry,hR=M(()=>{"use strict";Oc();vF=1e6;l(Si,"maxSerialTripCount");By=class{static{l(this,"BenchmarkResult")}constructor(e,t,n,s,o=null,i=0){this.medianMs=e,this.minMs=t,this.trimmedMeanMs=o??e,this.cv=i,this.samples=n,this._totalBytes=s}get throughputGBs(){return this.minMs<=0||!this._totalBytes?0:this._totalBytes/(this.minMs*1e6)}};l(fR,"robustStats");Ry=class{static{l(this,"BenchmarkRunner")}constructor(e,t={}){this.target=e,this.warmup=t.warmup??3,this.repeat=t.repeat??10,this.minRepeatMs=t.minRepeatMs??0,this.maxCv=t.maxCv??0,this.maxReMeasures=t.maxReMeasures??1,this.measurer=t.measurer||null,this._warn=t.warn||null,this._bufferCache=new Map}_record(e,t){this._warn&&this._warn(e,null,t)}_getOrAllocBuffers(e){let t=0,n=[];for(let[,i]of e.bufferMap){let a=Math.max(i.numel(),1);n.push(a);let u=i.sizeInBytes();u>0&&(t+=u)}let s=n.join(","),o=this._bufferCache.get(s);o||(o=n.map(i=>new Float32Array(i)),this._bufferCache.set(s,o));for(let i of o)for(let a=0;a<i.length;a++)i[a]=Math.random()*2-1;return{buffers:o,totalBytes:t}}run(e){if(!e||!e.body)return null;if(!this.target.isCPU())return this.measurer?this._runMeasured(e):null;let t=new Ws(this.target),n;try{n=t.compile(e)}catch(p){return this._record("benchmark-compile",p),null}if(n.metadata.kind!=="js")return null;let s;try{s=new Function("return "+n.source)()}catch(p){return this._record("benchmark-construct-fn",p),null}let{buffers:o,totalBytes:i}=this._getOrAllocBuffers(e);for(let p=0;p<this.warmup;p++)try{s(...o)}catch(c){return this._record("benchmark-warmup-run",c),null}let a=[],u=null;for(let p=0;p<=this.maxReMeasures&&(this._collect(s,o,a),u=fR(a),!(this.maxCv<=0||u.cv<=this.maxCv));p++);return new By(u.median,u.min,a,i,u.trimmedMean,u.cv)}_runMeasured(e){if(e.shapeParams&&e.shapeParams.length>0||Si(e.body,1)>vF)return null;let t;try{t=new Ws(this.target).compile(e)}catch(a){return this._record("measured-compile",a),null}let n=[],s=0;for(let[,a]of e.bufferMap){let u=Math.max(a.sizeInBytes(),1);n.push(u),s+=u}let o;try{o=this.measurer(t,n,[],{warmup:this.warmup,repeat:this.repeat})}catch(a){return this._record("measurer",a),null}if(!o||o.length===0)return null;let i=fR(o);return new By(i.median,i.min,o,s,i.trimmedMean,i.cv)}_collect(e,t,n){let s=0,o=this.repeat*3;for(let i=0;i<o&&(i<this.repeat||s<this.minRepeatMs);i++){let a=performance.now();e(...t);let u=performance.now()-a;n.push(u),s+=u}}}});var My,dR=M(()=>{"use strict";My=class{static{l(this,"Deadline")}constructor(e=1/0,t=null){this._clock=t||(()=>performance.now()),this._endAt=e===1/0||e==null?1/0:this._clock()+e}get expired(){return this._clock()>=this._endAt}remainingMs(){return this._endAt===1/0?1/0:Math.max(0,this._endAt-this._clock())}}});function mR(r){let e=null;for(let w of r.ops())if(w.opName==="conv"){if(e)return null;e=w}else if(w.opName!=="return")return null;if(!e||(e.getAttr("groups")||1)!==1)return null;let t=e.getAttr("input_layout"),n=e.getAttr("kernel_layout");if(t&&t!=="NCHW"||n&&n!=="OIHW")return null;let s=e.getOperand(0).type,o=e.getOperand(1).type,i=e.getResult(0).type;if(!s||!o||!i||s.shape.length!==4||o.shape.length!==4||i.shape.length!==4||s.dtype!=="f32"||o.dtype!=="f32"||i.dtype!=="f32")return null;let a=e.getAttr("strides")||[1,1],u=e.getAttr("padding")||[[0,0],[0,0]],p=e.getAttr("dilation")||[1,1],c=s.shape[0],h=s.shape[1],f=s.shape[2],d=s.shape[3],m=o.shape[0],g=o.shape[2],y=o.shape[3],_=i.shape[2],b=i.shape[3];return![c,h,f,d,m,g,y,_,b].every(w=>typeof w=="number"&&w>0)||o.shape[1]!==h||i.shape[1]!==m||i.shape[0]!==c?null:{N:c,Cin:h,H:f,W:d,O:m,Kh:g,Kw:y,Oh:_,Ow:b,sH:a[0],sW:a[1],pH:u[0][0],pW:u[1][0],dH:p[0],dW:p[1]}}function NF(r,e,t){let{weight:n,input:s,output:o}=r,{N:i,Cin:a,H:u,W:p,O:c,Kh:h,Kw:f,Oh:d,Ow:m,sH:g,sW:y,pH:_,pW:b,dH:w,dW:x}=e,{BM:S,BN:T,BK:k,TM:A,TN:N}=t,v=c,O=i*d*m,R=a*h*f,B=h*f,E=d*m,z=T/N,D=S/A,J=z*D,j=Math.ceil(R/k),U=S*k,P=k*T,V=Math.ceil(U/J),he=Math.ceil(P/J),le=new me("ig_As",[U],"f32","shared"),Ie=new me("ig_Bs",[P],"f32","shared"),G=new me("ig_acc",[A*N],"f32","local"),K=new me("ig_af",[A],"f32","local"),ie=new me("ig_bf",[N],"f32","local"),ze=Ae("ig_bx"),Se=Ae("ig_by"),Qe=Ae("ig_tx"),It=Ae("ig_ty"),Jt=Ae("ig_tid"),Gt=Ae("ig_brow"),tt=Ae("ig_bcol"),ye=Ae("ig_k0"),He=l((Cr,Q_)=>we(Te(Cr,$(N)),Q_),"accIdx"),gn=Ae("ig_im"),jt=Ae("ig_in"),Zt=mn(gn,A,mn(jt,N,new Y(G,[He(gn,jt)],yf()))),Qt=v%S!==0,on=R%k!==0,Jn=O%T!==0,Bn=U%J!==0,yr=P%J!==0,Zn=Ae("ig_la"),pt=Ae("ig_aidx"),Qn=we(Gt,kn(pt,$(k))),mt=we(ye,sn(pt,$(k))),Or=kn(mt,$(B)),hs=sn(mt,$(B)),$s=kn(hs,$(f)),Br=sn(hs,$(f)),Xn=new q(n,[Qn,Or,$s,Br]);if(Qt||on){let Cr=Qt?dr(Qn,$(v)):null;on&&(Cr=Cr?ao(Cr,dr(mt,$(R))):dr(mt,$(R))),Xn=new xe(Cr,Xn,yf())}let _r=new Y(le,[pt],Xn);Bn&&(_r=new xe(dr(pt,$(U)),_r));let fo=Iy(Zn,V,new Ye(pt,we(Jt,Te(Zn,$(J))),_r)),Rr=Ae("ig_lb"),Mr=Ae("ig_bidx"),ds=we(ye,kn(Mr,$(T))),Qr=we(tt,sn(Mr,$(T))),Mi=kn(ds,$(B)),Vo=sn(ds,$(B)),Go=kn(Vo,$(f)),Ii=sn(Vo,$(f)),Xl=kn(Qr,$(E)),Fa=sn(Qr,$(E)),Ei=kn(Fa,$(m)),Li=sn(Fa,$(m)),ho=Ey(we(Te(Ei,$(g)),Te(Go,$(w))),$(_)),Ci=Ey(we(Te(Li,$(y)),Te(Ii,$(x))),$(b)),jo=new q(s,[Xl,Mi,ho,Ci]),mo=ao(ao(Ly(ho,$(0)),dr(ho,$(u))),ao(Ly(Ci,$(0)),dr(Ci,$(p))));on&&(mo=ao(mo,dr(ds,$(R)))),Jn&&(mo=ao(mo,dr(Qr,$(O)))),jo=new xe(mo,jo,yf());let qo=new Y(Ie,[Mr],jo);yr&&(qo=new xe(dr(Mr,$(P)),qo));let Di=Iy(Rr,he,new Ye(Mr,we(Jt,Te(Rr,$(J))),qo)),wt=Ae("ig_kk"),Fs=Ae("ig_fi"),ec=Ae("ig_fj"),q_=mn(Fs,A,new Y(K,[Fs],new q(le,[we(Te(we(Te(It,$(A)),Fs),$(k)),wt)]))),U_=mn(ec,N,new Y(ie,[ec],new q(Ie,[we(Te(wt,$(T)),we(Te(Qe,$(N)),ec))]))),za=Ae("ig_mi"),Va=Ae("ig_ni"),K_=mn(za,A,mn(Va,N,new Y(G,[He(za,Va)],we(new q(G,[He(za,Va)]),Te(new q(K,[za]),new q(ie,[Va])))))),W_=mn(wt,k,new te([q_,U_,K_])),ms=Ae("ig_kt"),Xr=new Ye(ye,Te(ms,$(k)),new te([fo,Di,new On,W_,new On])),br=Iy(ms,j,Xr),er=Ae("ig_wm"),Ir=Ae("ig_wn"),Uo=we(we(Gt,Te(It,$(A))),er),zs=we(we(tt,Te(Qe,$(N))),Ir),Er=kn(zs,$(E)),Lr=sn(zs,$(E)),go=kn(Lr,$(m)),Fn=sn(Lr,$(m)),Ko=new Y(o,[Er,Uo,go,Fn],new q(G,[He(er,Ir)]));if(Qt||Jn){let Cr=Qt?dr(Uo,$(v)):null;Jn&&(Cr=Cr?ao(Cr,dr(zs,$(O))):dr(zs,$(O))),Ko=new xe(Cr,Ko)}let yo=mn(er,A,mn(Ir,N,Ko)),H_=new te([Zt,br,yo]),nh=new Ve(G,"local",new Ve(K,"local",new Ve(ie,"local",H_))),Y_=new Ye(Jt,we(Te(It,$(z)),Qe),new Ye(Gt,Te(Se,$(S)),new Ye(tt,Te(ze,$(T)),nh))),J_=Math.ceil(O/T),Z_=Math.ceil(v/S),rh=ki(Se,"blockIdx.y",Z_,ki(ze,"blockIdx.x",J_,ki(It,"threadIdx.y",D,ki(Qe,"threadIdx.x",z,Y_))));return new Ve(le,"shared",new Ve(Ie,"shared",rh))}function gR(r,e,t){if(!yR(r,{M:e.O,N:e.N*e.Oh*e.Ow,K:e.Cin*e.Kh*e.Kw},t))return!1;let{BM:n,BN:s,BK:o,TM:i,TN:a}=t,u=e.O,p=e.N*e.Oh*e.Ow,c=e.Cin*e.Kh*e.Kw;if(u%n!==0||p%s!==0||c%o!==0||[n,s,o,i,a].some(f=>f%GT!==0))return!1;let h=n/i*(s/a);return!(n*o%(GT*h)!==0||o*s%h!==0)}function AF(r,e,t){let{weight:n,input:s,output:o}=r,{N:i,Cin:a,H:u,W:p,O:c,Kh:h,Kw:f,Oh:d,Ow:m,sH:g,sW:y,pH:_,pW:b,dH:w,dW:x}=e,{BM:S,BN:T,BK:k,TM:A,TN:N}=t,v=GT,O=c,R=i*d*m,B=a*h*f,E=h*f,z=d*m,D=u*p,J=a*u*p,j=T/N,U=S/A,P=j*U,V=B/k,he=S*k,le=k*T,Ie=he/P,G=le/P,K=Ie/v,ie=k/v,ze=(d-1)*g+(h-1)*w-_,Se=(m-1)*y+(f-1)*x-b,Qe=_>0||ze>=u,It=b>0||Se>=p,Jt=new me(n.name,[O*B>0?O*B:1],n.dtype,n.scope),Gt=new me(s.name,[i*J>0?i*J:1],s.dtype,s.scope),tt=new me("iv_As",[2*he],"f32","shared");tt.align16=!0;let ye=new me("iv_Bs",[2*le],"f32","shared");ye.align16=!0;let He=new me("iv_acc",[A*N],"f32","local"),gn=new me("iv_af",[A],"f32","local");gn.align16=!0;let jt=new me("iv_bf",[N],"f32","local");jt.align16=!0;let Zt=new me("iv_ra",[Ie],"f32","local");Zt.align16=!0;let Qt=new me("iv_rb",[G],"f32","local"),on=Ae("iv_bx"),Jn=Ae("iv_by"),Bn=Ae("iv_tx"),yr=Ae("iv_ty"),Zn=Ae("iv_tid"),pt=Ae("iv_brow"),Qn=Ae("iv_bcol"),mt=l((ms,Xr)=>we(Te(ms,$(N)),Xr),"accIdx"),Or=0,hs=Ae("iv_im"),$s=Ae("iv_in"),Br=mn(hs,A,mn($s,N,new Y(He,[mt(hs,$s)],yf()))),Xn=l(ms=>{let Xr=Or++,br=Ae("iv_la"+Xr),er=Ae("iv_c"+Xr),Ir=kn(er,$(ie)),Uo=Te(sn(er,$(ie)),$(v)),zs=mn(br,K,new Ye(er,we(Zn,Te(br,$(P))),new iu(Zt,Te(br,$(v)),Jt,we(Te(we(pt,Ir),$(B)),we(ms,Uo)),v))),Er=Ae("iv_lb"+Xr),Lr=Ae("iv_e"+Xr),go=kn(Lr,$(T)),Fn=sn(Lr,$(T)),Ko=we(Qn,Fn),yo=we(ms,go),H_=kn(yo,$(E)),nh=sn(yo,$(E)),Y_=kn(nh,$(f)),J_=sn(nh,$(f)),Z_=kn(Ko,$(z)),rh=sn(Ko,$(z)),Cr=kn(rh,$(m)),Q_=sn(rh,$(m)),X_=Ey(we(Te(Cr,$(g)),Te(Y_,$(w))),$(_)),eb=Ey(we(Te(Q_,$(y)),Te(J_,$(x))),$(b)),_E=we(we(we(Te(Z_,$(J)),Te(H_,$(D))),Te(X_,$(p))),eb),tb=new q(Gt,[_E]),Ga=null;if(Qe&&(Ga=ao(Ly(X_,$(0)),dr(X_,$(u)))),It){let xS=ao(Ly(eb,$(0)),dr(eb,$(p)));Ga=Ga?ao(Ga,xS):xS}Ga&&(tb=new xe(Ga,tb,yf()));let bE=mn(Er,G,new Ye(Lr,we(Zn,Te(Er,$(P))),new Y(Qt,[Er],tb)));return new te([zs,bE])},"prefetch"),_r=l((ms,Xr)=>{let br=Or++,er=Ae("iv_la"+br),Ir=Ae("iv_c"+br),Uo=kn(Ir,$(ie)),zs=Te(sn(Ir,$(ie)),$(v)),Er=[];for(let yo=0;yo<v;yo++)Er.push(new Y(tt,[we(ms,we(Te(we(zs,$(yo)),$(S)),Uo))],new q(Zt,[we(Te(er,$(v)),$(yo))])));let Lr=mn(er,K,new Ye(Ir,we(Zn,Te(er,$(P))),new te(Er))),go=Ae("iv_lb"+br),Fn=Ae("iv_e"+br),Ko=mn(go,G,new Ye(Fn,we(Zn,Te(go,$(P))),new Y(ye,[we(Xr,Fn)],new q(Qt,[go]))));return new te([Lr,Ko])},"commit"),fo=l((ms,Xr)=>{let br=Or++,er=Ae("iv_kk"+br),Ir=[],Uo=we(ms,we(Te(er,$(S)),Te(yr,$(A))));for(let Fn=0;Fn<A/v;Fn++)Ir.push(new iu(gn,$(Fn*v),tt,we(Uo,$(Fn*v)),v));let zs=we(Xr,we(Te(er,$(T)),Te(Bn,$(N))));for(let Fn=0;Fn<N/v;Fn++)Ir.push(new iu(jt,$(Fn*v),ye,we(zs,$(Fn*v)),v));let Er=Ae("iv_mi"+br),Lr=Ae("iv_ni"+br),go=mn(Er,A,mn(Lr,N,new Y(He,[mt(Er,Lr)],we(new q(He,[mt(Er,Lr)]),Te(new q(gn,[Er]),new q(jt,[Lr]))))));return mn(er,k,new te([...Ir,go]))},"computeMMA"),Rr=Ae("iv_kt"),Mr=Ae("iv_p"),ds=Ae("iv_pN"),Qr=Te(Mr,$(he)),Mi=Te(Mr,$(le)),Vo=Te(ds,$(he)),Go=Te(ds,$(le)),Ii=dr(Rr,$(V-1)),Xl=new te([Xn($(0)),_r($(0),$(0)),new On]),Fa=new Ye(Mr,sn(Rr,$(2)),new Ye(ds,sn(we(Rr,$(1)),$(2)),new te([new xe(Ii,Xn(Te(we(Rr,$(1)),$(k)))),fo(Qr,Mi),new xe(Ii,new te([_r(Vo,Go),new On]))]))),Ei=Iy(Rr,V,Fa),Li=Ae("iv_wm"),ho=Ae("iv_wn"),Ci=we(we(pt,Te(yr,$(A))),Li),jo=we(we(Qn,Te(Bn,$(N))),ho),mo=kn(jo,$(z)),qo=sn(jo,$(z)),Di=kn(qo,$(m)),wt=sn(qo,$(m)),Fs=new Y(o,[mo,Ci,Di,wt],new q(He,[mt(Li,ho)])),ec=mn(Li,A,mn(ho,N,Fs)),q_=new te([Br,Xl,Ei,ec]),U_=new Ve(He,"local",new Ve(gn,"local",new Ve(jt,"local",new Ve(Zt,"local",new Ve(Qt,"local",q_))))),za=new Ye(Zn,we(Te(yr,$(j)),Bn),new Ye(pt,Te(Jn,$(S)),new Ye(Qn,Te(on,$(T)),U_))),Va=R/T,K_=O/S,W_=ki(Jn,"blockIdx.y",K_,ki(on,"blockIdx.x",Va,ki(yr,"threadIdx.y",U,ki(Bn,"threadIdx.x",j,za))));return new Ve(tt,"shared",new Ve(ye,"shared",W_))}function yR(r,e,t){let{BM:n,BN:s,BK:o,TM:i,TN:a}=t;if(![n,s,o,i,a].every(m=>typeof m=="number"&&m>0)||n%i!==0||s%a!==0)return!1;let u=s/a,p=n/i,c=u*p,h=r.warpSize||32;return!(c%h!==0||c>(r.maxThreadsPerBlock||1024)||(n*o+o*s)*4*2>(r.sharedMemoryBytes||49152)||i*a+i+a+h>(r.registersPerThread||255))}function BF(r,e){for(let t of OF)if(gR(r,e,t))return t;return null}function _R(r,e,t){let n=r.func,s=n.convInfo;if(!s)return!1;let i=io(n.body).find(_=>/^conv_acc_/.test(_));if(!i)return!1;let a=af(n.body,i);if(!a||a.reads.length<2||a.writes.length<1)return!1;let u=a.reads[0].buffer,p=a.reads[1].buffer,c=a.writes[0].buffer;if(!u||!p||!c)return!1;let h=s.O,f=s.N*s.Oh*s.Ow,d=s.Cin*s.Kh*s.Kw;if(d<128||f<64)return!1;let m=t&&t.convConfig,g=t&&t.convNoVec,y=null;if(!g){let _=m?gR(e,s,m)?m:null:BF(e,s);_&&(y=AF({weight:p,input:u,output:c},s,_))}if(!y){let _=m?yR(e,{M:h,N:f,K:d},m)?m:null:Sy(e,{M:h,N:f,K:d});if(!_)return!1;y=NF({weight:p,input:u,output:c},s,_)}return r.func.body=y,r.func._setChild&&r.func._setChild("body",y),r.func.gpuRegisterBlocked=!0,!0}var $,yf,Ae,we,Ey,Te,kn,sn,dr,Ly,ao,Iy,mn,ki,GT,OF,jT=M(()=>{"use strict";De();xo();Al();ky();$=l(r=>new L(r),"I"),yf=l(()=>new fe(0),"FZERO"),Ae=l(r=>new Ne(r,"i32"),"IV"),we=l((r,e)=>new C("+",r,e),"ADD"),Ey=l((r,e)=>new C("-",r,e),"SUB"),Te=l((r,e)=>new C("*",r,e),"MUL"),kn=l((r,e)=>new C("//",r,e),"DIV"),sn=l((r,e)=>new C("%",r,e),"MOD"),dr=l((r,e)=>new qe("lt",r,e),"LT"),Ly=l((r,e)=>new qe("ge",r,e),"GE"),ao=l((r,e)=>new C("&&",r,e),"AND"),Iy=l((r,e,t)=>new de(r,$(0),$(e),F.SERIAL,t),"forS"),mn=l((r,e,t)=>new de(r,$(0),$(e),F.UNROLLED,t),"forU"),ki=l((r,e,t,n)=>new de(r,$(0),$(t),F.THREAD_BINDING,n,e),"forT");l(mR,"detectPureConv");l(NF,"buildImplicitGemmConv");GT=4;l(gR,"vectorizableConvConfig");l(AF,"buildVectorizedImplicitGemmConv");l(yR,"validConvConfig");OF=[{BM:128,BN:64,BK:8,TM:8,TN:8},{BM:64,BN:64,BK:8,TM:8,TN:8},{BM:64,BN:64,BK:8,TM:4,TN:8},{BM:64,BN:32,BK:8,TM:8,TN:8},{BM:32,BN:64,BK:8,TM:4,TN:8}];l(BF,"pickVectorizedConvConfig");l(_R,"applyImplicitGemmConv")});function bR(r,e,t,n,s,o,i,a){let u=new me(r,[t,n],e.dtype,"shared"),p=new Y(u,[s,o],new q(e,[i,a]));return{tile:u,fill:p}}function IF(r,e,t){let{A:n,B:s,C:o,M:i,N:a,K:u,transB:p}=r,c=u/t,h=Bl("ts_by"),f=Bl("ts_bx"),d=Bl("ts_ty"),m=Bl("ts_tx"),g=Bl("ts_ko"),y=Bl("ts_ki"),_=new me("ts_acc",[1],"f32","local"),b=Rl(Cy(h,Jr(e)),d),w=Rl(Cy(f,Jr(e)),m),x=Cy(g,Jr(t)),S=bR("ts_As",n,e,t,d,m,b,Rl(x,m)),T=bR("ts_Bs",s,t,e,d,m,p?w:Rl(x,d),p?Rl(x,d):w),k=MF(y,t,new Y(_,[Jr(0)],Rl(new q(_,[Jr(0)]),Cy(new q(S.tile,[d,y]),new q(T.tile,[y,m]))))),A=new te([S.fill,T.fill,new On,k,new On]),N=RF(g,c,A),v=new Y(_,[Jr(0)],new fe(0)),O=new Y(o,[b,w],new q(_,[Jr(0)])),R=new Ve(_,"local",new te([v,N,O]));return Dy(h,"blockIdx.y",i/e,Dy(f,"blockIdx.x",a/e,new Ve(S.tile,"shared",new Ve(T.tile,"shared",Dy(d,"threadIdx.y",e,Dy(m,"threadIdx.x",e,R))))))}function EF(r,e){let{M:t,N:n,K:s}=e,o=16,i=16;return o*o>(r.maxThreadsPerBlock||1024)||(o*i+i*o)*4>(r.sharedMemoryBytes||49152)||t%o!==0||n%o!==0||s%i!==0||t<o||n<o||s<i?null:{BS:o,BK:i}}function LF(r,e){let n=io(r.func.body).filter(o=>/^q?conv_(init|acc)_/.test(o));if(n.length===0)return!1;let s=new Rs(e);for(let o of n)s.applyToBlock(r,o);return!0}function CF(r,e,t={}){if(!e.isGPU())return!1;let n=oR(r.func);if(!n)return!1;let s=n.dims;if(t&&t.primitiveMatmul&&(s.batch||1)===1&&!n.epilogue){let a=EF(e,s);if(a){let u=IF(s,a.BS,a.BK);return r.func.body=u,r.func._setChild&&r.func._setChild("body",u),r.func.gpuRegisterBlocked=!0,!0}}let o=Sy(e,s);if(!o)return!1;let i=CT(s,o,n.epilogue);return r.func.body=i,r.func._setChild&&r.func._setChild("body",i),r.func.gpuRegisterBlocked=!0,!0}function Py(r,e,t={}){if(!e.isGPU()||e.isWebGPU&&e.isWebGPU())return!1;let n=CF(r,e,t);return n||(n=_R(r,e,t)),n||(n=LF(r,e)),n}var Jr,Bl,Rl,Cy,RF,MF,Dy,qT=M(()=>{"use strict";ky();Al();jT();ro();De();xo();Jr=l(r=>new L(r),"I"),Bl=l(r=>new Ne(r,"i32"),"IV"),Rl=l((r,e)=>new C("+",r,e),"ADD"),Cy=l((r,e)=>new C("*",r,e),"MUL"),RF=l((r,e,t)=>new de(r,Jr(0),Jr(e),F.SERIAL,t),"forS"),MF=l((r,e,t)=>new de(r,Jr(0),Jr(e),F.UNROLLED,t),"forU"),Dy=l((r,e,t,n)=>new de(r,Jr(0),Jr(t),F.THREAD_BINDING,n,e),"forT");l(bR,"stageTileToShared");l(IF,"buildTiledSharedMatmul");l(EF,"pickTiledConfig");l(LF,"applyDeterministicGpuConv");l(CF,"applyDeterministicGpuMatmul");l(Py,"applyDeterministicGpuSchedule")});function UT(r){return r*1664525+1013904223&2147483647}function PF(r,e,t){let n={...r};for(let s of e.variables)t.rngFloat()<t.mutationRate&&(n[s.name]=s.sample(o=>t.rng(o)));return n}function wR(r={}){return r.strategy==="random"?new KT({numTrials:r.numTrials,seed:r.seed,deadline:r.deadline}):new WT({populationSize:r.populationSize,numGenerations:r.numGenerations,mutationRate:r.mutationRate,eliteRatio:r.eliteRatio,seed:r.seed,deadline:r.deadline})}var $y,KT,WT,DF,TR=M(()=>{"use strict";l(UT,"nextLcg");$y=class{static{l(this,"SearchCandidate")}constructor(e,t,n){this.sketchName=e,this.params=t,this.score=n}},KT=class{static{l(this,"RandomSearch")}constructor(e={}){this.numTrials=e.numTrials||64,this.seed=e.seed||42,this.deadline=e.deadline||null,this._rngState=this.seed}_rng(e){return this._rngState=UT(this._rngState),this._rngState%e}_expired(){return this.deadline?this.deadline.expired:!1}search(e,t){let n=[];for(let s of e){if(this._expired())break;for(let o=0;o<this.numTrials&&!this._expired();o++){let i=s.sampleParams(u=>this._rng(u)),a=t(s,i);a&&n.push(new $y(s.name,i,a.score))}}return n.sort((s,o)=>o.score-s.score),{candidates:n,population:null}}},WT=class{static{l(this,"EvolutionarySearch")}constructor(e={}){this.populationSize=e.populationSize||32,this.numGenerations=e.numGenerations||10,this.mutationRate=e.mutationRate||.3,this.eliteRatio=e.eliteRatio||.2,this.seed=e.seed||42,this.deadline=e.deadline||null,this._rngState=this.seed}_rng(e){return this._rngState=UT(this._rngState),this._rngState%e}_rngFloat(){return this._rngState=UT(this._rngState),this._rngState/2147483647}_expired(){return this.deadline?this.deadline.expired:!1}search(e,t,n=null){let s=n&&n.length?n:this._initPopulation(e),o=new Map,i=l((u,p)=>{let c=u.name+"|"+JSON.stringify(p);if(o.has(c))return o.get(c);let h=t(u,p);return o.set(c,h),h},"evalMemo");for(let u=0;u<this.numGenerations&&!this._expired();u++){let p=[];for(let d of s){let m=i(d.sketch,d.params);m&&p.push({...d,score:m.score})}if(p.sort((d,m)=>m.score-d.score),p.length===0)break;let c=Math.max(1,Math.floor(p.length*this.eliteRatio)),h=p.slice(0,c),f=h.map(d=>({sketch:d.sketch,params:{...d.params}}));for(;f.length<this.populationSize;){let d=h[this._rng(h.length)],m=h[this._rng(h.length)];if(d.sketch!==m.sketch){let g=this._rngFloat()<.5?d:m;f.push(this._mutate(g.sketch,{...g.params}))}else f.push(this._mutate(d.sketch,this._crossover(d,m)))}s=f}let a=[];for(let u of s){let p=i(u.sketch,u.params);p&&a.push(new $y(u.sketch.name,u.params,p.score))}return a.sort((u,p)=>p.score-u.score),{candidates:a,population:s}}_initPopulation(e){let t=[];for(let n=0;n<this.populationSize;n++){let s=e[this._rng(e.length)],o=s.sampleParams(i=>this._rng(i));t.push({sketch:s,params:o})}return t}_crossover(e,t){if(e.sketch!==t.sketch)return{...e.params};let n={};for(let s of e.sketch.variables)n[s.name]=this._rngFloat()<.5?e.params[s.name]:t.params[s.name];return n}_mutate(e,t){let n={rngFloat:l(()=>this._rngFloat(),"rngFloat"),rng:l(i=>this._rng(i),"rng"),mutationRate:this.mutationRate},s=[PF,...DF],o=t;for(let i of s)o=i(o,e,n);return{sketch:e,params:o}}},DF=[];l(PF,"defaultResampleMutator");l(wR,"createSearchStrategy")});function zy(r){let e={"threadIdx.x":1,"threadIdx.y":1,"threadIdx.z":1},t=new Set,n=l(s=>{if(!(!s||typeof s!="object"||t.has(s))){if(t.add(s),s.type==="ForNode"&&e[s.threadTag]!==void 0){let o=s.extent&&s.extent.type==="IntImmNode"?s.extent.value:1;o>e[s.threadTag]&&(e[s.threadTag]=o)}for(let o in s){let i=s[o];Array.isArray(i)?i.forEach(n):i&&typeof i=="object"&&n(i)}}},"visit");return n(r.body),$F.reduce((s,o)=>s*e[o],1)}var $F,Fy,xR=M(()=>{"use strict";fa();Gw();xT();lf();TR();vT();$F=["threadIdx.x","threadIdx.y","threadIdx.z"];l(zy,"gpuThreadBlockSize");Fy=class{static{l(this,"BlockTuningSession")}constructor(e){this.target=e.target,this.primFunc=e.primFunc,this.blockName=e.blockName,this.sketches=e.sketches,this.benchmarkRunner=e.benchmarkRunner||null,this.config=e.config,this.deadline=e.deadline||null,this._warn=e.warn||(()=>{}),this._warnedEvalSketches=new Set;let t=this.sketches.some(o=>o.name==="fused");t?(this.learnedModel=new vl,this.costModel=new Nl(e.costModel.analytical,this.learnedModel)):(this.costModel=e.costModel,this.learnedModel=e.learnedModel);let n=t?null:XB(e.primFunc,e.blockName,e.blockMap);this.evalFunc=n||e.primFunc,this.evalBlockName=e.blockName,this.sketchByName=new Map;for(let o of this.sketches)this.sketchByName.set(o.name,o);let s=this.sketches.length===1&&typeof this.sketches[0].enumerate=="function";this.enumSketch=s?this.sketches[0]:null,this.enumParams=s?this.sketches[0].enumerate():null,this.strategy=wR({...this.config,deadline:this.deadline}),this.population=null,this._best=null,this.plateaued=!1}runRound(){let e=this._best?this._best.measuredScore:-1/0,t=this._produceCandidates();if(t.length===0)return this.plateaued=!0,0;if(this.benchmarkRunner)this._measureAndLearn(t);else{let s=t[0];this._consider({sketchName:s.sketchName,params:s.params,score:s.score,measuredScore:s.score}),this.plateaued=!0}let n=this._best?this._best.measuredScore:-1/0;return Math.max(0,n-e)}best(){return this._best?{sketchName:this._best.sketchName,params:this._best.params,score:this._best.score,medianMs:this._best.medianMs??null,minMs:this._best.minMs??null}:null}bestTrace(){if(!this._best)return null;let e=this.sketchByName.get(this._best.sketchName);if(!e)return null;try{let t=new Sn(Lo(this.primFunc));return e.instantiate(this._best.params)(t,this.blockName,this.target),t.trace.serialize()}catch(t){return this._warn("best-trace",this.blockName,t),null}}_produceCandidates(){if(this.enumSketch){let s=[];for(let o of this.enumParams){let i=this._evaluate(this.enumSketch,o);i&&s.push({sketchName:this.enumSketch.name,params:o,score:i.score})}return s.sort((o,i)=>i.score-o.score),s}let e=l((s,o)=>this._evaluate(s,o),"evaluator"),{candidates:t,population:n}=this.strategy.search(this.sketches,e,this.population);return this.population=n,t}_evaluate(e,t){try{let n=Lo(this.evalFunc),s=new Sn(n);if(e.instantiate(t)(s,this.evalBlockName,this.target),Ku.validate(n).length>0)return null;let i=this.target.maxThreadsPerBlock;return this.target.isGPU&&this.target.isGPU()&&i&&zy(n)>i?null:{score:this.costModel.score(n)}}catch(n){return this._warnedEvalSketches.has(e.name)||(this._warnedEvalSketches.add(e.name),this._warn("evaluate-candidate",this.blockName,n)),null}}_measureAndLearn(e){let t=e.slice(0,this.config.topKForBenchmark);for(let n of t){if(this.deadline&&this.deadline.expired)break;let s=this._measure(n);if(!s)continue;let o=-s.result.medianMs;this.learnedModel.addSample(s.features,o),this._consider({sketchName:n.sketchName,params:n.params,score:o,measuredScore:o,medianMs:s.result.medianMs,minMs:s.result.minMs})}this.learnedModel.train()}_measure(e){let t=this.sketchByName.get(e.sketchName);if(!t)return null;let n,s;try{n=Lo(this.primFunc),t.instantiate(e.params)(new Sn(n),this.blockName,this.target),s=Lo(this.evalFunc),t.instantiate(e.params)(new Sn(s),this.evalBlockName,this.target)}catch(i){return this._warn("measure-candidate",this.blockName,i),null}let o=this.benchmarkRunner.run(n);return o?{result:o,features:Ra.extractStatements(s)}:null}_consider(e){(!this._best||e.measuredScore>this._best.measuredScore)&&(this._best=e)}}});var HT,Vy,SR=M(()=>{"use strict";HT=class{static{l(this,"GradientSchedulerPolicy")}pick(e){let t=e.filter(i=>!i.plateaued);if(t.length===0)return null;let n=t.filter(i=>i.rounds===0);if(n.length>0)return n[0];let s=null,o=-1/0;for(let i of t){let a=i.gainEwma!==void 0?i.gainEwma:i.lastGain,u=i.weight*a;u>o&&(o=u,s=i)}return s}},Vy=class{static{l(this,"TaskScheduler")}constructor(e=null){this.policy=e||new HT}run(e,t,n={}){let s=n.maxRoundsPerTask??8,o=n.plateauPatience??2,i=n.gainEwmaAlpha??.5;for(let a of e)a.rounds=0,a.lastGain=0,a.gainEwma=0,a.stale=0,a.plateaued=!1;for(;!(t&&t.expired);){let a=this.policy.pick(e);if(!a)break;let u=a.session.runRound();a.rounds++,a.lastGain=u,a.gainEwma=i*u+(1-i)*a.gainEwma,u<=0?a.stale++:a.stale=0,(a.session.plateaued||a.stale>=o||a.rounds>=s)&&(a.plateaued=!0)}}}});function FF(r){if(r.isCPU())return null;let e=$v(r.kind);if(!e)throw new Error("hardwareMeasure requested for target '"+r.kind+"' but no measurer is registered for it; the corresponding runtime must be loaded (Node: import '#io/cuda_runtime') before compiling");return e}var Gy,jy,kR=M(()=>{"use strict";fa();vT();lR();pR();hR();dR();VT();Al();PT();ro();qT();xR();lf();De();SR();rw();l(FF,"resolveMeasurer");Gy=class{static{l(this,"AutotuneConfig")}constructor(e={}){this.strategy=e.strategy||"evolutionary",this.numTrials=e.numTrials||64,this.populationSize=e.populationSize||32,this.numGenerations=e.numGenerations||10,this.mutationRate=e.mutationRate,this.eliteRatio=e.eliteRatio,this.seed=e.seed||42,this.timeBudgetMs=e.timeBudgetMs||3e4,this.clock=e.clock||null,this.tuningDB=e.tuningDB||null,this.useTuningDB=e.useTuningDB!==!1,this.measurer=e.measurer||null,this.hardwareMeasure=e.hardwareMeasure??!1,this.enableBenchmark=e.enableBenchmark??(this.hardwareMeasure||!!e.measurer),this.benchmarkWarmup=e.benchmarkWarmup??3,this.benchmarkRepeat=e.benchmarkRepeat??10,this.benchmarkMaxCv=e.benchmarkMaxCv??0,this.topKForBenchmark=e.topKForBenchmark??5,this.maxRoundsPerTask=e.maxRoundsPerTask??8,this.plateauPatience=e.plateauPatience??2,this.schedulerPolicy=e.schedulerPolicy||null,this.onWarning=e.onWarning||null}},jy=class{static{l(this,"Autotuner")}constructor(e,t={},n=null){this.target=e,this.config=t instanceof Gy?t:new Gy(t),this.trace=n,this._funcName=null,this.config.hardwareMeasure&&(this.config.measurer=FF(e)),this.analyticalModel=new by(e),this.learnedModel=new vl,this.costModel=new Nl(this.analyticalModel,this.learnedModel),this.db=this.config.tuningDB instanceof gf?this.config.tuningDB:new gf;let s=l((o,i,a)=>this._warn(o,i,a),"warn");this.benchmarkRunner=this.config.enableBenchmark?new Ry(e,{warmup:this.config.benchmarkWarmup,repeat:this.config.benchmarkRepeat,maxCv:this.config.benchmarkMaxCv,measurer:this.config.measurer,warn:s}):null,this.scheduler=new Vy(this.config.schedulerPolicy)}_warn(e,t,n){let s=n&&n.message?n.message:String(n);if(this.config.onWarning)try{this.config.onWarning({stage:e,func:this._funcName,block:t||null,message:s,error:n})}catch(o){}this.trace&&this.trace.warn("autotune",this._funcName,`${e}${t?" ["+t+"]":""}: ${s}`)}tune(e,t=null){this._funcName=e.name;let n=t?[t]:io(e.body),s=Ay(e.body),o=df(e),i=new My(this.config.timeBudgetMs,this.config.clock),a=new Map,u=new Map;for(let h of n){let f=Oy(e,h,this.target,s);u.set(h,f);let d=a.get(f);if(d){d.weight++;continue}if(this.config.useTuningDB&&this.db.has(f)){a.set(f,{key:f,kind:"cache",cached:this.db.lookup(f),weight:1});continue}let m=zT(e,h,this.target,s,{richGpu:this.config.richGpu??!!this.config.measurer,dag:o});if(m.length===0){a.set(f,{key:f,kind:"empty",weight:1});continue}let g=new Fy({target:this.target,primFunc:e,blockName:h,blockMap:s,sketches:m,costModel:this.costModel,learnedModel:this.learnedModel,benchmarkRunner:this.benchmarkRunner,config:this.config,deadline:i,warn:l((y,_,b)=>this._warn(y,_,b),"warn")});a.set(f,{key:f,kind:"session",session:g,weight:1})}let p=[...a.values()].filter(h=>h.kind==="session");p.length>0&&this.scheduler.run(p,i,this.config);let c=new Map;for(let h of n){let f=a.get(u.get(h));if(f.kind==="cache"){c.set(h,{sketchName:f.cached.sketchName,params:f.cached.params,score:f.cached.score,fromCache:!0});continue}if(f.kind==="empty")continue;let d=f.session.best();if(d){if(this.config.useTuningDB&&!f.stored){let m=new mf(f.key,d.sketchName,d.params,d.score,f.session.bestTrace(),this.db.version);m.medianMs=d.medianMs||null,m.minMs=d.minMs||null,this.db.store(f.key,m),f.stored=!0}c.set(h,{sketchName:d.sketchName,params:d.params,score:d.score,fromCache:!1,medianMs:d.medianMs,minMs:d.minMs})}}return c}tuneAndApply(e,t=null){let n=this.tune(e,t);if(n.size>0){let s=this._applyBestSchedule(e,n);if(s)return{func:s.func,results:n,applied:!0}}return{func:e,results:n,applied:!1}}_applyBestSchedule(e,t){let n=this._buildDefaultSchedule(e),s=!!n&&this._scheduleIsValid(n),o=s&&this._isStrongBackendSchedule(n),i=this._buildTunedSchedule(e,t),a=!!i&&this._scheduleIsValid(i),u=a&&this._isStrongBackendSchedule(i),p=!o||u&&this.config.measurer!=null;return a&&p?(this._adoptSchedule(e,i),{func:e}):(i&&!a?this._warn("tuned-schedule-invalid",null,new Error("tuned schedule exceeds target thread-block limit; falling back to default")):a&&o&&this._warn("baseline-preferred",null,new Error("cost-model-only tuning cannot displace the deterministic GPU schedule without hardware measurement; keeping the deterministic kernel")),s?(this._adoptSchedule(e,n),{func:e}):a?(this._adoptSchedule(e,i),{func:e}):(this._warn("no-valid-schedule",null,new Error("neither tuned nor default schedule is valid; leaving function unscheduled")),null))}_isStrongBackendSchedule(e){return!!e&&e.gpuRegisterBlocked===!0}_buildTunedSchedule(e,t){try{let n=Lo(e),s=new Sn(n),o=Ay(n.body),i=df(n),a=new Set,u=[];for(let c of t)if(c[1].sketchName==="fused"){let h=vy(n,i,c[0],dt);h&&a.add(h),u.unshift(c)}else u.push(c);let p=new Set;for(let[c,h]of u)if(!(a.has(c)||p.has(h))&&(p.add(h),!(!h.sketchName||!h.params)))try{let d=zT(n,c,this.target,o,{richGpu:this.config.richGpu??!!this.config.measurer,dag:i}).find(m=>m.name===h.sketchName);d&&this._fitsThreadBlock(n,c,d,h.params)&&d.instantiate(h.params)(s,c,this.target)}catch(f){this._warn("apply-tuned-block",c,f);continue}return this._scheduleResidualBlocks(s,a),n}catch(n){return this._warn("build-tuned-schedule",null,n),null}}_scheduleResidualBlocks(e,t){let n=null;for(let s of io(e.func.body))if(!(t.has(s)||this._blockIsParallelized(e,s))){n||(n=new Rs(this.target));try{n.applyToBlock(e,s)}catch(o){this._warn("residual-block",s,o)}}}_blockIsParallelized(e,t){let n;try{n=e.getLoops(t)}catch(s){return this._warn("block-loops",t,s),!0}for(let s of n)if(s.kind===F.THREAD_BINDING||s.kind===F.PARALLEL||s.kind===F.VECTORIZED)return!0;return!1}_scheduleIsValid(e){return!this.target.isGPU||!this.target.isGPU()||!this.target.maxThreadsPerBlock?!0:zy(e)<=this.target.maxThreadsPerBlock}_buildDefaultSchedule(e){try{let t=Lo(e),n=new Sn(t);return Py(n,this.target,this.config)||new Rs(this.target).applyToAllBlocks(n),t}catch(t){return this._warn("build-default-schedule",null,t),null}}_adoptSchedule(e,t){Object.assign(e,t),e._setChild("body",e.body)}_fitsThreadBlock(e,t,n,s){if(!this.target.isGPU||!this.target.isGPU()||!this.target.maxThreadsPerBlock)return!0;try{let o=Lo(e);return n.instantiate(s)(new Sn(o),t,this.target),zy(o)<=this.target.maxThreadsPerBlock}catch(o){return this._warn("fits-thread-block",t,o),!1}}}});var qy,vR=M(()=>{"use strict";xi();fa();ro();kR();qT();qy=class extends Wn{static{l(this,"SchedulePass")}constructor(e){super("SchedulePass","scheduling"),this.config=e,this.target=e.target,this.snapshotPoint="afterScheduling",this._autotuner=null,this._policy=null}begin(e){let t=this.config.scheduling;t.autotune?this._autotuner=new jy(this.target,t,e.trace):t.enabled&&(this._policy=new Rs(this.target,null,e.trace))}run(e,t){let n=this.config.scheduling,s=t.trace;if(n.autotune){if(e.cublasInfo||e._tensorIntrin)return;let o=performance.now(),i=this._autotuner.tuneAndApply(e),a=performance.now()-o,u=0,p=0;if(i&&i.results){p=i.results.size;for(let[c,h]of i.results)h.fromCache&&u++,s.explainsEnabled&&s.explain("schedule",c,h.sketchName,`autotuned: best of search${h.fromCache?" (cached)":""}, score ${h.score!=null?h.score.toFixed(3):"n/a"}`,{target:this.target.name,params:h.params})}s.autotuneStats(e.name,{durationMs:a,blockCount:p,applied:!!(i&&i.applied),cacheHits:u})}else if(n.enabled||n.gpuTiling){if(e.cublasInfo||e._tensorIntrin)return;let o=performance.now(),i=new Sn(e);!Py(i,this.target,n)&&n.enabled&&this._policy.applyToAllBlocks(i),s.functionEvent("scheduling",e.name,{durationMs:performance.now()-o})}}}});function NR(r){let e={analyzer:new Os,simp:null};e.simp=new fm(e.analyzer);let t=Zr(r.body,e);return r.body=t,r._setChild("body",t),r}function zF(r,e,t){let n=r.analyzer.getVarBound(e);return t&&t.type==="IntImmNode"&&t.value>0?r.analyzer.bind(e,0,t.value-1):r.analyzer.setVarBound(e,null),n}function Zr(r,e){if(!r||typeof r!="object")return r;switch(r.type){case"ForNode":{let t=zF(e,r.loopVar.name,r.extent),n=Zr(r.body,e);e.analyzer.setVarBound(r.loopVar.name,t);let s=new de(r.loopVar,r.min,r.extent,r.kind,n,r.threadTag);return r.annotations&&(s.annotations=r.annotations),s}case"BlockNode":{let t=[];for(let i of r.iterVars)i.iterVar&&(t.push([i.iterVar.name,e.analyzer.getVarBound(i.iterVar.name)]),e.analyzer.setVarBound(i.iterVar.name,i.binding?la(e.analyzer,i.binding):null));let n=r.iterVars.map(VF(e)),s=Zr(r.body,e),o=r.initBody?Zr(r.initBody,e):null;for(let[i,a]of t)e.analyzer.setVarBound(i,a);return new pe(r.name,n,r.reads,r.writes,s,o)}case"SeqNode":return new te(r.stmts.map(t=>Zr(t,e)));case"IfThenElseNode":{let t=Hn(r.condition,e);return Mo(e.analyzer,t)?Zr(r.thenBody,e):ua(e.analyzer,t)?r.elseBody?Zr(r.elseBody,e):new te([]):new xe(t,Zr(r.thenBody,e),r.elseBody?Zr(r.elseBody,e):null)}case"BufferStoreNode":return new Y(r.buffer,r.indices.map(t=>Hn(t,e)),Hn(r.value,e));case"LetStmtNode":return new Ye(r.variable,Hn(r.value,e),Zr(r.body,e));case"AllocateNode":return new Ve(r.buffer,r.scope,Zr(r.body,e));case"WhileNode":return new Zo(r.condVar,Zr(r.condBody,e),Zr(r.loopBody,e));case"EvaluateNode":return new ou(Hn(r.value,e));default:return r}}function VF(r){return e=>{if(!e.iterVar||!e.binding)return e;let t=Hn(e.binding,r);return new qi(e.iterVar,t,e.kind)}}function Hn(r,e){if(!r||typeof r!="object"||!r.type)return r;switch(r.type){case"IntImmNode":case"FloatImmNode":case"VariableNode":return r;case"BufferLoadNode":return new q(r.buffer,r.indices.map(t=>Hn(t,e)));case"MathOpNode":{let t=Hn(r.a,e),n=r.b?Hn(r.b,e):null;return e.simp.simplify(new C(r.op,t,n))}case"CompareNode":{let t=Hn(r.a,e),n=Hn(r.b,e);return e.simp.simplify(new qe(r.direction,t,n))}case"CastNode":return new rt(Hn(r.expr,e),r.fromDtype,r.toDtype);case"CallExternNode":return new ke(r.externName,r.args.map(t=>Hn(t,e)),r.dtype);case"IfThenElseNode":{let t=Hn(r.condition,e),n=Hn(r.thenBody,e),s=r.elseBody?Hn(r.elseBody,e):null;return Mo(e.analyzer,t)?n:s!==null&&ua(e.analyzer,t)?s:new xe(t,n,s)}default:return r}}var AR=M(()=>{"use strict";Uu();Hc();De();l(NR,"simplifyPrimFunc");l(zF,"bindLoopVar");l(Zr,"simplifyStmt");l(VF,"simplifyIterVar");l(Hn,"simplifyExpr")});var Uy,OR=M(()=>{"use strict";xi();AR();Uy=class extends Wn{static{l(this,"SimplifyPass")}constructor(){super("SimplifyPass","simplify")}run(e,t){let n=performance.now();return NR(e),t.trace.functionEvent("simplify",e.name,{durationMs:performance.now()-n}),e}}});function BR(r){return!!r&&typeof r=="object"&&typeof r.name=="string"&&r.dtype!==void 0&&r.shape!==void 0&&r.type===void 0}var YT,JT,GF,Ky,RR=M(()=>{"use strict";YT=class{static{l(this,"BufferInterval")}constructor(e,t,n,s){this.buffer=e,this.firstUse=t,this.lastUse=n,this.scope=s}get size(){return this.buffer.sizeInBytes()}overlaps(e){return this.firstUse<=e.lastUse&&e.firstUse<=this.lastUse}},JT=class{static{l(this,"BufferLivenessResult")}constructor(e,t,n){this.intervals=e,this.stmtOrder=t,this.paramBuffers=n}isParam(e){return this.paramBuffers.has(e)}getTemporaries(){let e=[];for(let[t,n]of this.intervals)this.paramBuffers.has(t)||e.push(n);return e}interfere(e,t){let n=this.intervals.get(e),s=this.intervals.get(t);return!n||!s?!1:n.overlaps(s)}},GF=new Set(["_parent","_parentKey","_parentIdx"]);l(BR,"isBuffer");Ky=class{static{l(this,"BufferLiveness")}static analyze(e){let t=new Map,n=[],s=0,o=new Set;for(let[,h]of e.bufferMap)o.add(h);let i=[],a=l(h=>{if(!h)return;let f=t.get(h);f?(s<f.firstUse&&(f.firstUse=s),s>f.lastUse&&(f.lastUse=s)):(f=new YT(h,s,s,h.scope),t.set(h,f)),i.push(h)},"touch"),u=l((h,f)=>{if(!(!h||typeof h!="object"||f.has(h))){if(f.add(h),BR(h)){a(h);return}BR(h.buffer)&&a(h.buffer);for(let d of Object.keys(h)){if(GF.has(d)||d==="buffer")continue;let m=h[d];if(!(!m||typeof m!="object"))if(Array.isArray(m))for(let g of m)u(g,f);else u(m,f)}}},"touchAll"),p=l((h,f)=>{for(let d=h;d<i.length;d++){let m=t.get(i[d]);m&&f>m.lastUse&&(m.lastUse=f)}},"extendRegion"),c=l(h=>{if(h)switch(h.type){case"SeqNode":for(let f of h.stmts)c(f);break;case"ForNode":{let f=s,d=i.length;u(h.min,new Set),u(h.extent,new Set),c(h.body);let m=s>f?s-1:f;p(d,m);break}case"WhileNode":{let f=s,d=i.length;u(h.condVar,new Set),c(h.condBody),c(h.loopBody);let m=s>f?s-1:f;p(d,m);break}case"BlockNode":n.push({idx:s,node:h});for(let f of h.reads)a(f.buffer);for(let f of h.writes)a(f.buffer);u(h.body,new Set),h.initBody&&u(h.initBody,new Set),s++;break;case"AllocateNode":a(h.buffer),c(h.body);break;case"IfThenElseNode":{let f=s,d=i.length;u(h.condition,new Set),c(h.thenBody),h.elseBody&&c(h.elseBody);let m=s>f?s-1:f;p(d,m);break}case"LetStmtNode":u(h.value,new Set),c(h.body);break;case"EvaluateNode":u(h.value,new Set);break;default:u(h,new Set);break}},"walk");return c(e.body),new JT(t,n,o)}}});function jF(r,e){if(r.shape.length!==e.shape.length)return!1;for(let t=0;t<r.shape.length;t++)if(r.shape[t]!==e.shape[t])return!1;return!0}function Ml(r,e){if(r===e)return!0;if(!r||!e||typeof r!="object"||typeof e!="object"||r.type!==e.type)return!1;switch(r.type){case"VariableNode":return r.name===e.name;case"IntImmNode":return r.value===e.value;case"FloatImmNode":return r.value===e.value;case"MathOpNode":return r.op===e.op&&Ml(r.a,e.a)&&Ml(r.b,e.b);case"CompareNode":return r.direction===e.direction&&Ml(r.a,e.a)&&Ml(r.b,e.b);case"CastNode":return r.toDtype===e.toDtype&&Ml(r.expr,e.expr);case"BufferLoadNode":return r.buffer===e.buffer&&QT(r.indices,e.indices);default:return!1}}function QT(r,e){if(r.length!==e.length)return!1;for(let t=0;t<r.length;t++)if(!Ml(r[t],e[t]))return!1;return!0}function MR(r,e){r&&Ss(r,e)}function qF(r,e,t){let n=[],s=[];for(let p of[r.body,r.initBody])p&&MR(p,c=>{c.type==="BufferStoreNode"&&c.buffer===t?n.push(c):c.type==="BufferLoadNode"&&c.buffer===e&&s.push(c)});if(n.length!==1||s.length===0)return!1;let o=n[0],i=o.indices;for(let p of s)if(!QT(p.indices,i))return!1;let a=new Set;for(let p of[o.value,...o.indices])p&&MR(p,c=>{c.type==="BufferLoadNode"&&c.buffer===e&&a.add(c)});let u=!0;for(let p of s)if(!a.has(p)){u=!1;break}if(!u){let p=o.value;if(!(p&&p.type==="BufferLoadNode"&&p.buffer===e&&QT(p.indices,i)))return!1}return!0}function UF(r,e){if(r)for(let t of gu(r,n=>n.type==="BlockNode",{kinds:"stmt"}))e.push(t)}var ZT,Wy,IR=M(()=>{"use strict";Gr();ZT=class{static{l(this,"InplaceCandidate")}constructor(e,t,n){this.srcBuffer=e,this.dstBuffer=t,this.reason=n}},Wy=class{static{l(this,"InplaceAnalysis")}static analyze(e,t,n=new Set){let s=[],o=[];UF(e.body,o);let i=new Map;for(let p of t.stmtOrder)i.set(p.node,p.idx);let a=new Map;for(let p of t.stmtOrder)for(let c of p.node.reads){let h=a.get(c.buffer);(h===void 0||p.idx>h)&&a.set(c.buffer,p.idx)}let u=new Set;for(let p of o){let c=i.get(p);if(c!==void 0)for(let h of p.writes){let f=h.buffer;if(!(t.isParam(f)&&!n.has(f)))for(let d of p.reads){let m=d.buffer;if(m===f||u.has(m)||t.isParam(m)&&!n.has(m)||!jF(m,f)||m.dtype!==f.dtype||m.scope!==f.scope)continue;let g=t.intervals.get(m),y=t.intervals.get(f);if(!(!g||!y)&&g.lastUse<=y.firstUse){let _=a.get(m);if(_===void 0||_<=c){if(!qF(p,m,f))continue;s.push(new ZT(m,f,`${m.name} last used at ${g.lastUse}, ${f.name} first used at ${y.firstUse}`)),u.add(m);break}}}}}return s}};l(jF,"shapesMatch");l(Ml,"exprEqual");l(QT,"indexListEqual");l(MR,"walkNodes");l(qF,"isInplaceComputeSafe");l(UF,"collectBlocks")});function ER(r,e){let t=0,n=r.length;for(;t<n;){let s=t+n>>1;r[s].offset<=e.offset?t=s+1:n=s}r.splice(t,0,e)}function LR(r){return{best:null,consider(e,t,n){return t<n?null:r==="best-fit"?((this.best===null||t<this.best.gap)&&(this.best={offset:e,gap:t}),null):e},result(e){return this.best!==null?this.best.offset:e}}}var XT,ex,Hy,CR=M(()=>{"use strict";l(ER,"insertByOffset");l(LR,"gapSelector");XT=class{static{l(this,"MemoryBlock")}constructor(e,t,n){this.offset=e,this.size=t,this.buffer=n}get end(){return this.offset+this.size}overlaps(e){return this.offset<e.end&&e.offset<this.end}},ex=class{static{l(this,"MemoryPool")}constructor(e,t=64,n="best-fit"){this.scope=e,this.alignment=t,this.strategy=n,this.blocks=[],this.peakUsage=0}allocate(e,t){let n=this._align(e),s=this._findFreeOffset(n);return this.placeAt(s,n,t)}placeAt(e,t,n){let s=this._align(t),o=new XT(e,s,n);ER(this.blocks,o);let i=e+s;return i>this.peakUsage&&(this.peakUsage=i),o}_align(e){return Math.ceil(e/this.alignment)*this.alignment}_findFreeOffset(e){let t=this.blocks,n=0,s=LR(this.strategy);for(let o of t){let i=this._align(n),a=o.offset-i,u=s.consider(i,a,e);if(u!==null)return u;o.end>n&&(n=o.end)}return s.result(this._align(n))}fragmentation(){if(this.peakUsage===0)return 0;let e=this.blocks.reduce((t,n)=>t+n.size,0);return Math.max(0,1-e/this.peakUsage)}release(e){let t=this.blocks.indexOf(e);t>=0&&this.blocks.splice(t,1)}},Hy=class{static{l(this,"BufferAssignment")}constructor(){this.assignments=new Map,this.pools=new Map,this.inplaceMap=new Map,this.effLastUse=new Map}assign(e,t=[],n=64,s="best-fit"){for(let c of t)this.inplaceMap.set(c.dstBuffer,c.srcBuffer);let o=new Map;for(let c of e)o.set(c.buffer,c);let i=new Map;for(let c of e)i.set(c.buffer,c.lastUse);let a=new Map;for(let[c,h]of this.inplaceMap)!o.has(c)||!o.has(h)||(a.has(h)||a.set(h,[]),a.get(h).push(c));if(a.size>0){let c=new Map;for(let h of e){let f=h.buffer;if(c.get(f)===1)continue;let d=[f];for(;d.length;){let m=d[d.length-1],g=a.get(m);if(c.get(m)===void 0){if(c.set(m,0),g)for(let y of g)c.get(y)===void 0&&d.push(y);continue}if(d.pop(),c.get(m)!==1){if(g){let y=i.get(m);for(let _ of g){let b=i.get(_);b>y&&(y=b)}i.set(m,y)}c.set(m,1)}}}}this.effLastUse=i;let u=[...e].sort((c,h)=>{let f=c.size,d=h.size,m=f>0,g=d>0;if(m&&g){let y=d-f;if(y!==0)return y}else if(m!==g)return m?-1:1;return c.firstUse-h.firstUse}),p=new Map;for(let c of u){let h=c.buffer;if(this.inplaceMap.has(h)){let w=this.inplaceMap.get(h),x=this.assignments.get(w);if(x){this.assignments.set(h,{offset:x.offset,size:c.size,scope:c.scope,pool:x.pool,inplaceOf:w});continue}}let f=c.size;if(f===0)continue;if(f<0){this.assignments.set(h,{offset:0,size:0,scope:c.scope,pool:c.scope,inplaceOf:null,isDynamic:!0});continue}let d=c.scope,m=this.pools.get(d);m||(m=new ex(d,n),this.pools.set(d,m)),p.has(d)||p.set(d,[]);let g=p.get(d),y=i.get(h)??c.lastUse,_=this._interferenceOffset(g,c.firstUse,y,m._align(f),n,s),b=m.placeAt(_,f,h);ER(g,{firstUse:c.firstUse,lastUseEff:y,offset:b.offset,size:b.size}),this.assignments.set(h,{offset:b.offset,size:b.size,scope:d,pool:d,inplaceOf:null})}return this}_interferenceOffset(e,t,n,s,o,i){let a=[];for(let c of e)c.firstUse<=n&&t<=c.lastUseEff&&a.push([c.offset,c.offset+c.size]);let u=0,p=LR(i);for(let[c,h]of a){let f=Math.ceil(u/o)*o,d=c-f,m=p.consider(f,d,s);if(m!==null)return m;h>u&&(u=h)}return p.result(Math.ceil(u/o)*o)}getOffset(e){let t=this.assignments.get(e);return t?t.offset:-1}getAssignment(e){return this.assignments.get(e)||null}peakMemory(e=null){if(e){let n=this.pools.get(e);return n?n.peakUsage:0}let t=0;for(let[,n]of this.pools)t+=n.peakUsage;return t}}});function Jy(r,e,t){if(!r||typeof r!="object"||t.has(r))return!1;if(t.add(r),r.type==="BufferLoadNode"&&(!e||r.buffer===e))return!0;for(let n of Object.keys(r)){if(n==="_parent"||n==="_parentKey"||n==="_parentIdx")continue;let s=r[n];if(!(!s||typeof s!="object")){if(Array.isArray(s)){for(let o of s)if(Jy(o,e,t))return!0}else if(Jy(s,e,t))return!0}}return!1}function DR(r){let e=new Set,t=new Map,n=new Set,s=[r.body];for(;s.length>0;){let o=s.pop();if(!(!o||typeof o!="object"||n.has(o))){if(n.add(o),o.type==="BufferStoreNode"&&o.buffer){let i=o.buffer;for(let u of o.indices)if(Jy(u,null,new Set)){e.add(i);break}Jy(o.value,i,new Set)&&e.add(i);let a=t.get(i);a||(a=[],t.set(i,a)),a.push(o.value)}for(let i of Object.keys(o)){if(i==="_parent"||i==="_parentKey"||i==="_parentIdx")continue;let a=o[i];if(Array.isArray(a))for(let u of a)u&&typeof u=="object"&&s.push(u);else a&&typeof a=="object"&&s.push(a)}}}for(let[o,i]of t){let a=l(u=>u&&(u.type==="IntImmNode"||u.type==="FloatImmNode")&&u.value===0,"isConstZero");i.every(a)&&e.add(o),i.length===1&&i[0]&&(i[0].type==="IntImmNode"||i[0].type==="FloatImmNode")&&e.add(o)}return e}function KF(r,e){let t=new Set,n=[r];for(;n.length>0;){let s=n.pop();if(!(!s||typeof s!="object"||t.has(s))){t.add(s);for(let o of Object.keys(s)){if(o==="_parent"||o==="_parentKey"||o==="_parentIdx")continue;let i=s[o];if(i&&typeof i=="object"&&e.has(i)){s[o]=e.get(i);continue}if(Array.isArray(i)){for(let a=0;a<i.length;a++){let u=i[a];u&&typeof u=="object"&&e.has(u)?i[a]=e.get(u):u&&typeof u=="object"&&n.push(u)}continue}i&&typeof i=="object"&&n.push(i)}}}}var tx,Yy,PR=M(()=>{"use strict";RR();IR();CR();De();Zb();tx=class{static{l(this,"MemoryPlan")}constructor(e,t,n){this.assignment=e,this.liveness=t,this.inplaceCandidates=n,this.aliasMap=new Map}peakMemory(e=null){return this.assignment.peakMemory(e)}getReport(){let e=new Map;for(let[s,o]of this.assignment.pools)e.set(s,{peakUsage:o.peakUsage,numBuffers:0,numReused:0});for(let[s,o]of this.assignment.assignments){let i=e.get(o.scope);i&&(i.numBuffers++,o.inplaceOf&&i.numReused++)}let t=this.liveness.getTemporaries().length,n=this.inplaceCandidates.length;return{peakMemory:this.assignment.peakMemory(),scopeBreakdown:e,totalTemporaries:t,totalInplace:n,materializedReuse:this.aliasMap.size,assignments:this.assignment.assignments}}},Yy=class{static{l(this,"MemoryPlanner")}constructor(e={}){this.alignment=e.alignment||64,this.enableInplace=e.enableInplace!==!1,this.allocStrategy=e.allocStrategy||"best-fit",this.poolAllocation=e.poolAllocation||!1}plan(e){let t=Ky.analyze(e),n=t.getTemporaries(),s=[];this.enableInplace&&(s=Wy.analyze(e,t));let o=new Hy;return o.assign(n,s,this.alignment,this.allocStrategy),new tx(o,t,s)}planAndRewrite(e){let t=this.plan(e);return{func:this._insertAllocations(e,t),plan:t}}_insertAllocations(e,t){let n=t.liveness.getTemporaries();if(n.length===0)return e;let s=new Map;this.poolAllocation?this._assignPoolOffsets(e,t,n):(s=this._buildReuseAliases(n,t,e),s.size>0&&KF(e.body,s)),t.aliasMap=s;let o=[...n].sort((u,p)=>p.firstUse-u.firstUse),i=e.body,a=new Set;for(let u of o){let p=u.buffer;if(s.has(p))continue;let c=t.assignment.getAssignment(p);c&&(c.inplaceOf||a.has(p)||(a.add(p),i=new Ve(p,c.isDynamic?"dynamic":c.scope,i)))}return e.body=i,e._setChild("body",i),e}_assignPoolOffsets(e,t,n){let s=DR(e);for(let o of n){let i=o.buffer;if(s.has(i)||i.scope!=="global")continue;let a=t.assignment.getAssignment(i);!a||a.inplaceOf||a.isDynamic||a.size>0&&(i.poolByteOffset=a.offset)}}_buildReuseAliases(e,t,n){let s=DR(n),o=new Set(t.assignment.inplaceMap.values()),i=t.assignment.effLastUse,a=l(c=>i.get(c.buffer)??c.lastUse,"lastUseOf"),u=new Map;for(let c of e){let h=c.buffer,f=t.assignment.getAssignment(h);if(!f||f.inplaceOf||f.isDynamic||o.has(h)||h.numel()<=0||s.has(h))continue;let d=`${h.scope}|${h.dtype}|${h.shape.join(",")}|${h.strides.join(",")}`,m=u.get(d);m||(m=[],u.set(d,m)),m.push(c)}let p=new Map;for(let c of u.values()){if(c.length<2)continue;c.sort((f,d)=>f.firstUse-d.firstUse||a(f)-a(d));let h=new ku((f,d)=>f.lastUse-d.lastUse);for(let f of c){let d=h.peek();d&&d.lastUse<f.firstUse?(h.pop(),d.lastUse=a(f),h.push(d),p.set(f.buffer,d.rep)):h.push({rep:f.buffer,lastUse:a(f)})}}return p}};l(Jy,"exprLoadsBuffer");l(DR,"collectFreshZeroDependent");l(KF,"rewriteBufferAliases")});var Zy,$R=M(()=>{"use strict";xi();PR();Zy=class extends Wn{static{l(this,"MemoryPlanPass")}constructor(e){super("MemoryPlanPass","memoryPlanning"),this.config=e,this._planner=null}begin(e){let t=this.config.memory.alignment||this.config.target?.cacheLineSizeBytes||64;this._planner=new Yy({alignment:t,enableInplace:this.config.memory.inplaceReuse,allocStrategy:this.config.memory.allocStrategy,poolAllocation:this.config.memory.poolAllocation})}run(e,t){if(e.gpuRegisterBlocked)return;let n=performance.now(),{plan:s}=this._planner.planAndRewrite(e),o=s.getReport();t.trace.memoryStats(e.name,{durationMs:performance.now()-n,peakMemory:o.peakMemory,totalTemporaries:o.totalTemporaries,totalInplace:o.totalInplace})}}});function Il(r){return r&&r.type==="IntImmNode"?r.value:null}function WF(r){if(!r||r.type!=="ForNode"||!FR.has(r.kind)||Il(r.min)!==0)return null;let e=Il(r.extent);if(e===null)return null;let t=r.body;if(!t||t.type!=="ForNode"||!FR.has(t.kind)||Il(t.min)!==0)return null;let n=Il(t.extent);if(n===null||n<=0)return null;let s=t.body;if(!s||s.type!=="IfThenElseNode"||s.elseBody)return null;let o=HF(s.condition,r.loopVar.name,t.loopVar.name,n);if(o===null||o<=0)return null;let i=Math.floor(o/n),a=o-i*n;if(a===0||i<1||e!==i+1)return null;let u=new Os;if(u.bind(r.loopVar.name,0,i-1),u.bind(t.loopVar.name,0,n-1),!Mo(u,s.condition))return null;let p=s.thenBody,c=new de(t.loopVar,new L(0),new L(n),t.kind,p,t.threadTag),h=new de(r.loopVar,new L(0),new L(i),r.kind,c,r.threadTag),f=ti(Ma(p),g=>g.type==="VariableNode"&&g.name===r.loopVar.name?new L(i):g),d=new Ne(t.loopVar.name,t.loopVar.dtype),m=new de(d,new L(0),new L(a),t.kind,f,t.threadTag);return new te([h,m])}function HF(r,e,t,n){if(!r)return null;let s,o;if(r.type==="MathOpNode"&&r.op==="<")s=r.a,o=r.b;else if(r.type==="CompareNode"&&r.direction==="lt")s=r.a,o=r.b;else return null;let i=Il(o);if(i===null||!s||s.type!=="MathOpNode"||s.op!=="+")return null;let a=s.a,u=s.b;return!u||u.type!=="VariableNode"||u.name!==t||!a||a.type!=="MathOpNode"||a.op!=="*"||!a.a||a.a.type!=="VariableNode"||a.a.name!==e||Il(a.b)!==n?null:i}var FR,Qy,zR=M(()=>{"use strict";xi();De();Gr();lf();Uu();Hc();FR=new Set([F.SERIAL,F.PARALLEL,F.VECTORIZED,F.UNROLLED]),Qy=class extends Wn{static{l(this,"LoopPartitionPass")}constructor(){super("LoopPartitionPass","loopPartition")}run(e,t){let n=ti(e.body,s=>WF(s)||s);return n&&n!==e.body&&(e.body=n,e._setChild("body",n)),e}};l(Il,"intImm");l(WF,"partitionLoop");l(HF,"matchFlatGuard")});function Xy(r){let e=r.body;if(!e||e.type!=="BlockNode")return null;let t=e.body;if(!t||t.type!=="BufferStoreNode")return null;let n=t,s=n.value;if(!s||s.type!=="MathOpNode"||!YF.has(s.op))return null;let o=null,i=null;if(s.a&&s.a.type==="BufferLoadNode"&&s.a.buffer===n.buffer?(o=s.a,i=s.b):s.b&&s.b.type==="BufferLoadNode"&&s.b.buffer===n.buffer&&(o=s.b,i=s.a),!o)return null;let a=nx(n.indices),u=nx(o.indices);if(a!==u||a.includes("?"))return null;let p=n.indices.map(f=>{if(f.type!=="VariableNode")return f;for(let d of e.iterVars)if(d.iterVar&&d.iterVar.name===f.name&&d.binding)return d.binding;return f}),c=r.loopVar.name,h=nx(p);return h.includes("?")||h.includes("$"+c)?null:{store:n,loadSide:o,valueSide:i,outerIndices:p,block:e,op:n.value.op}}function nx(r){return r.map(rx).join(",")}function rx(r){return r?r.type==="VariableNode"?"$"+r.name:r.type==="IntImmNode"?String(r.value):r.type==="MathOpNode"?"("+rx(r.a)+r.op+(r.b?rx(r.b):"")+")":"?":"?"}var YF,sx=M(()=>{"use strict";YF=new Set(["+","*","max","min"]);l(Xy,"detectAccumulator");l(nx,"indicesKey");l(rx,"exprKey")});function uo(r){if(!(!r||typeof r!="object"||!r.type))switch(r.type){case"ForNode":{let e=Xy(r);r.accumulator=e,e||uo(r.body);return}case"BlockNode":uo(r.body),r.initBody&&uo(r.initBody);return;case"SeqNode":for(let e of r.stmts)uo(e);return;case"LetStmtNode":case"AllocateNode":uo(r.body);return;case"IfThenElseNode":uo(r.thenBody),r.elseBody&&uo(r.elseBody);return;case"WhileNode":uo(r.condBody),uo(r.loopBody);return;default:return}}var e_,VR=M(()=>{"use strict";xi();sx();e_=class extends Wn{static{l(this,"AccumulatorDetectionPass")}constructor(){super("AccumulatorDetectionPass","accumulatorDetect")}run(e,t){return uo(e.body),e}};l(uo,"annotateStmt")});function ZF(r){let e=[],t=[r.body];for(;t.length>0;){let s=t.pop();if(s){if(s.type==="BlockNode"){e.push(s),t.push(s.body);continue}if(s.body&&t.push(s.body),s.stmts)for(let o of s.stmts)t.push(o);s.thenBody&&t.push(s.thenBody),s.elseBody&&t.push(s.elseBody)}}if(e.length===0)return null;let n=null;for(let s of e){if(!s.name.includes("matmul"))return null;s.reads.length>=2&&s.writes.length>=1&&(n=s)}return n}function QF(r){let e=ZF(r);if(!e)return null;let t=e.reads[0].buffer,n=e.reads[1].buffer,s=e.writes[0].buffer;if(!GR.has(t.dtype)||!GR.has(n.dtype)||s.dtype!=="f32"||t.shape.length!==2||n.shape.length!==2||s.shape.length!==2)return null;let o=s.shape[0],i=s.shape[1],a=t.shape[1];if(![o,i,a].every(p=>typeof p=="number"&&p>0&&p%JF===0))return null;let u=new Set;for(let[,p]of r.bufferMap)u.add(p.name);return!u.has(t.name)||!u.has(n.name)||!u.has(s.name)?null:{M:o,N:i,K:a,a:t.name,b:n.name,c:s.name}}var JF,GR,t_,jR=M(()=>{"use strict";xi();fa();JF=16,GR=new Set(["f16","bf16"]);l(ZF,"findMatmulBlock");l(QF,"detectWmmaMatmul");t_=class extends Wn{static{l(this,"AutoTensorizePass")}constructor(e){super("AutoTensorizePass","scheduling"),this.config=e,this.target=e.target}run(e,t){if(e.cublasInfo||e._tensorIntrin||!this.target||!this.target.isGPU())return;let n=QF(e);n&&(new Sn(e).tensorize("wmma_16x16x16_f16f16f32",n),t&&t.trace&&t.trace.explainsEnabled&&t.trace.explain("tensorize",e.name,"wmma_16x16x16_f16f16f32",`auto-tensorized ${n.M}x${n.N}x${n.K} f16 GEMM`,{target:this.target.name}))}}});function XF(r){let e=[],t=new Map,n=new Map,s=[{node:r,block:null}];for(;s.length>0;){let{node:o,block:i}=s.pop();if(o.type==="BufferLoadNode"&&o.buffer)t.set(o.buffer.name,(t.get(o.buffer.name)||0)+1),i&&i.reads.add(o.buffer.name);else if(o.type==="BufferStoreNode"&&o.buffer&&i){i.writes.add(o.buffer.name);let u=n.get(o.buffer.name);u||(u=new Set,n.set(o.buffer.name,u)),u.add(i.name)}let a=i;o.type==="BlockNode"&&(a={name:o.name,writes:new Set,reads:new Set,hasInit:o.initBody!=null},e.push(a));for(let u of en(o))s.push({node:u,block:a})}return{blocks:e,loadCount:t,storeWriters:n}}var n_,qR=M(()=>{"use strict";xi();fa();ro();Gr();l(XF,"analyzeFunc");n_=class extends Wn{static{l(this,"InlineReindexPass")}constructor(e){super("InlineReindexPass","scheduling"),this.config=e,this.target=e.target,this.snapshotPoint="afterInlineReindex"}run(e,t){if(!this.target.isGPU()||this.target.isWebGPU()||e.cublasInfo||e._tensorIntrin||i0(e))return;let n=this.config.scheduling;if(!(n.enabled||n.gpuTiling||n.autotune))return;let s=new Set;for(let[,c]of e.bufferMap)s.add(c.name);let{blocks:o,loadCount:i,storeWriters:a}=XF(e.body),u=new Sn(e),p=!1;for(let c of o){if(c.hasInit||c.writes.size===0)continue;let h=[...c.writes];if(h.every(f=>!s.has(f)&&a.get(f).size===1)&&h.some(f=>(i.get(f)||0)>0)&&[...c.reads].every(f=>(a.get(f)?a.get(f).size:0)<=1))try{u.computeInlineBlock(c.name),p=!0}catch{}}p&&s0(e)}}});function ox(r,e,t=ez){return t.filter(n=>n.phase===r).sort((n,s)=>n.priority-s.priority).map(n=>n.factory(e)).filter(Boolean)}var ez,UR=M(()=>{"use strict";ez=[];l(ox,"tirPassesForPhase")});function KR(r){let e=[];for(let t of ox("pre",r))e.push(t);e.push(new n_(r)),r.optimization.tensorize&&e.push(new t_(r)),e.push(new qy(r)),r.optimization.loopPartition&&e.push(new Qy),e.push(new Uy),e.push(new Zy(r)),r.optimization.detectAccumulators&&e.push(new e_);for(let t of ox("post",r))e.push(t);return e}var WR=M(()=>{"use strict";vR();OR();$R();zR();VR();jR();qR();UR();l(KR,"buildTirPipeline")});var _f,HR=M(()=>{"use strict";De();_f=class{static{l(this,"TensorVerifier")}constructor(){this.errors=[],this.boundVars=new Set}verify(e){if(this.errors=[],this.boundVars.clear(),!(e instanceof Jo))return this.errors.push("Expected PrimFunc at root"),this.errors;for(let t of e.params)this.boundVars.add(t.name);if(e.shapeParams)for(let t of e.shapeParams)this.boundVars.add(t.name);return this.visit(e.body),this.errors}visit(e){let t=[e];for(;t.length>0;){let n=t.pop();if(n)switch(n.type){case"SeqNode":for(let s=n.stmts.length-1;s>=0;s--)t.push(n.stmts[s]);break;case"ForNode":this.boundVars.has(n.loopVar.name)&&this.errors.push(`Loop variable ${n.loopVar.name} already bound`),this.boundVars.add(n.loopVar.name),t.push({type:"_unbind",name:n.loopVar.name}),t.push(n.body);break;case"BlockNode":for(let s of n.iterVars)s.iterVar&&(this.boundVars.has(s.iterVar.name)&&this.errors.push(`Block variable ${s.iterVar.name} already bound`),this.boundVars.add(s.iterVar.name));for(let s=n.iterVars.length-1;s>=0;s--)n.iterVars[s].iterVar&&t.push({type:"_unbind",name:n.iterVars[s].iterVar.name});t.push(n.body),n.initBody&&t.push(n.initBody);break;case"AllocateNode":n.buffer||this.errors.push("Allocate missing buffer"),t.push(n.body);break;case"LetStmtNode":this._visitExpr(n.value),this.boundVars.add(n.variable.name),t.push({type:"_unbind",name:n.variable.name}),t.push(n.body);break;case"IfThenElseNode":this._visitExpr(n.condition),n.elseBody&&t.push(n.elseBody),t.push(n.thenBody);break;case"WhileNode":n.condVar||this.errors.push("WhileNode missing condition variable"),t.push(n.loopBody),t.push(n.condBody);break;case"BufferStoreNode":if(n.buffer||this.errors.push("BufferStore missing buffer"),(!n.indices||n.indices.length!==n.buffer.shape.length)&&this.errors.push(`BufferStore rank mismatch for ${n.buffer?n.buffer.name:"unknown"}`),n.indices)for(let s of n.indices)this._visitExpr(s);this._visitExpr(n.value);break;case"EvaluateNode":this._visitExpr(n.value);break;case"_unbind":this.boundVars.delete(n.name);break;default:this._visitExpr(n);break}}}_visitExpr(e){if(e)switch(e.type){case"BufferLoadNode":if(e.buffer||this.errors.push("BufferLoad missing buffer"),(!e.indices||e.indices.length!==e.buffer.shape.length)&&this.errors.push(`BufferLoad rank mismatch for ${e.buffer?e.buffer.name:"unknown"}`),e.indices)for(let t of e.indices)this._visitExpr(t);break;case"MathOpNode":this._visitExpr(e.a),e.b&&this._visitExpr(e.b);break;case"CompareNode":this._visitExpr(e.a),this._visitExpr(e.b);break;case"CastNode":this._visitExpr(e.expr);break;case"CallExternNode":e.externName||this.errors.push("CallExtern missing function name");for(let t of e.args)this._visitExpr(t);break;case"IfThenElseNode":this._visitExpr(e.condition),this._visitExpr(e.thenBody),e.elseBody&&this._visitExpr(e.elseBody);break;case"VariableNode":this.boundVars.has(e.name)||this.errors.push(`Unbound variable used: ${e.name}`);break;case"BlockRealizeNode":e.binding&&this._visitExpr(e.binding);break;case"IntImmNode":case"FloatImmNode":break}}}});function ix(r){let e=[];r.functionCount===0&&e.push(new vt("Module has no functions"));for(let t of r)bf(t,e);return e}function bf(r,e=[]){if(!r.entryBlock)return e.push(new vt("Function has no entry block",null,r)),e;r.entryBlock.arguments.length!==r.inputTypes.length&&e.push(new vt(`Entry block has ${r.entryBlock.arguments.length} args, expected ${r.inputTypes.length}`,null,r));for(let s=0;s<r.entryBlock.arguments.length;s++){let o=r.entryBlock.arguments[s],i=r.inputTypes[s];i&&!o.type.equals(i)&&e.push(new vt(`Block arg ${s} type ${Rn(o.type)} != expected ${Rn(i)}`,null,r))}let t=new Set;for(let s of r.entryBlock.arguments)t.add(s);for(let s of r.body)YR(s,t);for(let s of r.body)JR(s,r,t,e);let n=r.getReturnOp();if(!n)e.push(new vt("Missing return op",null,r));else{n.numOperands!==r.outputTypes.length&&e.push(new vt(`Return has ${n.numOperands} operands, function declares ${r.outputTypes.length} outputs`,n,r));for(let s=0;s<Math.min(n.numOperands,r.outputTypes.length);s++){let o=n.getOperand(s).type,i=r.outputTypes[s];i instanceof I&&o instanceof I&&(o.dtype!==i.dtype&&e.push(new vt(`Return operand ${s} dtype ${o.dtype} != expected ${i.dtype}`,n,r)),o.shapeCompatible(i)||e.push(new vt(`Return operand ${s} shape incompatible: ${Rn(o)} vs ${Rn(i)}`,n,r)))}}return e}function YR(r,e){for(let t of r.arguments)e.add(t);for(let t of r)for(let n=0;n<t.numResults;n++){let s=t.getResult(n);s&&e.add(s)}}function tz(r,e,t){let n=new Set;for(let u of r)n.add(u);let s=1,o=2,i=new Map,a=new Set;for(let u of r){if(i.get(u)!==void 0)continue;let p=[{op:u,i:0}];for(i.set(u,s);p.length>0;){let c=p[p.length-1],h=c.op;if(c.i<h.numOperands){let f=h.getOperand(c.i);c.i++;let d=f&&f.definingOp;if(!d||!n.has(d))continue;let m=i.get(d);m===s?a.has(d)||(a.add(d),t.push(new vt("participates in a value dependency cycle",d,e))):m===void 0&&(i.set(d,s),p.push({op:d,i:0}));continue}i.set(h,o),p.pop()}}}function JR(r,e,t,n){tz(r,e,n);for(let s of r)nz(s,e,t,n);if(r.size>0){let s=r.lastOp;if(r.parentRegion&&r.parentRegion.parentOp){let o=r.parentRegion.parentOp.opName,i=ne.get(o);i&&i.hasRegions&&(s.isTerminator()||n.push(new vt(`Block in ${o} region must end with terminator, got '${s.opName}'`,s,e)))}}}function nz(r,e,t,n){for(let o=0;o<r.numOperands;o++){let i=r.getOperand(o);if(!i){n.push(new vt(`Operand ${o} is null`,r,e));continue}if(!(i instanceof Wo)){n.push(new vt(`Operand ${o} is not a Value`,r,e));continue}t.has(i)||n.push(new vt(`Operand ${o} used before definition`,r,e))}for(let o=0;o<r.numResults;o++){let i=r.getResult(o);i||n.push(new vt(`Result ${o} is null`,r,e)),i&&i.definingOp!==r&&n.push(new vt(`Result ${o} definingOp mismatch`,r,e))}let s=ne.get(r.opName);if(!s){!["return","yield"].includes(r.opName)&&!ne.has(r.opName)&&n.push(new vt(`Unknown op '${r.opName}'`,r,e));return}s.numOperands>=0&&r.numOperands!==s.numOperands&&n.push(new vt(`'${r.opName}' expects ${s.numOperands} operands, got ${r.numOperands}`,r,e)),s.numResults>=0&&r.numResults!==s.numResults&&n.push(new vt(`'${r.opName}' expects ${s.numResults} results, got ${r.numResults}`,r,e));for(let o of s.attrs)o.required&&!r.hasAttr(o.name)&&n.push(new vt(`'${r.opName}' missing required attribute '${o.name}'`,r,e));if(s.hasRegions&&s.numRegions>0&&r.numRegions!==s.numRegions&&n.push(new vt(`'${r.opName}' expects ${s.numRegions} regions, got ${r.numRegions}`,r,e)),s.verify){let o=s.verify(r);if(o)for(let i of o)n.push(new vt(i,r,e))}if(s.inferResultTypes&&r.numResults>0){let o=[];for(let a=0;a<r.numOperands;a++)o.push(r.getOperand(a).type);let i=s.inferResultTypes(o,r.attributes,r.results.map(a=>a.type));if(i)for(let a=0;a<Math.min(i.length,r.numResults);a++){let u=r.getResult(a).type,p=i[a];p instanceof I&&u instanceof I&&(u.dtype!==p.dtype&&n.push(new vt(`Result ${a} dtype ${u.dtype} != inferred ${p.dtype}`,r,e)),u.shapeCompatible(p)||n.push(new vt(`Result ${a} shape ${Rn(u)} incompatible with inferred ${Rn(p)}`,r,e)))}}for(let o of r.regions){let i=new Set(t);for(let a of o)YR(a,i);for(let a of o)JR(a,e,i,n)}}var vt,ZR=M(()=>{"use strict";Oe();lh();At();vt=class{static{l(this,"VerificationError")}constructor(e,t=null,n=null){this.message=e,this.op=t,this.func=n}toString(){let e="";return this.func&&(e+=`[${this.func.name}] `),this.op&&(e+=`op '${this.op.opName}' (id=${this.op.id}): `),e+this.message}};l(ix,"verifyModule");l(bf,"verifyFunction");l(YR,"collectScopeDefs");l(tz,"detectCycles");l(JR,"verifyBlock");l(nz,"verifyOperation")});var r_,ax,El,ux,lx=M(()=>{"use strict";_T();Oe();r_=class{static{l(this,"ValueObserver")}constructor(){this.min=1/0,this.max=-1/0,this.count=0,this.histogram=null}update(e){for(let t=0;t<e.length;t++){let n=e[t];n<this.min&&(this.min=n),n>this.max&&(this.max=n)}this.count+=e.length,this.histogram&&this.histogram.update(e)}enableHistogram(e=2048){this.histogram=new ax(e)}},ax=class{static{l(this,"HistogramCollector")}constructor(e=2048){this.numBins=e,this.bins=new Float64Array(e),this.rangeMin=0,this.rangeMax=0,this.initialized=!1,this.totalCount=0}update(e){this.initialized||this._initRange(e);let t=this.rangeMax-this.rangeMin;if(t<=0)return;for(let s=0;s<e.length;s++){let o=e[s];if(o<this.rangeMin||o>this.rangeMax){this._expandAndRebucket(e);return}}let n=this.numBins/t;for(let s=0;s<e.length;s++){let o=Math.min(this.numBins-1,Math.floor((e[s]-this.rangeMin)*n));this.bins[o]++}this.totalCount+=e.length}_initRange(e){let t=1/0,n=-1/0;for(let s=0;s<e.length;s++)e[s]<t&&(t=e[s]),e[s]>n&&(n=e[s]);t===n&&(n=t+1),this.rangeMin=t,this.rangeMax=n,this.initialized=!0}_expandAndRebucket(e){let t=this.rangeMin,n=this.rangeMax;for(let c=0;c<e.length;c++)e[c]<t&&(t=e[c]),e[c]>n&&(n=e[c]);let s=new Float64Array(this.bins),o=this.rangeMin,a=this.rangeMax-o;this.rangeMin=t,this.rangeMax=n;let u=n-t,p=this.numBins/u;if(this.bins.fill(0),a>0){let c=a/this.numBins;for(let h=0;h<this.numBins;h++){if(s[h]===0)continue;let f=o+(h+.5)*c,d=Math.min(this.numBins-1,Math.floor((f-t)*p));this.bins[d]+=s[h]}}for(let c=0;c<e.length;c++){let h=Math.min(this.numBins-1,Math.floor((e[c]-t)*p));this.bins[h]++}this.totalCount+=e.length}computePercentileThreshold(e){let t=this.totalCount*e,n=0,s=(this.rangeMax-this.rangeMin)/this.numBins;for(let o=0;o<this.numBins;o++)if(n+=this.bins[o],n>=t)return this.rangeMin+(o+1)*s;return this.rangeMax}computeEntropyThreshold(e){if(this.totalCount===0)return this.rangeMax;let t=new Float64Array(this.numBins);for(let i=0;i<this.numBins;i++)t[i]=this.bins[i]/this.totalCount;let n=this.numBins,s=1/0;for(let i=e;i<=this.numBins;i++){let a=i/e,u=new Float64Array(i);for(let c=0;c<e;c++){let h=Math.floor(c*a),f=Math.min(i,Math.floor((c+1)*a)),d=0;for(let g=h;g<f;g++)d+=t[g];let m=f-h;if(m>0){let g=d/m;for(let y=h;y<f;y++)u[y]=g}}let p=0;for(let c=0;c<i;c++)t[c]>0&&u[c]>0&&(p+=t[c]*Math.log(t[c]/u[c]));p<s&&(s=p,n=i)}let o=(this.rangeMax-this.rangeMin)/this.numBins;return this.rangeMin+n*o}},El=class{static{l(this,"CalibrationCollector")}constructor(e="minmax"){this.observers=new Map,this.mode=e}attach(e){for(let t of e.ops())for(let n=0;n<t.numResults;n++){let s=t.getResult(n);if(s.type instanceof I&&Lt(s.type.dtype)){let o=new r_;(this.mode==="entropy"||this.mode==="percentile")&&o.enableHistogram(),this.observers.set(s,o)}}for(let t of e.args)if(t.type instanceof I&&Lt(t.type.dtype)){let n=new r_;(this.mode==="entropy"||this.mode==="percentile")&&n.enableHistogram(),this.observers.set(t,n)}}observe(e,t){let n=this.observers.get(e);n&&n.update(t)}getResult(){return new ux(this.observers,this.mode)}},ux=class{static{l(this,"CalibrationResult")}constructor(e,t){this._observers=e,this._mode=t}getRange(e){let t=this._observers.get(e);return!t||t.count===0?null:{min:t.min,max:t.max}}getQuantParams(e,t,n){let s=this.getRange(e);if(!s)return null;let o=t===kr.PER_TENSOR_ASYMMETRIC;if(this._mode==="percentile"&&this._observers.get(e).histogram){if(o)return hr.fromRange(s.min,s.max,t,n);let a=this._observers.get(e).histogram.computePercentileThreshold(.999);return hr.fromRange(-a,a,t,n)}if(this._mode==="entropy"&&this._observers.get(e).histogram){if(o)return hr.fromRange(s.min,s.max,t,n);let i=this._observers.get(e).histogram,a=n==="ui8"?256:255,u=i.computeEntropyThreshold(a);return hr.fromRange(-u,u,t,n)}return hr.fromRange(s.min,s.max,t,n)}hasData(e){let t=this._observers.get(e);return t&&t.count>0}values(){return this._observers.keys()}}});function sz(r,e){let t=[],n=new Set;for(let s of r.ops())if(e.has(s.opName))for(let o=0;o<s.numOperands;o++){let i=s.getOperand(o);if(n.has(i)||!(i.type instanceof I)||!Lt(i.type.dtype))continue;let a=i.definingOp;a&&a.opName==="constant"||(n.add(i),t.push(i))}return t}function oz(r,e){let t=e.map(d=>d.type),n=new un(r.name,r.inputTypes,[...r.outputTypes,...t]),s=new Map,o=r.entryBlock.arguments,i=n.entryBlock.arguments;for(let d=0;d<o.length;d++)s.set(o[d],i[d]);let a=r.entryBlock.opsArray(),u=new Set(a),p=wo(a,d=>u.has(d),"ignore"),c=new Map;for(let d of p)d.opName!=="return"&&c.set(d,d.clone(s));for(let d of a)d.opName!=="return"&&n.entryBlock.pushOp(c.get(d));let h=r.getReturnOp(),f=[];if(h)for(let d of h.operands)f.push(s.get(d)||d);for(let d of e)f.push(s.get(d));return n.entryBlock.pushOp(new Re("return",f,[],{})),n}function QR(r,e,t,n={}){if(e.isGPU&&e.isGPU())throw new Error("collectCalibration: synchronous calibration is unavailable for async (GPU) targets; precompute calibration on a CPU/WASM target or supply quantization.calibration directly");if(!t||t.length===0)throw new Error("collectCalibration: at least one calibration batch is required");let s=n.quantizableOps||rz,o=n.mode||"minmax",i=sz(r,s),a=new El(o);a.attach(r);let u=a.getResult();if(i.length===0)return u;let p=oz(r,i),c=new wn("__calibrate__");c.addFunction(p);let h=r.outputTypes.length,d=(n.compileFn||iz)(c,e);for(let m of t){let g=Array.isArray(m)?m:[m],y=p.outputTypes.map(_=>new Float32Array(Math.max(1,_.numel())));d.run(r.name,...g,...y);for(let _=0;_<i.length;_++)a.observe(i[_],y[h+_])}return u}function iz(r,e){throw new Error("collectCalibration: opts.compileFn is required (pass the compileModule function to avoid a circular import)")}var rz,XR=M(()=>{"use strict";xs();Yo();sr();Oe();ji();lx();rz=new Set(["dot","conv","add","mul","sub"]);l(sz,"activationTargets");l(oz,"buildCaptureFunction");l(QR,"collectCalibration");l(iz,"defaultCompileFn")});function px(r){let e=0;for(let t=0;t<r.numResults;t++){let n=r.getResult(t).type;n instanceof I&&n.isFullyStatic&&(e+=n.sizeInBytes())}return e}var cx,Ll,fx,s_,eM=M(()=>{"use strict";At();Oe();nf();cx=class{static{l(this,"Partition")}constructor(e,t){this.id=e,this.target=t,this.ops=[],this.opSet=new Set,this._inputValues=null,this._outputValues=null,this._memoryBytes=0}addOp(e){this.opSet.has(e)||(this.ops.push(e),this.opSet.add(e),this._inputValues=null,this._outputValues=null,this._memoryBytes+=px(e))}hasOp(e){return this.opSet.has(e)}merge(e){for(let t of e.ops)this.addOp(t)}computeIO(){if(this._inputValues&&this._outputValues)return;let{inputs:e,outputs:t}=_i(this.opSet,this.ops);this._inputValues=e,this._outputValues=t}getInputValues(){return this.computeIO(),this._inputValues}getOutputValues(){return this.computeIO(),this._outputValues}get size(){return this.ops.length}get memoryBytes(){return this._memoryBytes}};l(px,"estimateOpMemory");Ll=class{static{l(this,"PartitionerConfig")}constructor(e={}){this.targets=e.targets||[],this.defaultTarget=e.defaultTarget||null,this.opTargetOverrides=e.opTargetOverrides||new Map,this.memoryLimits=e.memoryLimits||new Map,this.minPartitionSize=e.minPartitionSize||1,this.costWeights={transferCost:1,loadBalance:.5,...e.costWeights||{}}}},fx=class{static{l(this,"PartitionResult")}constructor(e,t,n){this.partitions=e,this.opToPartition=t,this.transferEdges=n}getPartition(e){return this.opToPartition.get(e)||null}getPartitionsForTarget(e){return this.partitions.filter(t=>t.target===e||t.target.name===e.name)}get numPartitions(){return this.partitions.length}},s_=class{static{l(this,"GraphPartitioner")}constructor(e){this.config=e instanceof Ll?e:new Ll(e),this._supportCache=new Map,this._buildSupportMap()}partition(e){let t=this._collectPartitionableOps(e),n=this._assignTargets(t),s=this._buildPartitions(t,n),o=this._mergeSmallPartitions(s,n),i=this._computeTransferEdges(o),a=new Map;for(let u of o)for(let p of u.ops)a.set(p,u);return new fx(o,a,i)}_buildSupportMap(){for(let e of this.config.targets){let t=new Set;for(let n of ne.names())this._targetSupportsOp(e,n)&&t.add(n);this._supportCache.set(e,t)}}_targetSupportsOp(e,t){if(e.hasLibraryOp(t))return!0;let n=ne.get(t);return n?n.isConstant||n.isTerminator?!0:e.isGPU()?n.isElementwise||n.isReduction||n.isBroadcast||n.isInjective||n.getAttr("gpuCapable")===!0:e.isCPU()?!0:e.isWasm()?!n.isOpaque&&t!=="custom_call":!1:!1}_collectPartitionableOps(e){let t=[];for(let n of e.ops()){let s=ne.get(n.opName);!s||s.isTerminator||t.push(n)}return t}_assignTargets(e){let t=new Map;for(let n of e){let s=n.getAttr("device");if(s){let a=this._resolveDeviceAttr(s);if(a){t.set(n,a);continue}}let o=this.config.opTargetOverrides.get(n.opName);if(o){t.set(n,o);continue}let i=this._selectBestTarget(n);t.set(n,i)}return t}_resolveDeviceAttr(e){return typeof e=="string"?this.config.targets.find(t=>t.name===e||t.kind===e)||null:e}_selectBestTarget(e){let t=this.config.defaultTarget||this.config.targets[0],n=-1/0;for(let s of this.config.targets){let o=this._supportCache.get(s);if(!o||!o.has(e.opName))continue;let i=this._scoreTargetForOp(s,e);i>n&&(n=i,t=s)}return t}_scoreTargetForOp(e,t){let n=ne.get(t.opName);if(!n)return 0;let s=0;if(e.hasLibraryOp(t.opName)&&(s+=100),n.isReduction||n.isElementwise){let o=0;for(let i=0;i<t.numOperands;i++){let a=t.getOperand(i).type;a instanceof I&&a.isFullyStatic&&(o+=a.numel())}e.isGPU()&&o>1024?s+=50:e.isCPU()&&o<=1024&&(s+=30)}return s+=e.computeTFLOPs*10,s}_buildPartitions(e,t){let{partitions:n}=tf(e,{sort:l(i=>this._topologicalSort(i),"sort"),labelOf:l(i=>t.get(i),"labelOf"),sameLabel:l((i,a)=>i===a||i.name===a.name,"sameLabel"),canMerge:l((i,a,u)=>this._fitsMemoryLimit(i,a,u),"canMerge"),onAttach:l((i,a)=>{i.memoryBytes=(i.memoryBytes||0)+px(a)},"onAttach")}),s=new Map;for(let i of n){let a=new cx(i.id,i.label);for(let u of i.ops)a.addOp(u);s.has(i.label.name)||s.set(i.label.name,[]),s.get(i.label.name).push(a)}let o=[];for(let i of s.values())for(let a of i)o.push(a);return o}_fitsMemoryLimit(e,t,n){let s=this.config.memoryLimits.get(n.name);if(!s)return!0;let o=px(t);return e.memoryBytes+o<=s}_mergeSmallPartitions(e,t){if(e.length<=1)return e;let n=new Map;for(let d of e)for(let m of d.ops)n.set(m,d);let s=new Set,o=new Map,i=new Map;for(let d of e)s.has(d)||o.set(d,new Set);for(let d of e){if(s.has(d))continue;let m=o.get(d);for(let g of d.ops)for(let y=0;y<g.numResults;y++)for(let _ of g.getResult(y).uses()){let b=n.get(_.user);b&&b!==d&&m.add(b)}}let a=l((d,m)=>{let g=o.get(d);for(let y of o.get(m))y!==d&&g.add(y);g.delete(m);for(let[y,_]of o)_.has(m)&&(_.delete(m),y!==d&&_.add(d));o.delete(m),i=new Map},"mergeSucc"),u=l(d=>{let m=i.get(d);if(m)return m;m=new Set;let g=[...o.get(d)];for(;g.length>0;){let y=g.pop();if(m.has(y))continue;m.add(y);let _=o.get(y);if(_)for(let b of _)g.push(b)}return i.set(d,m),m},"reachOf"),p=l((d,m)=>{for(let g of o.get(d))if(g!==m&&u(g).has(m))return!0;return!1},"pathThroughIntermediate"),c=l((d,m)=>p(d,m)||p(m,d),"mergeCreatesCycle"),h=[];for(let d=0;d<e.length;d++){let m=e[d];if(s.has(m))continue;if(m.size>=this.config.minPartitionSize){h.push(m);continue}let g=-1,y=-1/0;for(let _=0;_<e.length;_++){if(d===_)continue;let b=e[_];if(s.has(b)||b.target.name!==m.target.name||c(m,b))continue;let w=this.config.costWeights.transferCost*this._mergeScore(m,b);w>y&&(y=w,g=_)}if(g>=0){let _=e[g];a(_,m),_.merge(m);for(let b of m.ops)n.set(b,_);s.add(m)}else h.push(m)}let f=new Set;for(let d of h)for(let m of d.ops)f.add(m);for(let d of e){let m=!1;for(let g of d.ops)if(!f.has(g)){m=!0;break}if(m){h.push(d);for(let g of d.ops)f.add(g)}}return h}_mergeScore(e,t){let n=0,s=new Set;for(let o of e.ops)for(let i=0;i<o.numResults;i++)s.add(o.getResult(i));for(let o of t.ops)for(let i=0;i<o.numOperands;i++)s.has(o.getOperand(i))&&n++;return n}_computeTransferEdges(e){let t=[],n=new Set,s=new Map;for(let o of e)for(let i of o.ops)s.set(i,o);for(let o of e)for(let i of o.ops)for(let a=0;a<i.numOperands;a++){let u=i.getOperand(a).definingOp;if(!u)continue;let p=s.get(u);if(!p||p===o)continue;let c=i.getOperand(a),h=`${p.id}|${o.id}|${c.id}`;n.has(h)||(n.add(h),t.push({src:p,dst:o,value:c,sizeBytes:c.type instanceof I&&c.type.isFullyStatic?c.type.sizeInBytes():0}))}return t}_topologicalSort(e){let t=new Set(e),n=new Map,s=new Map;for(let u of e)n.set(u,0),s.set(u,[]);for(let u of e)for(let p=0;p<u.numOperands;p++){let c=u.getOperand(p).definingOp;c&&t.has(c)&&(s.get(c).push(u),n.set(u,n.get(u)+1))}let o=[];for(let u of e)n.get(u)===0&&o.push(u);let i=[],a=0;for(;a<o.length;){let u=o[a++];i.push(u);for(let p of s.get(u)){let c=n.get(p)-1;n.set(p,c),c===0&&o.push(p)}}return i}}});var o_,i_,tM=M(()=>{"use strict";nn();sr();xs();eM();nf();Kt();o_=class extends Ge{static{l(this,"GraphPartitionPass")}constructor(e={}){super("GraphPartitionPass"),this.partitionerConfig=new Ll(e),this.partitionResult=null}run(e,t){if(this.partitionerConfig.targets.length<2)return oe.UNCHANGED;let n=new s_(this.partitionerConfig);return this.partitionResult=n.partition(e),this.partitionResult.numPartitions<=1?oe.UNCHANGED:(this._annotateOps(),this._insertTransferOps(e),this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,numPartitions:this.partitionResult.numPartitions,transferEdges:this.partitionResult.transferEdges.length,level:ue.DEBUG}),e.bumpVersion(),oe.CHANGED)}_annotateOps(){for(let e of this.partitionResult.partitions)for(let t of e.ops)t.setAttr("partition_id",e.id),t.setAttr("partition_target",e.target.name)}_insertTransferOps(e){let t=e.entryBlock,n=this._buildOrderIndex(t),{useMap:s,firstInPart:o}=this._buildInsertionIndex(t);for(let i of this.partitionResult.transferEdges){let a=i.value,u=i.src.target.name,p=i.dst.target.name;if(u===p)continue;let c=new Re("copy_to_device",[a],[a.type],{src_device:u,dst_device:p});c.setAttr("partition_id",i.dst.id),c.setAttr("partition_target",p);let h=s.get(i.dst),f=h&&h.get(a)||o.get(i.dst)||null,d=a.definingOp;if(f&&d&&n.has(d)&&n.has(f)&&n.get(d)>=n.get(f))t.insertAfter(c,d);else if(f)t.insertBefore(c,f);else if(d&&n.has(d))t.insertAfter(c,d);else{let g=e.getReturnOp();g?t.insertBefore(c,g):t.pushOp(c)}let m=c.getResult(0);for(let g of i.dst.ops)for(let y=0;y<g.numOperands;y++)g.getOperand(y)===a&&g.replaceOperand(y,m)}}_buildInsertionIndex(e){let t=[...new Set(this.partitionResult.transferEdges.map(o=>o.dst))],n=new Map,s=new Map;for(let o of t)n.set(o,new Map);for(let o of e.ops())for(let i of t){if(!i.hasOp(o))continue;s.has(i)||s.set(i,o);let a=n.get(i);for(let u=0;u<o.numOperands;u++){let p=o.getOperand(u);a.has(p)||a.set(p,o)}}return{useMap:n,firstInPart:s}}_buildOrderIndex(e){let t=new Map,n=0;for(let s of e.ops())t.set(s,n++);return t}},i_=class extends Ge{static{l(this,"PartitionMaterializationPass")}constructor(e={}){super("PartitionMaterializationPass"),this.targets=e.targets||[]}run(e,t){let n=this._collectPartitions(e);if(n.size<=1)return oe.UNCHANGED;let s=this._getModule(e);if(!s)return oe.UNCHANGED;let o=this._materializePartitions(e,n);for(let i of o)s.addFunction(i);return this._rewriteOriginalFunction(e,o,n),this.trace&&this.trace.level>=ue.DEBUG&&this.trace.emit({type:"pass_detail",passName:this.name,partitions:n.size,subFunctions:o.length,level:ue.DEBUG}),e.bumpVersion(),oe.CHANGED}_collectPartitions(e){let t=new Map;for(let n of e.ops()){let s=n.getAttr("partition_id");s!==void 0&&(t.has(s)||t.set(s,{id:s,target:n.getAttr("partition_target"),ops:[]}),t.get(s).ops.push(n))}return t}_getModule(e){return e._module||null}_materializePartitions(e,t){let n=[];for(let[s,o]of t){let i=new Set(o.ops),{inputs:a,outputs:u}=_i(i,o.ops),p=a.map(_=>_.type),c=u.map(_=>_.type),h=`${e.name}_partition_${s}`,f=new un(h,p,c);f._partitionTarget=o.target;let d=new Map;for(let _=0;_<a.length;_++)d.set(a[_],f.args[_]);let m=Ba(o.ops);for(let _ of m)f.entryBlock.pushOp(_.clone(d));let g=u.map(_=>d.get(_)||_),y=new Re("return",g,[]);f.entryBlock.pushOp(y),n.push(f)}return n}_rewriteOriginalFunction(e,t,n){for(let s of t)e.setAttr?.(`sub_${s.name}`,s._partitionTarget)}}});function wf(r){return sc(r,-1)}function az(r,e,t){let n=t.sharedMemoryBytes||16384,s=0;for(let u of e.entryBlock.ops())if(u.opName!=="yield")for(let p=0;p<u.numResults;p++){let c=u.getResult(p).type;if(!c||!c.shape)continue;let h=wf(c.shape);if(h<0)return!0;let f=h*bn(c.dtype);f>s&&(s=f)}let o=r.getAttr("num_carry"),i=r.getAttr("num_xs"),a=0;for(let u=0;u<o;u++){let p=r.getOperand(i+u).type,c=p&&p.shape?wf(p.shape):-1;c>0&&(a+=c*bn(p.dtype))}return 3*s+2*a>n}function uz(r,e){let n=r.regions[0].entryBlock,s=r.getAttr("num_carry"),o=[],i=null;for(let b of n.ops()){if(b.opName==="yield"){i=b;continue}o.push(b)}if(!i)return null;let a=new Set(o),u=new Set(n.arguments),p=[],c=new Set,h=[],f=new Set;for(let b of o)for(let w=0;w<b.numOperands;w++){let x=b.getOperand(w),S=x.definingOp;if(!u.has(x)&&!(S&&a.has(S))){if(S&&oo(S)){f.has(S)||(f.add(S),h.push(S));continue}c.has(x)||(c.add(x),p.push(x))}}let d=[...n.arguments,...p];for(let b of d)if(!b.type||!b.type.isFullyStatic)return null;let m=[];for(let b=0;b<i.numOperands;b++)m.push(i.getOperand(b));let g=new un(e,d.map(b=>b.type),m.map(b=>b.type)),y=new Map;for(let b=0;b<d.length;b++)y.set(d[b],g.args[b]);for(let b of h)g.entryBlock.pushOp(b.clone(y));for(let b of o)g.entryBlock.pushOp(b.clone(y));let _=m.map(b=>y.get(b));return _.some(b=>b===void 0)?null:(g.entryBlock.pushOp(new Re("return",_,[])),{bodyFunc:g,captured:p,numCarry:s,numYs:m.length-s})}function lz(r,e,t,n,s){let{getSlot:o,newSlot:i,steps:a,addedFuncs:u}=n,p=new wn(r.name+"_mod");p.addFunction(r);let c=Ti(p,s);if(c){let h=c.plan,f=e.length,d=new Map,m=!0;for(let g of h.returnFixups||[]){if(g.kind!=="copy"){m=!1;break}d.set(g.pos,g.srcSlot)}if(m){let g=l(x=>{let S=f+x;return d.has(S)?d.get(S):h.argSlots[S]},"retSlot"),y=new Map;for(let x=0;x<f;x++)y.set(h.argSlots[x],o(e[x]));for(let x of h.intermediates)y.set(x.slot,i(x.shape,x.dtype));for(let x=0;x<t.length;x++){let S=g(x);y.has(S)||y.set(S,o(t[x]))}let _=l(x=>{let S=y.get(x);return S===void 0?null:S},"mapSlot"),b=[],w=!0;for(let x of h.steps){let S=x.inputSlots.map(_),T=x.outputSlots.map(_);if(S.includes(null)||T.includes(null)){w=!1;break}b.push({name:x.name,inputSlots:S,outputSlots:T})}if(w){for(let x of b)a.push(x);for(let x of p.functions())u.push(x);return!0}}}return a.push({name:r.name,inputSlots:e.map(o),outputSlots:t.map(o)}),u.push(r),!0}function nM(r,e,t){let n=dy({ops:r,opSet:new Set(r)},e,new Map);return n?lz(n.subFunc,n.inputs,n.outputs,t,1):!1}function cz(r,e,t){let{getSlot:n,newSlot:s,steps:o,scanLoops:i,addedFuncs:a}=t,u=r.regions[0];if(!u||!u.entryBlock)return!1;let p=uz(r,e);if(!p)return!1;let{bodyFunc:c,captured:h,numCarry:f,numYs:d}=p,m=r.getAttr("num_xs"),g=new wn(e+"_mod");g.addFunction(c);let y=Ti(g,2),_=y?y.plan:null,b=[],w=[];for(let P=0;P<f;P++){let V=r.getOperand(m+P).type;b.push(V.shape),w.push(V.dtype)}let x=[],S=[];for(let P=0;P<m;P++){let V=r.getOperand(P).type;x.push(V.shape.slice(1)),S.push(V.dtype)}let T=[],k=[];for(let P=0;P<d;P++){let V=r.getResult(f+P).type;T.push(V.shape.slice(1)),k.push(V.dtype)}let A=b.map((P,V)=>s(P,w[V])),N=b.map((P,V)=>s(P,w[V])),v=x.map((P,V)=>s(P,S[V])),O=[],R=[],B=[],E=[];for(let P=0;P<f;P++)R.push(n(r.getOperand(m+P)));for(let P=0;P<f;P++)B.push(n(r.getResult(P)));for(let P of h)E.push(n(P));let z=[],D=[];for(let P=0;P<m;P++)z.push(n(r.getOperand(P)));for(let P=0;P<d;P++)D.push(n(r.getResult(f+P)));let J=o.length;if(!_)O=T.map((P,V)=>s(P,k[V])),o.push({name:c.name,inputSlots:[...v,...A,...E],outputSlots:[...N,...O]}),a.push(c);else{let P=m+f+h.length,V=new Map;for(let G of _.returnFixups||[]){if(G.kind!=="copy")return!1;V.set(G.pos,G.srcSlot)}let he=l(G=>{let K=P+G;return V.has(K)?V.get(K):_.argSlots[K]},"bodyReturnSlot"),le=new Map;for(let G=0;G<m;G++)le.set(_.argSlots[G],v[G]);for(let G=0;G<f;G++)le.set(_.argSlots[m+G],A[G]);for(let G=0;G<h.length;G++)le.set(_.argSlots[m+f+G],E[G]);for(let G of _.intermediates)le.set(G.slot,s(G.shape,G.dtype));for(let G=0;G<f;G++){let K=he(G);le.has(K)||le.set(K,N[G])}for(let G=0;G<d;G++){let K=he(f+G),ie=le.get(K);ie===void 0&&(ie=s(T[G],k[G]),le.set(K,ie)),O.push(ie)}let Ie=l(G=>{let K=le.get(G);return K===void 0?null:K},"mapSlot");for(let G of _.steps){let K=G.inputSlots.map(Ie),ie=G.outputSlots.map(Ie);if(K.includes(null)||ie.includes(null))return!1;o.push({name:G.name,inputSlots:K,outputSlots:ie})}for(let G of g.functions())a.push(G)}let j=o.length,U=r.getOperand(0).type.shape[0];return typeof U!="number"||U<0?!1:(i.push({T:U,loopStart:J,loopEnd:j,carry:b.map((P,V)=>({a:A[V],b:N[V],initSlot:R[V],finalSlot:B[V],bytes:wf(P)*bn(w[V])})),xs:v.map((P,V)=>({xtSlot:P,xsSlot:z[V],stepBytes:wf(x[V])*bn(S[V])})),ys:O.map((P,V)=>({ytSlot:P,ysSlot:D[V],stepBytes:wf(T[V])*bn(k[V])}))}),!0)}function hx(r,e,t=!1){if(!e||typeof e.isWebGPU!="function"||!e.isWebGPU()||r.functionCount!==1)return null;let n=r.functions().next().value,s=n.getReturnOp();if(!s)return null;let o=[];for(let R of n.ops())R.opName==="scan"&&o.push(R);if(o.length===0)return null;let i=new Set,a=l(R=>{if(!(!R||i.has(R))){i.add(R);for(let B=0;B<R.numOperands;B++)a(R.getOperand(B).definingOp)}},"visit");for(let R=0;R<s.numOperands;R++)a(s.getOperand(R).definingOp);for(let R of o)if(!i.has(R))return null;if(!t){let R=!1;for(let B of o){let E=B.regions[0];if(E&&E.entryBlock&&az(B,E,e)){R=!0;break}}if(!R)return null}let u=[];for(let R of i)oo(R)||qt(R.opName)||u.push(R);let p=Ba(u),c=new Map,h=0,f=l(R=>{let B=c.get(R);return B===void 0&&(B=h++,c.set(R,B)),B},"getSlot");for(let R of n.args)f(R);let d=[],m=l((R,B)=>{let E=h++;return d.push({slot:E,shape:[...R],dtype:B}),E},"newSlot"),g=[],y=[],_=[],b={getSlot:f,newSlot:m,steps:g,scanLoops:y,addedFuncs:_},w=new Set(o),x=[],S=0,T=0;for(let R of p)if(w.has(R)){if(x.length&&!nM(x,n.name+"_seg"+S++,b)||(x=[],!cz(R,n.name+"_scan"+T++,b)))return null}else x.push(R);if(x.length&&!nM(x,n.name+"_seg"+S++,b))return null;let k=n.args.map(f),A=[],N=new Set;for(let R=0;R<s.numOperands;R++){let B=s.getOperand(R),E=k.length;if(c.has(B)&&!(B.isBlockArgument&&B.isBlockArgument())){let z=f(B);N.has(z)?(k.push(h++),A.push({pos:E,kind:"copy",srcSlot:z})):(N.add(z),k.push(z))}else if(B.isBlockArgument&&B.isBlockArgument())k.push(h++),A.push({pos:E,kind:"copy",srcSlot:f(B)});else return null}let v=new Set(k),O=new Set;for(let[R,B]of c)if(!(v.has(B)||O.has(B))){if(O.add(B),!R.type||!R.type.isFullyStatic)return null;d.some(E=>E.slot===B)||d.push({slot:B,shape:[...R.type.shape],dtype:R.type.dtype})}r.removeFunction(n.name);for(let R of _)r.addFunction(R);return{plan:{numSlots:h,argSlots:k,intermediates:d,steps:g,returnFixups:A,scanLoops:y}}}var rM=M(()=>{"use strict";sr();xs();Yo();my();Tr();Vn();Oe();l(wf,"numel");l(az,"isScanOversized");l(uz,"buildScanBodyFunction");l(lz,"inlineNativeSplit");l(nM,"emitSegment");l(cz,"emitScanLoop");l(hx,"splitGraphForScan")});function Tf(r){dx.push(r),dx.sort((e,t)=>(e.priority??100)-(t.priority??100))}function pz(r){for(let e of dx)if(e.applies(r))return e;return null}function sM(r,e){let t=pz(e);return t?t.run(r,e):null}var dx,oM=M(()=>{"use strict";my();rM();dx=[];l(Tf,"registerGraphSplitStrategy");l(pz,"selectGraphSplitStrategy");l(sM,"splitGraph");Tf({name:"cublas",priority:10,applies:l(r=>r.config.matmulBackend==="cublas","applies"),run:l(r=>TB(r),"run")});Tf({name:"cuda-attention",priority:15,applies:l(r=>r.cudaAttention,"applies"),run:l(r=>Ti(r,1),"run")});Tf({name:"cuda-native-matmul-chain",priority:20,applies:l(r=>r.cudaMatmulChain,"applies"),run:l(r=>Ti(r),"run")});Tf({name:"cuda-native-conv-chain",priority:25,applies:l(r=>r.cudaConvChain,"applies"),run:l(r=>Ti(r),"run")});Tf({name:"webgpu",priority:30,applies:l(r=>r.isWebGPU,"applies"),run:l((r,e)=>{let t=hx(r,e.target);return!t&&bB(r,e.target.maxThreadsPerBlock||256)&&(t=hx(r,e.target,!0),t||(t=Ti(r,2))),t},"run")})});function iM(r,e){let t=new od;for(let[,n]of r.bufferMap)t.paramBuffers.add(n.name);return fz(r.body,t,e),dz(r,t,e),mz(r.body,t),t}function fz(r,e,t){let n=[r];for(;n.length>0;){let s=n.pop();if(!(!s||typeof s!="object")){switch(s.type){case"ForNode":if(s.loopVar&&e.locals.set(s.loopVar.name,"i32"),s.kind===F.THREAD_BINDING&&s.threadTag){let o=s.extent&&s.extent.type==="IntImmNode"?s.extent.value:0,i=!s.extent||s.extent.type!=="IntImmNode",a={varName:s.loopVar.name,extent:o,isDynamic:i,extentNode:s.extent};e.threadBindings.has(s.threadTag)?e.threadBindings.get(s.threadTag).push(a):e.threadBindings.set(s.threadTag,[a])}break;case"LetStmtNode":s.variable&&e.locals.set(s.variable.name,ei(s.variable.dtype));break;case"BufferStoreNode":case"BufferLoadNode":s.buffer&&e.usedBuffers.set(s.buffer.name,s.buffer);break;case"AllocateNode":s.buffer&&e.allocatedBuffers.add(s.buffer.name),s.scope==="shared"&&s.buffer&&e.sharedBuffers.push(s.buffer);break;case"CallExternNode":s.externName&&!cv(s.externName)&&e.externCalls.set(s.externName,{argCount:s.args?s.args.length:0,dtype:s.dtype||"f32"});break;case"BlockNode":if(s.reads)for(let o of s.reads)o.buffer&&e.usedBuffers.set(o.buffer.name,o.buffer);if(s.writes)for(let o of s.writes)o.buffer&&e.usedBuffers.set(o.buffer.name,o.buffer);if(s.iterVars)for(let o of s.iterVars)o.iterVar&&e.locals.set(o.iterVar.name,ei(o.iterVar.dtype)),o.binding&&typeof o.binding=="object"&&o.binding.type&&n.push(o.binding);break}if(s.body&&n.push(s.body),s.value&&typeof s.value=="object"&&s.value.type&&n.push(s.value),s.stmts)for(let o of s.stmts)n.push(o);if(s.thenBody&&n.push(s.thenBody),s.elseBody&&n.push(s.elseBody),s.initBody&&n.push(s.initBody),s.condBody&&n.push(s.condBody),s.loopBody&&n.push(s.loopBody),s.condition&&typeof s.condition=="object"&&s.condition.type&&n.push(s.condition),s.a&&typeof s.a=="object"&&s.a.type&&n.push(s.a),s.b&&typeof s.b=="object"&&s.b.type&&n.push(s.b),s.expr&&typeof s.expr=="object"&&s.expr.type&&n.push(s.expr),s.args)for(let o of s.args)typeof o=="object"&&o!==null&&o.type&&n.push(o);if(s.indices)for(let o of s.indices)typeof o=="object"&&o!==null&&o.type&&n.push(o)}}}function dz(r,e,t){let n=e.memoryLayout.alignment,s=0,o=l(i=>{let a=i.shape.some(c=>typeof c!="number"||c<0),u=i.numel();if(!a&&u>=0)return u*bn(i.dtype);let p=1;for(let c of i.shape)p*=typeof c=="number"&&c>0?c:1;return Math.max(hz,p*bn(i.dtype))},"bufBytes");for(let[,i]of r.bufferMap)s=Math.ceil(s/n)*n,e.memoryLayout.bufferOffsets.set(i.name,s),s+=o(i);for(let[i,a]of e.usedBuffers)e.memoryLayout.bufferOffsets.has(i)||(s=Math.ceil(s/n)*n,e.memoryLayout.bufferOffsets.set(i,s),s+=o(a));e.memoryLayout.totalBytes=s}function mz(r,e){let t=new Map;for(let n of gu(r,s=>s.type==="BufferStoreNode"&&s.buffer&&!e.paramBuffers.has(s.buffer.name))){let s=n.buffer.name;t.has(s)||t.set(s,[]),t.get(s).push(n.value)}for(let[n,s]of t)if(s.every(i=>i&&i.type==="IntImmNode"&&i.value===0||i&&i.type==="FloatImmNode"&&i.value===0)&&e.zeroBuffers.add(n),s.length===1&&s[0]){let i=s[0];(i.type==="IntImmNode"||i.type==="FloatImmNode")&&e.constantBuffers.set(n,i.value)}}var hz,aM=M(()=>{"use strict";Tc();Vn();De();Gr();l(iM,"scanMetadata");l(fz,"walkTree");hz=65536;l(dz,"computeMemoryLayout");l(mz,"detectZeroBuffers")});function xf(r,e,t){let n=typeof r.offset=="number"?r.offset:0;if(e.length===0)return new L(n);if(e.length===1)return n===0?e[0]:new C("+",e[0],new L(n));let s=[];n!==0&&s.push(new L(n));for(let o=0;o<e.length;o++){let i=e[o];if(i.type==="IntImmNode"&&i.value===0)continue;let a=r.strides[o];if(a===1)s.push(i);else if(typeof a=="number"&&a>=0)s.push(new C("*",i,new L(a)));else{let u=gz(r,o,t);s.push(new C("*",i,u))}}return s.length===0?new L(0):s.reduce((o,i)=>new C("+",o,i))}function gz(r,e,t){let n=[];for(let s=e+1;s<r.shape.length;s++){let o=r.shape[s];typeof o=="number"&&o>=0?n.push(new L(o)):n.push(yz(r,s,t))}return n.length===0?new L(1):n.reduce((s,o)=>new C("*",s,o))}function yz(r,e,t){let n=r.shape[e];if(n instanceof Ce)return uu(n,s=>new Ne(qa(s),"index"));if(t){let s=`${r.name}:${e}`,o=t.get(s);if(o)return new Ne(o.name,"index")}return new L(1)}var uM=M(()=>{"use strict";De();Vs();Eb();l(xf,"flattenIndex");l(gz,"computeDynamicStride");l(yz,"resolveShapeParam")});function lM(r,e){let t=iM(r,e),n={shapeParamMap:r.shapeParamMap,accCounter:0,metadata:t},s=is(r.body,n);return new rd(r.name,r.params,s,r.bufferMap,r.shapeParams,r.shapeParamMap,t)}function is(r,e){if(!r||typeof r!="object")return r;switch(r.type){case"ForNode":return _z(r,e);case"BlockNode":return bz(r,e);case"SeqNode":return wz(r,e);case"BufferStoreNode":return Tz(r,e);case"LetStmtNode":return xz(r,e);case"AllocateNode":return Sz(r,e);case"IfThenElseNode":return kz(r,e);case"WhileNode":return vz(r,e);case"EvaluateNode":return new ou(mr(r.value,e));case"SyncThreadsNode":return r;default:return r}}function _z(r,e){let t=r.accumulator!==void 0?r.accumulator:Xy(r);if(t)return Nz(r,t,e);let n=is(r.body,e);return new de(r.loopVar,r.min,r.extent,r.kind,n,r.threadTag)}function bz(r,e){let t=[];for(let o of r.iterVars)o.iterVar&&o.binding&&t.push({name:o.iterVar.name,dtype:o.iterVar.dtype,expr:mr(o.binding,e)});let n=is(r.body,e),s=r.initBody?is(r.initBody,e):null;return t.length>0?s?(n=new te([new mu(t,s),new mu(t,n)]),s=null):n=new mu(t,n):s&&(n=new te([s,n])),n}function wz(r,e){let t=[];for(let n of r.stmts)t.push(is(n,e));return new te(t)}function Tz(r,e){let t=xf(r.buffer,r.indices,e.shapeParamMap),n=mr(r.value,e),s=r.buffer.dtype||ir(r.value);return new wc(r.buffer,t,n,s)}function xz(r,e){return new Ye(r.variable,mr(r.value,e),is(r.body,e))}function Sz(r,e){return new Ve(r.buffer,r.scope,is(r.body,e))}function kz(r,e){return new xe(mr(r.condition,e),is(r.thenBody,e),r.elseBody?is(r.elseBody,e):null)}function vz(r,e){return new Zo(r.condVar,is(r.condBody,e),is(r.loopBody,e))}function mr(r,e){if(!r||typeof r!="object"||!r.type)return r;switch(r.type){case"BufferLoadNode":{let t=xf(r.buffer,r.indices,e.shapeParamMap),n=new bc(r.buffer,t,r.buffer.dtype);return No(n),n}case"MathOpNode":{let t=mr(r.a,e),n=r.b?mr(r.b,e):null,s={...r,a:t,b:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),No(s),s}case"CompareNode":{let t=mr(r.a,e),n=mr(r.b,e),s={...r,a:t,b:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),No(s),s}case"CastNode":{let t=mr(r.expr,e),n={...r,expr:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),No(n),n}case"CallExternNode":{let t=r.args.map(s=>mr(s,e)),n={...r,args:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),No(n),n}case"IfThenElseNode":{let t=mr(r.condition,e),n=mr(r.thenBody,e),s=r.elseBody?mr(r.elseBody,e):null,o={...r,condition:t,thenBody:n,elseBody:s};return Object.setPrototypeOf(o,Object.getPrototypeOf(r)),No(o),o}default:return No(r),r}}function Nz(r,e,t){let n=`_acc_${t.accCounter++}`,s=ir(e.loadSide);t.metadata.locals.set(n,s);let o=new Map;for(let d of e.block.iterVars)d.iterVar&&d.binding&&o.set(d.iterVar.name,d.binding);let i=o.size>0?Ar(e.valueSide,o):e.valueSide,a=xf(e.store.buffer,e.outerIndices,t.shapeParamMap),u=new bc(e.store.buffer,a,e.store.buffer.dtype);No(u);let p=mr(i,t),c=xf(e.store.buffer,e.outerIndices,t.shapeParamMap),h=new wc(e.store.buffer,c,null,e.store.buffer.dtype),f=e.block.initBody?o.size>0?Sf(e.block.initBody,o):e.block.initBody:null;return new sd({localName:n,dtype:s,op:e.op,initLoad:u,loopVar:r.loopVar,extent:r.extent,loopKind:r.kind,body:p,flushStore:h,initBody:f?is(f,t):null})}function Ar(r,e){if(!r||typeof r!="object"||!r.type)return r;if(r.type==="VariableNode"&&e.has(r.name))return e.get(r.name);if(r.type==="BufferLoadNode"){let t=r.indices.map(o=>Ar(o,e));if(!t.some((o,i)=>o!==r.indices[i]))return r;let s={...r,indices:t};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}if(r.type==="MathOpNode"){let t=Ar(r.a,e),n=r.b?Ar(r.b,e):null;if(t===r.a&&n===r.b)return r;let s={...r,a:t,b:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}if(r.type==="CompareNode"){let t=Ar(r.a,e),n=Ar(r.b,e);if(t===r.a&&n===r.b)return r;let s={...r,a:t,b:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}if(r.type==="CastNode"){let t=Ar(r.expr,e);if(t===r.expr)return r;let n={...r,expr:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),n}if(r.type==="CallExternNode"){let t=r.args.map(o=>Ar(o,e));if(!t.some((o,i)=>o!==r.args[i]))return r;let s={...r,args:t};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}if(r.type==="IfThenElseNode"){let t=Ar(r.condition,e),n=Ar(r.thenBody,e),s=r.elseBody?Ar(r.elseBody,e):null;if(t===r.condition&&n===r.thenBody&&s===r.elseBody)return r;let o={...r,condition:t,thenBody:n,elseBody:s};return Object.setPrototypeOf(o,Object.getPrototypeOf(r)),o}return r}function Sf(r,e){if(!r||typeof r!="object"||!r.type)return r;switch(r.type){case"BufferStoreNode":{let t=r.indices.map(o=>Ar(o,e)),n=Ar(r.value,e);if(t.every((o,i)=>o===r.indices[i])&&n===r.value)return r;let s={...r,indices:t,value:n};return Object.setPrototypeOf(s,Object.getPrototypeOf(r)),s}case"SeqNode":{let t=r.stmts.map(s=>Sf(s,e)),n={...r,stmts:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),n}case"ForNode":{let t=Sf(r.body,e);if(t===r.body)return r;let n={...r,body:t};return Object.setPrototypeOf(n,Object.getPrototypeOf(r)),n}case"IfThenElseNode":{let t=Ar(r.condition,e),n=Sf(r.thenBody,e),s=r.elseBody?Sf(r.elseBody,e):null,o={...r,condition:t,thenBody:n,elseBody:s};return Object.setPrototypeOf(o,Object.getPrototypeOf(r)),o}default:return r}}var cM=M(()=>{"use strict";Tc();aM();uM();De();sx();l(lM,"lowerToLIR");l(is,"lowerStmt");l(_z,"lowerForNode");l(bz,"lowerBlockNode");l(wz,"lowerSeqNode");l(Tz,"lowerBufferStore");l(xz,"lowerLetStmt");l(Sz,"lowerAllocate");l(kz,"lowerIfThenElse");l(vz,"lowerWhile");l(mr,"lowerExpr");l(Nz,"lowerAccumulator");l(Ar,"substituteVars");l(Sf,"substituteVarsStmt")});function pM(r){let e=[],t={errors:e,boundVars:new Set,path:[],bufferNames:new Set};if(!r||r.type!=="LIRFunc")return e.push(new lo("root must be LIRFunc")),e;for(let[,n]of r.bufferMap)t.bufferNames.add(n.name);for(let n of r.shapeParams)t.boundVars.add(n.name);if(r.metadata)for(let[n]of r.metadata.locals)t.boundVars.add(n);return as(r.body,t),e}function as(r,e){if(!(!r||typeof r!="object")){switch(e.path.push(r.type),r.type){case"ForNode":Az(r,e);break;case"SeqNode":for(let t of r.stmts)as(t,e);break;case"LIRFlatStoreNode":Oz(r,e);break;case"LIRAccumulatorNode":Bz(r,e);break;case"LIRBindingsNode":Rz(r,e);break;case"LetStmtNode":{vn(r.value,e);let t=e.boundVars.has(r.variable.name);e.boundVars.add(r.variable.name),as(r.body,e),t||e.boundVars.delete(r.variable.name);break}case"AllocateNode":{let t=r.buffer?e.bufferNames.has(r.buffer.name):!0;r.buffer&&e.bufferNames.add(r.buffer.name),as(r.body,e),r.buffer&&!t&&e.bufferNames.delete(r.buffer.name);break}case"IfThenElseNode":vn(r.condition,e),as(r.thenBody,e),r.elseBody&&as(r.elseBody,e);break;case"WhileNode":as(r.condBody,e),as(r.loopBody,e);break;case"EvaluateNode":vn(r.value,e);break;default:break}e.path.pop()}}function Az(r,e){r.extent?vn(r.extent,e):e.errors.push(new lo("ForNode missing extent",[...e.path]));let t=!0;r.loopVar?(t=e.boundVars.has(r.loopVar.name),e.boundVars.add(r.loopVar.name)):e.errors.push(new lo("ForNode missing loopVar",[...e.path])),as(r.body,e),r.loopVar&&!t&&e.boundVars.delete(r.loopVar.name)}function Oz(r,e){r.buffer||e.errors.push(new lo("LIRFlatStoreNode missing buffer",[...e.path])),vn(r.offsetExpr,e),vn(r.value,e)}function Bz(r,e){r.localName||e.errors.push(new lo("LIRAccumulatorNode missing localName",[...e.path])),r.dtype||e.errors.push(new lo("LIRAccumulatorNode missing dtype",[...e.path]));let t=r.localName?e.boundVars.has(r.localName):!0;r.localName&&e.boundVars.add(r.localName),vn(r.initLoad,e),vn(r.body,e),as(r.flushStore,e),r.initBody&&as(r.initBody,e),r.localName&&!t&&e.boundVars.delete(r.localName)}function Rz(r,e){let t=[];for(let n of r.bindings)vn(n.expr,e),e.boundVars.has(n.name)||t.push(n.name),e.boundVars.add(n.name);as(r.body,e);for(let n of t)e.boundVars.delete(n)}function vn(r,e){if(!(!r||typeof r!="object"||!r.type))switch(r.type){case"LIRFlatLoadNode":r.buffer||e.errors.push(new lo("LIRFlatLoadNode missing buffer",[...e.path])),vn(r.offsetExpr,e);break;case"MathOpNode":vn(r.a,e),r.b&&vn(r.b,e);break;case"CompareNode":vn(r.a,e),vn(r.b,e);break;case"CastNode":vn(r.expr,e);break;case"CallExternNode":if(r.args)for(let t of r.args)vn(t,e);break;case"IfThenElseNode":vn(r.condition,e),vn(r.thenBody,e),r.elseBody&&vn(r.elseBody,e);break;case"VariableNode":r.name!==void 0&&!e.boundVars.has(r.name)&&e.errors.push(new lo(`unbound variable '${r.name}'`,[...e.path]));break;case"IntImmNode":case"FloatImmNode":break;default:break}}var lo,fM=M(()=>{"use strict";lo=class{static{l(this,"LIRVerificationError")}constructor(e,t){this.message=e,this.nodePath=t||[]}toString(){let e=this.nodePath.length>0?` at ${this.nodePath.join(" > ")}`:"";return`LIR verification: ${this.message}${e}`}};l(pM,"verifyLIR");l(as,"verifyStmt");l(Az,"verifyForNode");l(Oz,"verifyFlatStore");l(Bz,"verifyAccumulator");l(Rz,"verifyBindings");l(vn,"verifyExpr")});function hM(r,e,t={}){return new Do({target:e,...t}).compileFunction(r)}var a_,mx,Do,u_=M(()=>{"use strict";Yo();xs();VO();am();GO();AB();WR();Ui();Oc();tm();HR();ZR();lx();XR();tM();oM();jT();Kt();P0();cT();cM();fM();Kt();a_=class{static{l(this,"CompilerConfig")}constructor(e={}){this.target=e.target,this.verify=e.verify!==!1,this.verifyMode=e.verify==="full"?"full":"normal",this.errorMode=e.errorMode||"strict";let t=this.target&&typeof this.target.isWebGPU=="function"&&this.target.isWebGPU(),s=this.target&&typeof this.target.isGPU=="function"&&this.target.isGPU()&&!t;this.fusion={enabled:!0,strategy:"priority",...e.fusion},this.scheduling={enabled:t,autotune:!1,gpuTiling:s,...e.scheduling},this.matmulBackend=e.matmulBackend||"native",this.quantization={enabled:!1,...e.quantization},this.optimization={layout:!1,rematerialization:!1,rematConfig:{},fastMath:!1,maxSimplifyIterations:8,loopPartition:!1,detectAccumulators:!1,tensorize:!1,...e.optimization},this.memory={alignment:64,inplaceReuse:!0,allocStrategy:"best-fit",poolAllocation:!1,...e.memory},this.partition={enabled:!1,targets:[],defaultTarget:null,opTargetOverrides:new Map,memoryLimits:new Map,minPartitionSize:1,costWeights:{},...e.partition},this.passContext=e.passContext||null,this.loweringRules=e.loweringRules||null,this.codegenEntries=e.codegenEntries||null;let o=e.trace||{};this.trace={level:o.level??ue.SILENT,sink:o.sink??null,irSnapshot:{afterGraphPasses:!1,afterLowering:!1,afterScheduling:!1,...o.irSnapshot||{}}}}get usePartition(){return this.partition.enabled&&this.partition.targets.length>=2}},mx=class{static{l(this,"CompilationResult")}constructor(e,t,n){this.module=e,this.trace=t,this.errors=n||[]}get succeeded(){return this.errors.length===0}get failedFunctions(){let e=new Set;for(let t of this.errors)t.funcName&&e.add(t.funcName);return e}run(e,...t){return this.module.run(e,...t)}async runAsync(e,...t){return this.module.runAsync(e,...t)}isAsync(e){return this.module.isAsync(e)}getSource(e){return this.module.getKernelSource(e)}listKernels(){return this.module.listKernels()}},Do=class r{static{l(this,"Compiler")}constructor(e){if(this.config=e instanceof a_?e:new a_(e),!this.config.target)throw new Error("Compiler requires a target");this.context=new Gg({loweringRules:this.config.loweringRules,codegenEntries:this.config.codegenEntries})}compile(e){let t=new im(this.config.trace),n=this.config.errorMode==="resilient",s=[],o=new Set,i=performance.now();t.phaseStart("compile");let a={compiler:this,trace:t,errors:s,failed:o,resilient:n,original:e,working:n?Ok(e):e,cudaMatmulChain:!1,split:null,primFuncs:null,lirFuncs:null,runtimeModule:null};for(let u of this._compilePhases())u.when&&!u.when(a)||u.run(a);if(t.phaseEnd("compile",performance.now()-i),!n&&s.length>0)throw new Error(s[0].toString());return new mx(a.runtimeModule,t,s)}_compilePhases(){return[{name:"verify:pre",when:l(e=>e.compiler.config.verify,"when"),run:l(e=>e.compiler._verifyGraph(e.working,"before graph passes",e.trace,e.errors,e.failed,e.resilient),"run")},{name:"calibrate",when:l(e=>{let t=e.compiler.config.quantization;return t.enabled&&t.calibrationData&&!t.calibration},"when"),run:l(e=>e.compiler._runCalibration(e.working,e.trace),"run")},{name:"graphPasses",run:l(e=>{e.cudaMatmulChain=e.compiler._runGraphPasses(e.working,e.original,e.trace,e.errors,e.failed,e.resilient)},"run")},{name:"partition",when:l(e=>e.compiler.config.usePartition,"when"),run:l(e=>e.compiler._runPartitioning(e.working,e.trace),"run")},{name:"split",run:l(e=>{let t=e.compiler.config,n=typeof t.target.isWebGPU=="function"&&t.target.isWebGPU(),s=typeof t.target.isGPU=="function"&&t.target.isGPU()&&!n,o=0,i=0;for(let p of e.working)for(let c of p.ops())c.opName==="conv"||c.opName==="quantized_conv"?o++:c.opName==="scaled_dot_product_attention"&&i++;let a=s&&o>=2,u=s&&i>0;e.split=sM(e.working,{config:t,target:t.target,cudaMatmulChain:e.cudaMatmulChain,cudaConvChain:a,cudaAttention:u,isWebGPU:n})},"run")},{name:"verify:post",when:l(e=>e.compiler.config.verify,"when"),run:l(e=>e.compiler._verifyGraph(e.working,"after graph passes",e.trace,e.errors,e.failed,e.resilient),"run")},{name:"lowering",run:l(e=>{if(e.primFuncs=e.compiler._lowerAll(e.working,e.trace,e.errors,e.failed,e.resilient),e.compiler.config.matmulBackend==="cublas")for(let t of e.primFuncs)t.cublasInfo=e.split&&e.split.cublasInfos?e.split.cublasInfos.get(t.name)||null:Cv(t)},"run")},{name:"tirPasses",run:l(e=>e.compiler._runTirPasses(e),"run")},{name:"verify:tensor",when:l(e=>e.compiler.config.verify,"when"),run:l(e=>e.compiler._verifyAll(e.primFuncs,e.errors,e.failed,e.resilient),"run")},{name:"lirLowering",run:l(e=>{e.lirFuncs=e.compiler._lowerToLIR(e.primFuncs,e.trace,e.errors,e.failed,e.resilient)},"run")},{name:"codegen",run:l(e=>{e.runtimeModule=e.compiler._codegen(e.lirFuncs,e.trace,e.errors,e.failed,e.resilient),e.split&&(e.runtimeModule.executionPlan=e.split.plan)},"run")}]}compileFunction(e){let t=new wn("single");return t.addFunction(e),this.compile(t)}calibrate(e,t="minmax"){let n=new El(t);for(let s of e)n.attach(s);return n}_runCalibration(e,t){let n=this.config.quantization,s=e.functionNames()[0],o=e.getFunction(s);if(!o)return;t.phaseStart("calibrate");let i=performance.now(),a=this.config.target,u=l((c,h)=>new r({target:h,verify:this.config.verify}).compile(c),"compileFn"),p=QR(o,a,n.calibrationData,{mode:n.calibrationMode||"minmax",quantizableOps:n.quantizableOps,compileFn:u});this.config.quantization={...n,calibration:p},t.phaseEnd("calibrate",performance.now()-i)}_runGraphPasses(e,t,n,s,o,i){let a=new oa,u=0;for(let m of e)for(let g of m.ops())g.opName==="dot"&&u++;let p=this.config.target,c=(p.getAttr&&p.getAttr("matmulChainThreshold"))??(p.kind==="cuda"?2:1/0),h=u>=c;for(let m of NB(this.config,this.config.target,{cudaMatmulChain:h,context:this.context}))a.addPass(m);a.setTrace(n),this.config.verifyMode==="full"&&a.setVerifyHook((m,g)=>(g?ix(m):bf(m)).map(_=>_.toString())),n.phaseStart("graphPasses");let f=performance.now(),d=a.run(e,{errorMode:i?"resilient":"strict",passContext:this.config.passContext});if(d.errors){for(let m of d.errors)s.push(m),n.errorEvent(m.phase,m.funcName,m.message,m.passName);if(d.failedFunctions){for(let m of d.failedFunctions)if(o.add(m),i&&t&&t!==e){let g=t.getFunction(m);g&&e.addFunction(zh(g))}}}if(n.phaseEnd("graphPasses",performance.now()-f),n.shouldSnapshot("afterGraphPasses")){let m=new lp;n.irDump("afterGraphPasses",m.printModule(e))}return h}_runPartitioning(e,t){let n=new oa;n.addPass(new o_(this.config.partition)),n.addPass(new i_({targets:this.config.partition.targets})),n.setTrace(t),t.phaseStart("partition");let s=performance.now();n.run(e),t.phaseEnd("partition",performance.now()-s)}_eachFunc(e,t,n,s,o,i,a){for(let u of e)if(!o.has(u.name))try{a(u)}catch(p){if(s.push(new Dn(t,u.name,p.message)),o.add(u.name),n.errorEvent(t,u.name,p.message),!i)break}}_lowerAll(e,t,n,s,o){t.phaseStart("lowering");let i=performance.now(),a=[];return this._eachFunc(e,"lowering",t,n,s,o,u=>{let p=performance.now(),c=Qh(u,this.config.target,this.context);this.config.target.isGPU&&this.config.target.isGPU()&&!(this.config.target.isWebGPU&&this.config.target.isWebGPU())&&(c.convInfo=mR(u)),t.functionEvent("lowering",u.name,{durationMs:performance.now()-p}),a.push(c),t.shouldSnapshot("afterLowering")&&t.irDump("afterLowering:"+u.name,jg(c))}),t.phaseEnd("lowering",performance.now()-i),a}_runTirPasses(e){let t=new qg;for(let n of KR(this.config))t.addPass(n);if(t.setTrace(e.trace),this.config.verifyMode==="full"){let n=new _f;t.setVerifyHook(s=>n.verify(s))}t.run(e.primFuncs,{trace:e.trace,errors:e.errors,failed:e.failed,resilient:e.resilient})}_verifyGraph(e,t,n,s,o,i){if(i){for(let u of e){if(o.has(u.name))continue;let p=bf?bf(u):[];if(p.length>0){let c=p.map(h=>h.toString()).join("; ");s.push(new Dn("verification",u.name,c)),o.add(u.name),n.errorEvent("verification",u.name,c)}}return}let a=ix(e);if(a.length>0)throw new Error("Graph verification failed ("+t+"): "+a.map(u=>u.toString()).join("; "))}_verifyAll(e,t,n,s){let o=new _f;for(let i of e){if(n.has(i.name))continue;let a=o.verify(i);if(a.length>0){let u=a.join("; ");if(s)t.push(new Dn("verification",i.name,u)),n.add(i.name);else throw new Error("TensorIR verification failed for "+i.name+": "+u)}}}_lowerToLIR(e,t,n,s,o){t.phaseStart("lirLowering");let i=performance.now(),a=[];return this._eachFunc(e,"lirLowering",t,n,s,o,u=>{let p=performance.now(),c=lM(u,this.config.target);if(u.cublasInfo&&(c.cublasInfo=u.cublasInfo),u.gpuRegisterBlocked&&(c.gpuRegisterBlocked=!0),this.config.verifyMode==="full"){let h=pM(c);if(h.length>0)throw new Error("LIR verification failed: "+h.map(f=>f.toString()).join("; "))}t.functionEvent("lirLowering",u.name,{durationMs:performance.now()-p}),a.push(c)}),t.phaseEnd("lirLowering",performance.now()-i),a}_codegen(e,t,n,s,o){t.phaseStart("codegen");let i=performance.now(),a=new Fu("compiled"),u=this.config.usePartition,p={matmulBackend:this.config.matmulBackend,context:this.context},c=new Map,h=l(d=>(c.has(d.name)||c.set(d.name,new Ws(d,p)),c.get(d.name)),"getBackend"),f=u?null:new Ws(this.config.target,p);return this._eachFunc(e,"codegen",t,n,s,o,d=>{let m=performance.now(),g;if(u){let _=d._partitionTarget,b=_?this.config.partition.targets.find(w=>w.name===_):this.config.target;g=h(b||this.config.target)}else g=f;let y=g.compile(d);a.addCompiledKernel(y),d.shapeParamMap&&d.shapeParamMap.size>0&&a.setShapeParamMap(d.name,d.shapeParamMap,d.bufferMap),t.codegenStats(d.name,{durationMs:performance.now()-m,sourceSize:y.source.length,targetName:y.target.name})}),t.phaseEnd("codegen",performance.now()-i),a}};l(hM,"compileGraph")});function Iz(r){return typeof r=="object"&&r!==null}function Ez(r){return Iz(r)&&"_impl"in r}function Lz(r){return(e,...t)=>{let n=ss();if(!n)throw new Error(`TRACING dispatch key active but no tracer is set for op '${r}'`);if(Mz.has(r)){let c=Nt.findOp(r);return Nt.redispatch(c,e,...t)}let s=[],o={},i=Sm(r),a=0,u=l(c=>c instanceof ft||Ez(c),"isTensor"),p=l(c=>{c instanceof ft||c.isSymbolic?s.push(c):s.push(n.captureConstant(c))},"pushTensor");for(let c of t)if(Array.isArray(c)&&c.length>0&&u(c[0]))for(let h of c)p(h);else u(c)?p(c):i&&(c!=null&&a<i.length&&(o[i[a]]=c),a++);return n.recordOp(r,s,o)}}function mM(){let r=Nt.listOps();for(let e of r){let t=Nt.findOp(e);if(!t)continue;let n=t.name,s=wr.fromUnboxed(Lz(n));t.entry.registerKernel(48,s)}}var Mz,gM=M(()=>{"use strict";Nn();tc();$i();ya();ga();km();Mz=new Set(["scatter","repeat","tile","split","chunk","roll","flip","cumsum","sort","argsort","topk"]);l(Iz,"_isObject");l(Ez,"_hasImpl");l(Lz,"_tracingKernel");l(mM,"registerTracingDispatch")});function Dz(r){return r&&r.shape&&r.shape.length>=2&&Cz.has(r.dtype)}function yM(r,e,t=Dz){let n=r.graph.functions().next().value;if(!n)return r;let s=n.entryBlock,o=r.numUserInputs,i=r.capturedParams,a=[];for(let h=0;h<i.length;h++){let f=o+h,d=s.getArgument(f);d&&t(i[h],h,d)&&a.push({j:h,argIndex:f,arg:d,param:i[h]})}if(a.length===0)return r;for(let h of a){let f=e(h.param),d=h.arg.type,m=Re,g=new m("constant",[],[d],{value:f,tensor_type:d}),y=s.firstOp;y?s.insertBefore(g,y):s.pushOp(g),h.arg.replaceAllUsesWith(g.getResult(0))}let u=new Set(a.map(h=>h.argIndex));s.removeArguments(u),n.inputTypes=Object.freeze(n.inputTypes.filter((h,f)=>!u.has(f)));let p=new Set(a.map(h=>h.j)),c=i.filter((h,f)=>!p.has(f));return{...r,capturedParams:c}}var Cz,_M=M(()=>{"use strict";sr();Cz=new Set(["f16","f32","f64"]);l(Dz,"defaultWeightPredicate");l(yM,"foldWeightParams")});function kf(r,e,t){let n=t.get(e.id);if(n)return n;let s=r.scalarConstant(0,e.type.dtype).getResult(0);return r.broadcast(s,e.type.shape,[]).getResult(0)}var co,vf=M(()=>{"use strict";l(kf,"gradOrZero");co=class{static{l(this,"GradAccumulator")}constructor(e){this._builder=e,this._pending=new Map,this._reduced=new Map}accumulate(e,t){if(!t)return;let n=this._pending.get(e);n||(n=[],this._pending.set(e,n)),n.push(t),this._reduced.delete(e)}_treeReduce(e){let t=e;for(;t.length>1;){let n=[];for(let s=0;s<t.length;s+=2)s+1<t.length?n.push(this._builder.add(t[s],t[s+1]).getResult(0)):n.push(t[s]);t=n}return t[0]}get(e){if(this._reduced.has(e))return this._reduced.get(e);let t=this._pending.get(e);if(!t||t.length===0)return null;let n=this._treeReduce(t);return this._reduced.set(e,n),n}has(e){let t=this._pending.get(e);return!!t&&t.length>0}}});function ce(r,e){gx.set(r,e)}function Ia(r){return gx.get(r)||null}function f_(r){let e=gx.get(r);if(e)return e;if(yx.has(r))return null;throw new Error(`autodiff: op '${r}' is on the gradient path but has no VJP rule and is not a registered gradient barrier. Register one with registerVJPRule('${r}', ...) or registerGradientBarrier('${r}').`)}function _x(r,e){bM.set(r,e)}function h_(r){return bM.get(r)||null}function bx(r){yx.add(r)}function d_(r){return yx.has(r)}var gx,yx,bM,ls=M(()=>{"use strict";gx=new Map,yx=new Set;l(ce,"registerVJPRule");l(Ia,"getVJPRule");l(f_,"requireVJPRuleOrBarrier");bM=new Map;l(_x,"registerRegionVJP");l(h_,"getRegionVJP");l(bx,"registerGradientBarrier");l(d_,"isGradientBarrier")});var Po,wx=M(()=>{"use strict";Po=new Set(["scan","if"])});function Nf(r){let e=new Set(r.arguments.map(i=>i.id)),t=l(i=>{for(let a of i.ops()){for(let u of a.results)e.add(u.id);for(let u of a.regions||[])for(let p of u.blocks){for(let c of p.arguments)e.add(c.id);t(p)}}},"addLocals");t(r);let n=new Set,s=[],o=l(i=>{for(let a of i.ops()){for(let u of a.operands)e.has(u.id)||n.has(u.id)||u.definingOp&&u.definingOp.opName==="constant"||(n.add(u.id),s.push(u));for(let u of a.regions||[])for(let p of u.blocks)o(p)}},"scan");return o(r),s}function m_(r,e){let t=r.scalarConstant(0,e.type.dtype).getResult(0);return r.broadcast(t,e.type.shape,[]).getResult(0)}function wM(r,e,t){let n=e.type.shape,s=n.map((a,u)=>u===0?t:0),o=n.map((a,u)=>u===0?t+1:a),i=r.slice(e,s,o).getResult(0);return r.reshape(i,n.slice(1)).getResult(0)}function $z(r,e,t){let n=t.slice(1),s=e.map(o=>r.reshape(o,[1,...n]).getResult(0));return s.length===1?r.reshape(s[0],t).getResult(0):r.concat(s,0).getResult(0)}function g_(r,e,t,n,s,o,i=new Map,a=[]){let u=new Map,p=new Map;for(let _=0;_<e.arguments.length;_++)u.set(e.arguments[_].id,t[_]),p.set(e.arguments[_],t[_]);for(let[_,b]of n)u.set(_,b);for(let _ of a)n.has(_.id)&&p.set(_,n.get(_.id));let c=[],h=null;for(let _ of e.ops())_.opName==="yield"?h=_:c.push(_);let f=l(_=>{if(u.has(_.id))return u.get(_.id);let b=_.definingOp;if(b&&b.opName==="constant"){let w=i.get(_.id);return w===void 0&&(w=r._buildOp("constant",[],[_.type],new Map(b.attributes),null).getResult(0),i.set(_.id,w)),u.set(_.id,w),p.set(_,w),w}return _},"matOperand");for(let _ of c){if(Po.has(_.opName)){for(let S of _.operands)f(S),p.set(S,u.get(S.id)??S);let x=_.clone(p);r.block.pushOp(x);for(let S=0;S<_.numResults;S++)u.set(_.getResult(S).id,x.getResult(S)),p.set(_.getResult(S),x.getResult(S));continue}let b=_.operands.map(f),w=r._buildOp(_.opName,b,_.results.map(x=>x.type),new Map(_.attributes),null);for(let x=0;x<_.numResults;x++)u.set(_.getResult(x).id,w.getResult(x)),p.set(_.getResult(x),w.getResult(x))}let d=h.operands.map(_=>u.get(_.id));if(o)return{forwardYields:d};let m=new co(r);for(let _=0;_<h.operands.length;_++)s[_]&&m.accumulate(h.operands[_].id,s[_]);for(let _=c.length-1;_>=0;_--){let b=c[_];if(b.opName==="constant")continue;let w=b.results.map(k=>m.get(k.id));if(w.every(k=>k===null))continue;if(Po.has(b.opName)){let k=h_(b.opName);k&&k(b,{accumulator:m,builder:r,materialize:f,needsGrad:Pz,scanCheckpoint:null});continue}let x=f_(b.opName);if(!x)continue;let S={builder:r,op:b,operands:b.operands.map(f),results:b.results.map(k=>u.get(k.id)),gradOutputs:w,attrs:b.attributes,full:l((k,A)=>r.broadcast(r.scalarConstant(k,A.dtype).getResult(0),A.shape,[]).getResult(0),"full")},T=x(S);if(T)for(let k=0;k<b.numOperands;k++)T[k]&&m.accumulate(b.getOperand(k).id,Tx(r,T[k],b.getOperand(k).type.shape))}let g=e.arguments.map(_=>m.get(_.id)),y=new Map;for(let _ of n.keys())y.set(_,m.get(_));return{forwardYields:d,gradArgs:g,gradFree:y}}function Fz(r,e,t,n,s){let o=r.regions[0].blocks[0],i=r.regions[1].blocks[0],a=n(r.getOperand(0)),u=[];for(let y=0;y<r.numResults;y++)u.push(e.get(r.getResult(y).id));let p=Nf(o),c=Nf(i),h=new Map(p.map(y=>[y.id,n(y)])),f=new Map(c.map(y=>[y.id,n(y)])),{gradFree:d}=g_(t,o,[],h,u,!1,new Map,p),{gradFree:m}=g_(t,i,[],f,u,!1,new Map,c),g=new Map;for(let y of p)g.set(y.id,y);for(let y of c)g.set(y.id,y);for(let[y,_]of g){if(!s.has(y))continue;let b=d.get(y),w=m.get(y);if(!b&&!w)continue;let x=m_(t,_),S=t.broadcast(a,_.type.shape,[]).getResult(0);e.accumulate(y,t.select(S,b??x,w??x).getResult(0))}}function zz(r,e){if(!r||e<=1)return null;if(r==="sqrt"||r===!0)return Math.max(1,Math.ceil(Math.sqrt(e)));if(typeof r=="number"&&r>=1){let t=Math.floor(r);return t>=e?null:t}return null}function Vz(r,e,t,n,s,o=null){let i=r.regions[0].blocks[0],a=r.getAttr("num_carry"),u=r.getAttr("num_xs"),p=r.numResults-a,c=[];for(let v=0;v<u;v++)c.push(r.getOperand(v));let h=[];for(let v=0;v<a;v++)h.push(r.getOperand(u+v));let f=c[0].type.shape[0],d=Nf(i),m=c.map(n),g=h.map(n),y=new Map(d.map(v=>[v.id,n(v)])),_=new Map,b=l(v=>m.map(O=>wM(t,O,v)),"sliceX"),w=l((v,O)=>g_(t,i,[...v,...O],y,null,!0,_,d).forwardYields.slice(0,a),"stepForward"),x=[];for(let v=0;v<p;v++)x.push(e.get(r.getResult(a+v).id));let S=[];for(let v=0;v<a;v++){let O=e.get(r.getResult(v).id);S.push(O??m_(t,g[v]))}let T=new Map,k=m.map(()=>new Array(f)),A=l((v,O,R)=>{let B=[...O,...R],E=x.map(j=>j===null?null:wM(t,j,v)),z=[...S,...E],{gradArgs:D,gradFree:J}=g_(t,i,B,y,z,!1,_,d);for(let j=0;j<u;j++)k[j][v]=D[j]??m_(t,O[j]);S=[];for(let j=0;j<a;j++)S.push(D[u+j]??m_(t,R[j]));for(let[j,U]of J){if(!U)continue;let P=T.get(j);T.set(j,P?t.add(P,U).getResult(0):U)}},"backwardStep"),N=zz(o,f);if(N){let v=Math.ceil(f/N),O=new Array(v),R=g;O[0]=R;for(let B=0;B<f;B++){R=w(b(B),R);let E=(B+1)/N;Number.isInteger(E)&&E<v&&(O[E]=R)}for(let B=v-1;B>=0;B--){let E=B*N,z=Math.min(E+N,f),D=[],J=[O[B]],j=O[B];for(let U=E;U<z;U++){let P=b(U);D.push(P),j=w(P,j),J.push(j)}for(let U=z-1;U>=E;U--)A(U,D[U-E],J[U-E])}}else{let v=g,O=[v],R=[];for(let B=0;B<f;B++){let E=b(B);R.push(E),v=w(E,v),O.push(v)}for(let B=f-1;B>=0;B--)A(B,R[B],O[B])}for(let v=0;v<u;v++)s.has(c[v].id)&&e.accumulate(c[v].id,$z(t,k[v],m[v].type.shape));for(let v=0;v<a;v++)s.has(h[v].id)&&e.accumulate(h[v].id,S[v]);for(let[v,O]of T)O&&s.has(v)&&e.accumulate(v,O)}var Pz,TM=M(()=>{"use strict";vf();ls();Af();wx();Pz={has:l(()=>!0,"has")};_x("scan",(r,e)=>Vz(r,e.accumulator,e.builder,e.materialize,e.needsGrad,e.scanCheckpoint));_x("if",(r,e)=>Fz(r,e.accumulator,e.builder,e.materialize,e.needsGrad));l(Nf,"regionFreeVars");l(m_,"zeroLike");l(wM,"sliceStep");l($z,"stackSteps");l(g_,"diffBodyStep");l(Fz,"buildCondBackward");l(zz,"resolveSegmentLength");l(Vz,"buildScanBackward")});function Of(r,{accumulator:e,builder:t,needsGrad:n,resolveValue:s,handleRegionOp:o=null}){for(let i=r.length-1;i>=0;i--){let a=r[i];if(a.opName==="return"||a.opName==="constant"||!a.results.some(g=>n.has(g.id)))continue;let p=[];for(let g=0;g<a.numResults;g++)p.push(e.get(a.getResult(g).id));if(p.every(g=>g===null)||o&&o(a))continue;let c=f_(a.opName);if(!c)continue;let h=new Array(a.numOperands);for(let g=0;g<a.numOperands;g++)h[g]=s(a.getOperand(g));let f=new Array(a.numResults);for(let g=0;g<a.numResults;g++)f[g]=s(a.getResult(g));let d=l((g,y)=>t.broadcast(t.scalarConstant(g,y.dtype).getResult(0),y.shape,[]).getResult(0),"full"),m=c({builder:t,op:a,operands:h,results:f,gradOutputs:p,attrs:a.attributes,full:d});if(m)for(let g=0;g<a.numOperands;g++){if(g>=m.length||!m[g])continue;let y=a.getOperand(g);n.has(y.id)&&e.accumulate(y.id,Tx(t,m[g],y.type.shape))}}}function xM(r){let e=[];for(let t of r.regions)t.blocks[0]&&e.push(...Nf(t.blocks[0]));return e}function Tx(r,e,t){let n=e.type.shape;if(n.length===t.length&&n.every((u,p)=>u===t[p]))return e;let s=n.length-t.length,o=[];for(let u=0;u<s;u++)o.push(u);for(let u=0;u<t.length;u++)t[u]===1&&n[s+u]!==1&&o.push(s+u);let i=e;if(o.length>0){let u=r.scalarConstant(0,e.type.dtype).getResult(0);i=r.reduce(e,u,o,"sum").getResult(0)}let a=i.type.shape;return a.length===t.length&&a.every((u,p)=>u===t[p])||(i=r.reshape(i,t).getResult(0)),i}var Gz,Bf,Af=M(()=>{"use strict";xs();es();Es();vf();ls();TM();wx();Gz=new Set(["neg","abs","sign","floor","ceil"]);l(Of,"backpropOps");l(xM,"regionControlFlowFreeVars");l(Tx,"reduceGradToOperandShape");Bf=class{static{l(this,"BackwardGraphBuilder")}constructor(e={}){this._rematPolicy=e.rematPolicy||null,this._checkpointPolicy=e.checkpointPolicy||null,this._scanCheckpoint=e.scanCheckpoint||null}build(e){if(this._checkpointPolicy)return this._buildCheckpointed(e);let n=at.compute(e).topologicalOrder,s=e.getReturnOp();if(!s)throw new Error("Forward function has no return op");let o=s.operands,i=e.args,a=this._computeGradReachability(e,n),{savedValues:u,savedValueIndices:p}=this._identifySavedValues(n,a,i),c=o.map(T=>T.type),h=u.map(T=>T.type),f=[...c,...h],d=[];for(let T=0;T<i.length;T++)a.has(i[T].id)&&d.push(i[T].type);let m=new un(`backward_${e.name}`,f,d),g=new An(m),y=m.args,_=y.slice(0,c.length),b=y.slice(c.length),w=new Map;for(let T=0;T<u.length;T++)w.set(u[T].id,b[T]);for(let T=0;T<i.length;T++)p.has(i[T].id)&&w.set(i[T].id,b[p.get(i[T].id)]);let x=new co(g);for(let T=0;T<o.length;T++){let k=o[T];x.accumulate(k.id,_[T])}Of(n,{accumulator:x,builder:g,needsGrad:a,resolveValue:l(T=>this._materialize(T,w,g),"resolveValue"),handleRegionOp:l(T=>{let k=h_(T.opName);return k?(k(T,{accumulator:x,builder:g,materialize:l(A=>this._materialize(A,w,g),"materialize"),needsGrad:a,scanCheckpoint:this._scanCheckpoint}),!0):!1},"handleRegionOp")});let S=[];for(let T=0;T<i.length;T++)a.has(i[T].id)&&S.push(kf(g,i[T],x));return g.returnOp(S),{backwardFunc:m,savedValues:u,gradInputIndices:this._getGradInputIndices(i,a)}}_materialize(e,t,n){if(t.has(e.id))return t.get(e.id);if(!e.definingOp)return e;let s=new Set([e.id]),o=[{val:e,i:0}];for(;o.length>0;){let i=o[o.length-1],a=i.val,u=a.definingOp;if(t.has(a.id)||!u){s.delete(a.id),o.pop();continue}if(i.i<u.numOperands){let f=u.getOperand(i.i);i.i++,f.definingOp&&!t.has(f.id)&&!s.has(f.id)&&(s.add(f.id),o.push({val:f,i:0}));continue}let p=new Array(u.numOperands);for(let f=0;f<u.numOperands;f++){let d=u.getOperand(f);p[f]=t.has(d.id)?t.get(d.id):d}let c=u.results.map(f=>f.type),h=n._buildOp(u.opName,p,c,new Map(u.attributes),null);for(let f=0;f<u.numResults;f++)t.set(u.getResult(f).id,h.getResult(f));s.delete(a.id),o.pop()}return t.has(e.id)?t.get(e.id):e}_computeGradReachability(e,t){let n=new Set,s=e.getReturnOp();for(let o of s.operands)n.add(o.id);for(let o=t.length-1;o>=0;o--){let i=t[o];if(!(i.opName==="return"||!i.results.some(u=>n.has(u.id)))){if(Po.has(i.opName)){for(let u=0;u<i.numOperands;u++)n.add(i.getOperand(u).id);for(let u of xM(i))n.add(u.id);continue}if(Ia(i.opName)&&!d_(i.opName))for(let u=0;u<i.numOperands;u++)n.add(i.getOperand(u).id)}}return n}_identifySavedValues(e,t,n){let s=[],o=new Map,i=new Set(n.map(h=>h.id));for(let h of e)if(!(h.opName==="return"||h.opName==="constant"||!Ia(h.opName)||!h.results.some(m=>t.has(m.id)))){for(let m=0;m<h.numOperands;m++){let g=h.getOperand(m);i.has(g.id)&&!o.has(g.id)&&(o.set(g.id,s.length),s.push(g))}for(let m=0;m<h.numResults;m++){let g=h.getResult(m);t.has(g.id)&&!o.has(g.id)&&this._shouldSaveResult(h)&&(o.set(g.id,s.length),s.push(g))}}let a=new Set(o.keys()),u=new Map(n.map(h=>[h.id,h])),p=new Set,c=l(h=>{if(a.has(h.id)||p.has(h.id))return;p.add(h.id);let f=h.definingOp;if(!f){i.has(h.id)&&!o.has(h.id)&&(o.set(h.id,s.length),s.push(u.get(h.id)));return}for(let d=0;d<f.numOperands;d++)c(f.getOperand(d))},"collect");for(let h of e)if(!(h.opName==="return"||h.opName==="constant")&&h.results.some(f=>t.has(f.id))){if(Po.has(h.opName)){for(let f of h.operands)c(f);for(let f of xM(h))c(f);continue}if(Ia(h.opName)){for(let f=0;f<h.numOperands;f++)c(h.getOperand(f));for(let f=0;f<h.numResults;f++)c(h.getResult(f))}}return{savedValues:s,savedValueIndices:o}}_shouldSaveResult(e){return this._rematPolicy?!this._rematPolicy.shouldRematerialize(e):!Gz.has(e.opName)}_getGradInputIndices(e,t){let n=[];for(let s=0;s<e.length;s++)t.has(e[s].id)&&n.push(s);return n}_buildCheckpointed(e){let n=at.compute(e).topologicalOrder,s=e.getReturnOp();if(!s)throw new Error("Forward function has no return op");let o=s.operands,i=e.args,a=this._computeGradReachability(e,n),u=this._checkpointPolicy.segment(n,e);for(let O of u)for(let R of O.ops)if(Po.has(R.opName))throw new Error(`Checkpointed backward does not support region control-flow op '${R.opName}'; build the backward without a checkpointPolicy, which differentiates scan/if via buildScanBackward/buildCondBackward.`);let p=new Set,c=[],h=new Map,f=new Map;for(let O of n)for(let R=0;R<O.numResults;R++){let B=O.getResult(R);f.set(B.id,B)}let d=new Set(i.map(O=>O.id));for(let O of i)a.has(O.id)&&!p.has(O.id)&&(p.add(O.id),h.set(O.id,c.length),c.push(O));for(let O of u){for(let R of O.boundaryInputs)if(!d.has(R)&&!p.has(R)){p.add(R);let B=f.get(R)||null;B&&(h.set(R,c.length),c.push(B))}for(let R of O.boundaryOutputs)if(!p.has(R)){p.add(R);let B=f.get(R)||null;B&&(h.set(R,c.length),c.push(B))}}for(let O of o)p.has(O.id)||(p.add(O.id),h.set(O.id,c.length),c.push(O));let m=o.map(O=>O.type),g=c.map(O=>O.type),y=[...m,...g],_=[];for(let O=0;O<i.length;O++)a.has(i[O].id)&&_.push(i[O].type);let b=new un(`backward_${e.name}`,y,_),w=new An(b),x=b.args,S=x.slice(0,m.length),T=x.slice(m.length),k=new Map;for(let O=0;O<c.length;O++)k.set(c[O].id,T[O]);let A=new co(w);for(let O=0;O<o.length;O++)A.accumulate(o[O].id,S[O]);let N=new Map;for(let O of n)if(O.opName==="constant"){let R=O.getResult(0).type,B=w._buildOp("constant",[],[R],new Map(O.attributes),null);N.set(O.getResult(0).id,B.getResult(0))}for(let O=u.length-1;O>=0;O--){let R=u[O],B=new Map;for(let E of R.ops){let z=new Array(E.numOperands);for(let j=0;j<E.numOperands;j++){let U=E.getOperand(j),P=B.get(U.id)||k.get(U.id)||N.get(U.id);z[j]=P||U}let D=E.results.map(j=>j.type),J=w._buildOp(E.opName,z,D,new Map(E.attributes),null);for(let j=0;j<E.numResults;j++)B.set(E.getResult(j).id,J.getResult(j))}Of(R.ops,{accumulator:A,builder:w,needsGrad:a,resolveValue:l(E=>B.get(E.id)||k.get(E.id)||N.get(E.id)||E,"resolveValue")})}let v=[];for(let O=0;O<i.length;O++)a.has(i[O].id)&&v.push(kf(w,i[O],A));return w.returnOp(v),{backwardFunc:b,savedValues:c,gradInputIndices:this._getGradInputIndices(i,a)}}}});var Ea,y_=M(()=>{"use strict";Ea=class{static{l(this,"RematPolicy")}constructor(e={}){this._maxRematDepth=e.maxRematDepth||1,this._sizeThreshold=e.sizeThreshold||1024*1024,this._alwaysRemat=new Set(e.alwaysRemat||["neg","abs","sign","floor","ceil","exp","log","sqrt","rsqrt","sin","cos","tanh"]),this._neverRemat=new Set(e.neverRemat||["matmul","dot","conv","reduce","custom_call","pool2d"])}shouldRematerialize(e){if(this._alwaysRemat.has(e.opName))return!0;if(this._neverRemat.has(e.opName))return!1;let t=e.numResults>0?e.getResult(0).type:null;return!t||!t.shape||t.numel()>this._sizeThreshold?!1:this._isElementwise(e)}_isElementwise(e){if(e.numOperands===0||e.numResults===0)return!1;let t=e.getResult(0).type.shape;for(let n=0;n<e.numOperands;n++){let s=e.getOperand(n).type;if(!s||!s.shape||s.shape.length!==t.length)return!1;for(let o=0;o<t.length;o++)if(s.shape[o]!==t[o])return!1}return!0}}});var Rf,xx=M(()=>{"use strict";xs();es();Es();vf();ls();y_();Af();Rf=class{static{l(this,"JointGraphBuilder")}constructor(e={}){this._rematPolicy=e.rematPolicy||new Ea(e.remat||{}),this._checkpointPolicy=e.checkpointPolicy||null}build(e){if(this._checkpointPolicy)return this._buildCheckpointed(e);let t=this._buildScaffold(e);return Of(t.topoOrder,{accumulator:t.accumulator,builder:t.builder,needsGrad:t.needsGrad,resolveValue:l(n=>t.valueMap.get(n.id)||n,"resolveValue")}),this._finish(t)}_buildCheckpointed(e){let t=this._buildScaffold(e),n=this._checkpointPolicy.segment(t.topoOrder,e);for(let s=n.length-1;s>=0;s--){let o=n[s],i=new Map;for(let a of o.ops){let u=new Array(a.numOperands);for(let h=0;h<a.numOperands;h++){let f=a.getOperand(h);u[h]=i.get(f.id)||t.valueMap.get(f.id)||f}let p=a.results.map(h=>h.type),c=t.builder._buildOp(a.opName,u,p,new Map(a.attributes),null);for(let h=0;h<a.numResults;h++)i.set(a.getResult(h).id,c.getResult(h))}Of(o.ops,{accumulator:t.accumulator,builder:t.builder,needsGrad:t.needsGrad,resolveValue:l(a=>i.get(a.id)||t.valueMap.get(a.id)||a,"resolveValue")})}return this._finish(t)}_buildScaffold(e){let n=at.compute(e).topologicalOrder;this._assertNoRegionControlFlow(n);let s=e.getReturnOp();if(!s)throw new Error("Forward function has no return op");let o=s.operands,i=e.args,a=o.map(w=>w.type),u=[...e.inputTypes,...a],p=[...e.outputTypes,...e.inputTypes],c=new un(`joint_${e.name}`,u,p),h=new An(c),f=c.args,d=f.slice(0,e.inputTypes.length),m=f.slice(e.inputTypes.length),g=new Map;for(let w=0;w<i.length;w++)g.set(i[w].id,d[w]);for(let w of n){if(w.opName==="return")continue;let x=new Array(w.numOperands);for(let k=0;k<w.numOperands;k++){let A=w.getOperand(k);x[k]=g.get(A.id)||A}let S=w.results.map(k=>k.type),T=h._buildOp(w.opName,x,S,new Map(w.attributes),null);for(let k=0;k<w.numResults;k++)g.set(w.getResult(k).id,T.getResult(k))}let y=o.map(w=>g.get(w.id)),_=this._computeGradReachability(e,n),b=new co(h);for(let w=0;w<o.length;w++)b.accumulate(o[w].id,m[w]);return{topoOrder:n,forwardInputs:i,forwardOutputs:o,fwdOutputValues:y,valueMap:g,builder:h,needsGrad:_,accumulator:b,jointFunc:c}}_finish(e){let t=[];for(let n=0;n<e.forwardInputs.length;n++)t.push(kf(e.builder,e.forwardInputs[n],e.accumulator));return e.builder.returnOp([...e.fwdOutputValues,...t]),{jointFunc:e.jointFunc,numForwardOutputs:e.forwardOutputs.length,numGradInputs:t.length}}_computeGradReachability(e,t){let n=new Set,s=e.getReturnOp();for(let o of s.operands)n.add(o.id);for(let o=t.length-1;o>=0;o--){let i=t[o];if(!(i.opName==="return"||!i.results.some(u=>n.has(u.id)))&&Ia(i.opName)&&!d_(i.opName))for(let u=0;u<i.numOperands;u++)n.add(i.getOperand(u).id)}return n}_assertNoRegionControlFlow(e){for(let t of e)if(Po.has(t.opName))throw new Error(`JointGraphBuilder does not support region control-flow op '${t.opName}'; use BackwardGraphBuilder (separate mode) without a checkpointPolicy, which differentiates scan/if.`)}}});var SM=M(()=>{"use strict"});function kM(r,e){let t=r.gradOutputs[0],[n,s]=r.operands,o=r.full(0,n.type),i=r.builder.compare(n,s,e).getResult(0),a=r.builder.select(i,t,o).getResult(0),u=r.builder.select(i,o,t).getResult(0);return[a,u]}function vM(r){let e=r.gradOutputs[0],[t]=r.operands,n=r.full(0,e.type),s=r.builder.where(t,e,n).getResult(0),o=r.builder.where(t,n,e).getResult(0);return[null,s,o]}var NM=M(()=>{"use strict";ls();ce("add",r=>{let e=r.gradOutputs[0];return[e,e]});ce("sub",r=>{let e=r.gradOutputs[0],t=r.builder.neg(e).getResult(0);return[e,t]});l(kM,"_minMaxVJP");ce("maximum",r=>kM(r,"ge"));ce("minimum",r=>kM(r,"le"));ce("clamp",r=>{let e=r.gradOutputs[0],[t,n,s]=r.operands,o=r.full(0,n.type),i=r.builder.compare(n,t,"ge").getResult(0),a=r.builder.where(i,e,o).getResult(0),u=r.builder.compare(n,s,"le").getResult(0);return[null,r.builder.where(u,a,o).getResult(0),null]});l(vM,"_whereVJP");ce("where",vM);ce("select",vM);ce("mul",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.builder.mul(e,n).getResult(0),o=r.builder.mul(e,t).getResult(0);return[s,o]});ce("div",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.builder.div(e,n).getResult(0),o=r.builder.mul(n,n).getResult(0),i=r.builder.neg(t).getResult(0),a=r.builder.mul(e,i).getResult(0),u=r.builder.div(a,o).getResult(0);return[s,u]});ce("neg",r=>{let e=r.gradOutputs[0];return[r.builder.neg(e).getResult(0)]});ce("pow",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.full(1,t.type),o=r.builder.sub(n,s).getResult(0),i=r.builder.pow(t,o).getResult(0),a=r.builder.mul(n,i).getResult(0),u=r.builder.mul(e,a).getResult(0),p=r.builder.pow(t,n).getResult(0),c=r.builder.log(t).getResult(0),h=r.builder.mul(p,c).getResult(0),f=r.builder.mul(e,h).getResult(0);return[u,f]})});function AM(r,e){let t=r.full(2/Math.sqrt(Math.PI),e.type),n=r.builder.mul(e,e).getResult(0),s=r.builder.neg(n).getResult(0),o=r.builder.exp(s).getResult(0);return r.builder.mul(t,o).getResult(0)}function OM(r,e){let t=r.builder,n=r.full(1,e.type),s=t.add(e,r.full(Yi,e.type)).getResult(0),o=t.div(n,s).getResult(0),i=t.log(s).getResult(0),a=t.sub(i,t.mul(r.full(.5,e.type),o).getResult(0)).getResult(0),u=t.mul(o,o).getResult(0),p=u;for(let c of vc)a=t.add(a,t.mul(r.full(c,e.type),p).getResult(0)).getResult(0),p=t.mul(p,u).getResult(0);for(let c=0;c<Yi;c++){let h=t.add(e,r.full(c,e.type)).getResult(0);a=t.sub(a,t.div(n,h).getResult(0)).getResult(0)}return a}var BM=M(()=>{"use strict";ls();Ji();ce("exp",r=>{let e=r.gradOutputs[0],t=r.results[0];return[r.builder.mul(e,t).getResult(0)]});ce("log",r=>{let e=r.gradOutputs[0],[t]=r.operands;return[r.builder.div(e,t).getResult(0)]});ce("sqrt",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.full(2,t.type),s=r.builder.mul(n,t).getResult(0);return[r.builder.div(e,s).getResult(0)]});ce("tanh",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.builder.mul(t,t).getResult(0),s=r.builder.sub(r.full(1,t.type),n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("sigmoid",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.builder.sub(r.full(1,t.type),t).getResult(0),s=r.builder.mul(t,n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("relu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(0,t.type),s=r.builder.compare(t,n,"gt").getResult(0);return[r.builder.select(s,e,n).getResult(0)]});ce("gelu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(1.702,t.type),s=r.full(1,t.type),o=r.builder.mul(n,t).getResult(0),i=r.builder.sigmoid(o).getResult(0),a=r.builder.sub(s,i).getResult(0),u=r.builder.mul(o,a).getResult(0),p=r.builder.add(s,u).getResult(0),c=r.builder.mul(i,p).getResult(0);return[r.builder.mul(e,c).getResult(0)]});ce("silu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.sigmoid(t).getResult(0),s=r.builder.sub(r.full(1,t.type),n).getResult(0),o=r.builder.mul(t,s).getResult(0),i=r.builder.add(n,r.builder.mul(n,o).getResult(0)).getResult(0);return[r.builder.mul(e,i).getResult(0)]});ce("sin",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.cos(t).getResult(0);return[r.builder.mul(e,n).getResult(0)]});ce("cos",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.sin(t).getResult(0),s=r.builder.neg(n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("abs",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.sign(t).getResult(0);return[r.builder.mul(e,n).getResult(0)]});l(AM,"_erfDerivIR");l(OM,"_digammaIR");ce("erf",r=>{let e=r.gradOutputs[0],[t]=r.operands;return[r.builder.mul(e,AM(r,t)).getResult(0)]});ce("erfc",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder.neg(AM(r,t)).getResult(0);return[r.builder.mul(e,n).getResult(0)]});ce("lgamma",r=>{let e=r.gradOutputs[0],[t]=r.operands;return[r.builder.mul(e,OM(r,t)).getResult(0)]});ce("gamma",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.results[0],s=r.builder.mul(n,OM(r,t)).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("log2",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(Math.LN2,t.type),s=r.builder.mul(t,n).getResult(0);return[r.builder.div(e,s).getResult(0)]});ce("log10",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(Math.LN10,t.type),s=r.builder.mul(t,n).getResult(0);return[r.builder.div(e,s).getResult(0)]});ce("exp2",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.full(Math.LN2,t.type),s=r.builder.mul(t,n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("square",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(2,t.type),s=r.builder.mul(n,t).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("reciprocal",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.builder.mul(t,t).getResult(0),s=r.builder.neg(n).getResult(0);return[r.builder.mul(e,s).getResult(0)]});ce("rsqrt",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.builder.mul(r.builder.mul(t,t).getResult(0),t).getResult(0),s=r.full(-.5,t.type),o=r.builder.mul(s,n).getResult(0);return[r.builder.mul(e,o).getResult(0)]})});var RM=M(()=>{"use strict";ls();ce("dot",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.op.getAttr("lhs_contracting"),o=r.op.getAttr("rhs_contracting"),i=r.op.getAttr("lhs_batch")||[],a=r.op.getAttr("rhs_batch")||[],u=t.type.rank,p=n.type.rank,c=e.type.rank,h=[];for(let _=0;_<u;_++)!s.includes(_)&&!i.includes(_)&&h.push(_);let f=[];for(let _=0;_<p;_++)!o.includes(_)&&!a.includes(_)&&f.push(_);let d=[];for(let _=i.length;_<i.length+f.length;_++)d.push(_+h.length);let m=[];for(let _=i.length;_<i.length+h.length;_++)m.push(_);let g=r.builder.dot(e,n,d,f,Array.from({length:i.length},(_,b)=>b),a).getResult(0),y=r.builder.dot(t,e,h,m,i,Array.from({length:i.length},(_,b)=>b)).getResult(0);return[g,y]});ce("matmul",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=l(p=>{let c=Array.from({length:p},(h,f)=>f);return c[p-2]=p-1,c[p-1]=p-2,c},"swapLastTwo"),o=r.builder.transpose(n,s(n.type.rank)).getResult(0),i=r.builder.matmul(e,o).getResult(0),a=r.builder.transpose(t,s(t.type.rank)).getResult(0),u=r.builder.matmul(a,e).getResult(0);return[i,u]});ce("conv",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.builder,o=r.op.getAttr("strides"),i=r.op.getAttr("padding"),a=r.op.getAttr("dilation")||o.map(()=>1),u=r.op.getAttr("groups")||1,p=r.op.getAttr("input_layout"),c=r.op.getAttr("kernel_layout");if(!(o.length===2&&o.every(A=>A===1)&&a.every(A=>A===1)&&u===1&&p==="NCHW"&&c==="OIHW"))throw new Error("conv VJP supports only 2D stride-1 dilation-1 groups-1 NCHW/OIHW conv");let f=n.type.shape,d=f[2],m=f[3],[g,y]=i,_=s.reverse(s.transpose(n,[1,0,2,3]).getResult(0),[2,3]).getResult(0),b=[[d-1-g[0],d-1-g[1]],[m-1-y[0],m-1-y[1]]],w=s.conv(e,_,[1,1],b).getResult(0),x=s.transpose(t,[1,0,2,3]).getResult(0),S=s.transpose(e,[1,0,2,3]).getResult(0),T=s.conv(x,S,[1,1],[g,y]).getResult(0),k=s.transpose(T,[1,0,2,3]).getResult(0);return[w,k]})});var MM=M(()=>{"use strict";ls();ce("reduce",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("reduce_type"),s=r.op.getAttr("dimensions"),o=t.type.shape,i=t.type.dtype,a=new Set(s),u=o.map((h,f)=>a.has(f)?1:h),p=o.map((h,f)=>f),c=r.builder.reshape(e,u).getResult(0);if(n==="sum")return[r.builder.broadcast(c,o,p).getResult(0),null];if(n==="mean"){let h=1;for(let y=0;y<o.length;y++)a.has(y)&&(h*=o[y]);let f=r.builder.broadcast(c,o,p).getResult(0),d=r.builder.scalarConstant(h,i).getResult(0),m=r.builder.broadcast(d,o,[]).getResult(0);return[r.builder.div(f,m).getResult(0),null]}if(n==="max"||n==="min"){let h=r.builder.reshape(r.results[0],u).getResult(0),f=r.builder.broadcast(h,o,p).getResult(0),d=r.builder.compare(t,f,"eq").getResult(0),m=r.builder.convert(d,i).getResult(0),g=r.builder.broadcast(c,o,p).getResult(0);return[r.builder.mul(g,m).getResult(0),null]}throw n==="prod"?new Error("reduce VJP for reduce_type='prod' is not implemented; provide a gradient rule or avoid differentiating reduce_prod"):new Error(`reduce VJP: unsupported reduce_type '${n}' on the gradient path (would silently drop the gradient)`)})});var IM=M(()=>{"use strict";ls();ce("reshape",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=t.type.shape;return[r.builder.reshape(e,n).getResult(0)]});ce("reverse",r=>[r.builder.reverse(r.gradOutputs[0],r.op.getAttr("dimensions")).getResult(0)]);ce("transpose",r=>{let e=r.gradOutputs[0],t=r.op.getAttr("permutation"),n=new Array(t.length);for(let s=0;s<t.length;s++)n[t[s]]=s;return[r.builder.transpose(e,n).getResult(0)]});ce("broadcast_in_dim",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("broadcast_dimensions"),s=t.type.shape,o=r.results[0].type.shape,i=t.type.dtype,a=[];for(let h=0;h<o.length;h++)if(!n.includes(h))a.push(h);else{let f=n.indexOf(h);s[f]===1&&o[h]!==1&&a.push(h)}if(a.length===0)return[e];let u=r.builder.scalarConstant(0,i).getResult(0),p=r.builder.reduce(e,u,a,"sum").getResult(0),c=p.type.shape;return c.length!==s.length||!c.every((h,f)=>h===s[f])?[r.builder.reshape(p,s).getResult(0)]:[p]});ce("slice",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("starts"),s=t.type.shape,o=e.type.shape,i=r.op.getAttr("strides")||s.map(()=>1),a=t.type.dtype,u=[...n],p=new Array(s.length),c=new Array(s.length);for(let f=0;f<s.length;f++)c[f]=i[f]-1,p[f]=s[f]-n[f]-(o[f]-1)*i[f]-1;let h=r.builder.scalarConstant(0,a).getResult(0);return[r.builder.pad(e,h,u,p,c).getResult(0)]});ce("concat",r=>{let e=r.gradOutputs[0],t=r.op.getAttr("dimension"),n=r.operands,s=[],o=0;for(let i=0;i<n.length;i++){let a=n[i].type.shape,u=new Array(a.length).fill(0),p=[...e.type.shape];u[t]=o,p[t]=o+a[t],s.push(r.builder.slice(e,u,p).getResult(0)),o+=a[t]}return s});ce("gather",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.builder.scalarConstant(0,t.type.dtype).getResult(0),o=r.builder.broadcast(s,t.type.shape,[]).getResult(0);return[r.builder.scatterAdd(o,n,e,{updateWindowDims:r.op.getAttr("offset_dims"),insertedWindowDims:r.op.getAttr("collapsed_slice_dims"),scatterDimsToOperandDims:r.op.getAttr("start_index_map"),indexVectorDim:r.op.getAttr("index_vector_dim")}).getResult(0),null]});ce("scatter",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.op.getAttr("inserted_window_dims"),o=t.type.shape.map((a,u)=>s.includes(u)?1:a),i=r.builder.gather(e,n,{offsetDims:r.op.getAttr("update_window_dims"),collapsedSliceDims:s,startIndexMap:r.op.getAttr("scatter_dims_to_operand_dims"),indexVectorDim:r.op.getAttr("index_vector_dim"),sliceSizes:o}).getResult(0);return[e,null,i]});ce("pad",r=>{let e=r.gradOutputs[0],t=r.op.getAttr("low"),n=r.op.getAttr("high"),[s]=r.operands,o=s.type.shape,i=[...t],a=new Array(o.length);for(let u=0;u<o.length;u++)a[u]=t[u]+o[u];return[r.builder.slice(e,i,a).getResult(0),null]})});var EM=M(()=>{"use strict";ls();es();ce("softmax",r=>{let e=r.gradOutputs[0],t=r.results[0],n=r.op.getAttr("axis"),s=t.type.dtype,o=t.type.shape,i=r.builder.mul(e,t).getResult(0),a=r.builder.scalarConstant(0,s).getResult(0),u=r.builder.reduce(i,a,[n],"sum").getResult(0),p=To(o.length,n),c=r.builder.broadcast(u,o,p).getResult(0),h=r.builder.sub(e,c).getResult(0);return[r.builder.mul(t,h).getResult(0)]});ce("log_softmax",r=>{let e=r.gradOutputs[0],t=r.results[0],[n]=r.operands,s=r.op.getAttr("axis"),o=t.type.dtype,i=t.type.shape,a=r.builder.exp(t).getResult(0),u=r.builder.scalarConstant(0,o).getResult(0),p=r.builder.reduce(e,u,[s],"sum").getResult(0),c=To(i.length,s),h=r.builder.broadcast(p,i,c).getResult(0),f=r.builder.mul(a,h).getResult(0);return[r.builder.sub(e,f).getResult(0)]});ce("layer_norm",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=r.op.getAttr("axis"),o=r.op.getAttr("epsilon"),i=t.type.dtype,a=t.type.shape,u=[s<0?a.length+s:s],p=a[u[0]],c=To(a.length,u[0]),h=r.builder.scalarConstant(0,i).getResult(0),f=r.builder.reduce(t,h,u,"mean").getResult(0),d=r.builder.broadcast(f,a,c).getResult(0),m=r.builder.sub(t,d).getResult(0),g=r.builder.mul(m,m).getResult(0),y=r.builder.reduce(g,h,u,"mean").getResult(0),_=r.full(o,y.type),b=r.builder.add(y,_).getResult(0),w=r.builder.rsqrt(b).getResult(0),x=r.builder.broadcast(w,a,c).getResult(0),S=[];for(let he=0;he<a.length;he++)u.includes(he)&&S.push(he);let T=r.builder.broadcast(n,a,S).getResult(0),k=r.builder.mul(e,T).getResult(0),A=r.full(p,t.type),N=r.builder.mul(A,k).getResult(0),v=r.builder.reduce(k,h,u,"sum").getResult(0),O=r.builder.broadcast(v,a,c).getResult(0),R=r.builder.mul(m,x).getResult(0),B=r.builder.mul(k,R).getResult(0),E=r.builder.reduce(B,h,u,"sum").getResult(0),z=r.builder.broadcast(E,a,c).getResult(0),D=r.builder.mul(R,z).getResult(0),J=r.builder.sub(N,r.builder.add(O,D).getResult(0)).getResult(0),j=r.builder.div(x,A).getResult(0),U=r.builder.mul(j,J).getResult(0),P=r.builder.reduce(r.builder.mul(e,R).getResult(0),h,c,"sum").getResult(0),V=r.builder.reduce(e,h,c,"sum").getResult(0);return[U,P,V]});ce("scaled_dot_product_attention",r=>{let e=r.gradOutputs[0],[t,n,s]=r.operands,o=r.op.getAttr("scale");if(r.op.getAttr("causal"))throw new Error("causal scaled_dot_product_attention VJP not supported");let i=r.builder,a=t.type.dtype,u=t.type.rank,p=[];for(let N=0;N<u;N++)p.push(N);p[u-2]=u-1,p[u-1]=u-2;let c=l(N=>i.transpose(N,p).getResult(0),"lastT"),h=i.matmul(t,c(n)).getResult(0),f=i.mul(h,r.full(o,h.type)).getResult(0),d=i.softmax(f,u-1).getResult(0),m=i.matmul(c(d),e).getResult(0),g=i.matmul(e,c(s)).getResult(0),y=i.scalarConstant(0,a).getResult(0),_=i.mul(g,d).getResult(0),b=i.reduce(_,y,[u-1],"sum").getResult(0),w=[];for(let N=0;N<u-1;N++)w.push(N);let x=i.broadcast(b,d.type.shape,w).getResult(0),S=i.mul(d,i.sub(g,x).getResult(0)).getResult(0),T=i.mul(S,r.full(o,S.type)).getResult(0),k=i.matmul(T,n).getResult(0),A=i.matmul(c(T),t).getResult(0);return[k,A,m]});ce("pool2d",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.builder,s=r.op.getAttr("pool_type"),o=r.op.getAttr("kernel_size"),i=r.op.getAttr("strides"),a=r.op.getAttr("padding"),u=r.op.getAttr("layout")||"NCHW",p=a.every(k=>k[0]===0&&k[1]===0),c=i[0]===o[0]&&i[1]===o[1];if(u!=="NCHW"||!p||!c||s!=="avg"&&s!=="max")throw new Error("pool2d VJP supports only non-overlapping (stride=kernel) avg/max pooling without padding, NCHW");let[h,f,d,m]=e.type.shape,[g,y]=o,_=t.type.shape,b=l(k=>n.reshape(n.broadcast(k,[h,f,d,g,m,y],[0,1,2,4]).getResult(0),_).getResult(0),"upsample"),w=b(e);if(s==="avg"){let k=r.full(g*y,t.type);return[n.div(w,k).getResult(0)]}let x=b(r.results[0]),S=n.compare(t,x,"eq").getResult(0),T=r.full(0,t.type);return[n.select(S,w,T).getResult(0)]});ce("batch_norm",r=>{let e=r.gradOutputs[0],[t,n,,s,o]=r.operands,i=r.op.getAttr("axis"),a=r.op.getAttr("epsilon"),u=t.type.dtype,p=t.type.shape,c=r.builder,h=To(p.length,i),f=c.scalarConstant(0,u).getResult(0),d=r.full(a,o.type),m=c.rsqrt(c.add(o,d).getResult(0)).getResult(0),g=c.broadcast(m,p,[i]).getResult(0),y=c.broadcast(s,p,[i]).getResult(0),_=c.broadcast(n,p,[i]).getResult(0),b=c.sub(t,y).getResult(0),w=c.mul(b,g).getResult(0),x=c.mul(e,_).getResult(0),S=c.mul(x,g).getResult(0),T=c.reduce(c.mul(e,w).getResult(0),f,h,"sum").getResult(0),k=c.reduce(e,f,h,"sum").getResult(0),A=c.neg(c.reduce(S,f,h,"sum").getResult(0)).getResult(0),N=c.mul(c.mul(g,g).getResult(0),g).getResult(0),v=r.full(-.5,t.type),O=c.mul(c.mul(c.mul(x,b).getResult(0),N).getResult(0),v).getResult(0),R=c.reduce(O,f,h,"sum").getResult(0);return[S,T,k,A,R]});ce("elu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.results[0],s=r.op.getAttr("alpha")??1,o=r.full(0,t.type),i=r.full(1,t.type),a=r.full(s,t.type),u=r.builder.compare(t,o,"gt").getResult(0),p=r.builder.add(n,a).getResult(0),c=r.builder.select(u,i,p).getResult(0);return[r.builder.mul(e,c).getResult(0)]});ce("leaky_relu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("negative_slope")??.01,s=r.full(0,t.type),o=r.full(1,t.type),i=r.full(n,t.type),a=r.builder.compare(t,s,"gt").getResult(0),u=r.builder.select(a,o,i).getResult(0);return[r.builder.mul(e,u).getResult(0)]});ce("celu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.op.getAttr("alpha")??1,s=r.full(0,t.type),o=r.full(1,t.type),i=r.builder.compare(t,s,"gt").getResult(0),a=r.full(n,t.type),u=r.builder.div(t,a).getResult(0),p=r.builder.exp(u).getResult(0),c=r.builder.select(i,o,p).getResult(0);return[r.builder.mul(e,c).getResult(0)]});ce("selu",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=1.0507009873554805,s=1.6732632423543772,o=r.full(0,t.type),i=r.full(n,t.type),a=r.builder.compare(t,o,"gt").getResult(0),u=r.full(s,t.type),p=r.builder.exp(t).getResult(0),c=r.builder.mul(u,p).getResult(0),h=r.builder.select(a,i,r.builder.mul(i,c).getResult(0)).getResult(0);return[r.builder.mul(e,h).getResult(0)]});ce("hardswish",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(-3,t.type),s=r.full(3,t.type),o=r.full(0,t.type),i=r.full(1,t.type),a=r.full(2,t.type),u=r.full(6,t.type),p=r.builder.compare(t,n,"le").getResult(0),c=r.builder.compare(t,s,"ge").getResult(0),h=r.builder.add(r.builder.mul(a,t).getResult(0),s).getResult(0),f=r.builder.div(h,u).getResult(0),d=r.builder.select(p,o,r.builder.select(c,i,f).getResult(0)).getResult(0);return[r.builder.mul(e,d).getResult(0)]});ce("hardsigmoid",r=>{let e=r.gradOutputs[0],[t]=r.operands,n=r.full(-3,t.type),s=r.full(3,t.type),o=r.full(0,t.type),i=r.full(1/6,t.type),a=r.builder.compare(t,n,"le").getResult(0),u=r.builder.compare(t,s,"ge").getResult(0),p=r.builder.select(a,o,r.builder.select(u,o,i).getResult(0)).getResult(0);return[r.builder.mul(e,p).getResult(0)]});ce("embedding",r=>{let e=r.gradOutputs[0],[t,n]=r.operands,s=n.type.rank,o=r.full(0,t.type);return[r.builder.scatter(o,n,e,{updateWindowDims:[s],insertedWindowDims:[0],scatterDimsToOperandDims:[0],indexVectorDim:s}).getResult(0),null]})});var LM=M(()=>{"use strict";ls();bx("stop_gradient");ce("stop_gradient",()=>[null]);for(let r of["compare","logical_not","argmax","argmin","iota"])bx(r)});var CM=M(()=>{"use strict";ls();vf();Af();xx();y_();SM();NM();BM();RM();MM();IM();EM();LM()});var DM={};nt(DM,{compileWithBackward:()=>b_});function __(r){return typeof r.then=="function"}function b_(r,e,t={}){let n=t.target??Us(),s=t.mode||"separate",o=t.rematPolicy||new Ea(t.remat||{}),i={target:n,verify:!1,...t,backward:void 0,mode:void 0,rematPolicy:void 0,remat:void 0},a=t.dynamic_shapes||null,u=[],p=null,c=null,h=null;function f(v){let O=T_((...B)=>r.forward(...B),v,{name:r.constructor.name||"compiled",dynamicShapes:a}),R=l(B=>{let E=B.graph.functions().next().value,z=s==="joint"?m(E,B,o):d(E,B,o);return z.shapeEnv=B.shapeEnv,z.outputSymShapes=B.outputSymShapes,z},"finish");return __(O)?O.then(R):R(O)}l(f,"_compile");function d(v,O,R){let B=new Bf({rematPolicy:R}),{backwardFunc:E,savedValues:z,gradInputIndices:D}=B.build(v),J=v.getReturnOp(),j=[...J.operands],U=j.length,P=new Map(v.args.map((Se,Qe)=>[Se.id,Qe])),V=new Map(j.map((Se,Qe)=>[Se.id,Qe])),he=[],le=new Map;for(let Se of z)P.has(Se.id)||V.has(Se.id)||le.has(Se.id)||(le.set(Se.id,U+he.length),he.push(Se));he.length>0&&(J.erase(),new An(v).returnOp([...j,...he]),v.outputTypes=Object.freeze([...j.map(Se=>Se.type),...he.map(Se=>Se.type)]));let Ie=z.map(Se=>P.has(Se.id)?{kind:"arg",index:P.get(Se.id)}:V.has(Se.id)?{kind:"output",index:V.get(Se.id)}:{kind:"output",index:le.get(Se.id)}),G=new wn("forward");G.addFunction(v);let K=new Do(i).compile(G),ie=new wn("backward");ie.addFunction(E);let ze=new Do(i).compile(ie);return{mode:"separate",fwdResult:K,bwdResult:ze,forwardFunc:v,backwardFunc:E,savedValues:z,savedSources:Ie,numRealOutputs:U,gradInputIndices:D,capturedParams:O.capturedParams,numUserInputs:O.numUserInputs,outputTypes:O.outputTypes,shapeEnv:O.shapeEnv,outputSymShapes:O.outputSymShapes}}l(d,"_compileSeparate");function m(v,O,R){let B=new Rf({rematPolicy:R}),{jointFunc:E,numForwardOutputs:z,numGradInputs:D}=B.build(v),J=new wn("joint");return J.addFunction(E),{mode:"joint",result:new Do(i).compile(J),jointFunc:E,numForwardOutputs:z,numGradInputs:D,capturedParams:O.capturedParams,numUserInputs:O.numUserInputs,outputTypes:O.outputTypes,inputTypes:v.inputTypes,shapeEnv:O.shapeEnv,outputSymShapes:O.outputSymShapes}}l(m,"_compileJoint");function g(v,O){return v.outputSymShapes&&v.shapeEnv?v.shapeEnv.resolveSymbolicShape(v.outputSymShapes[O]):[...v.outputTypes[O].shape]}l(g,"_resolveOutputShape");function y(v,O,R){let B=v.module||v;return B.executionPlan?B.runPlanAsync(B.executionPlan,R):v.isAsync(O)?v.runAsync(O,...R):(v.run(O,...R),null)}l(y,"_runK");function _(v,O){let B=v.fwdResult.listKernels()[0],E=O.length>0?O[0].device:"cpu",z=O.map(G=>Je(G)),J=v.capturedParams.map(G=>Je(G)),j=v.forwardFunc.outputTypes,U=v.numRealOutputs,P=new Array(j.length),V=new Array(j.length);for(let G=0;G<j.length;G++){let K=G<U?g(v,G):[...j[G].shape],ie=j[G].dtype,ze=cn(K),Se=Dt(ie);P[G]=new Se(Math.max(ze,1)),V[G]=K}let he=[...z,...J,...P],le=l(()=>({results:U===1?tn(P[0],V[0],j[0].dtype,E):Array.from({length:U},(K,ie)=>tn(P[ie],V[ie],j[ie].dtype,E)),inputArrays:z,paramArrays:J,outputArrays:P,device:E}),"build"),Ie=y(v.fwdResult,B,he);return Ie?Ie.then(le):le()}l(_,"_executeSeparateForward");function b(v,O,R){let E=v.bwdResult.listKernels()[0],z=O.map(ie=>Je(ie)),D=v.savedValues,J=v.savedSources,j=[...R.inputArrays,...R.paramArrays],U=new Array(D.length);for(let ie=0;ie<D.length;ie++){let ze=J[ie];U[ie]=ze.kind==="arg"?j[ze.index]:R.outputArrays[ze.index]}let P=v.backwardFunc,V=P.outputTypes.length,he=new Array(V),le=new Array(V);for(let ie=0;ie<V;ie++){let ze=[...P.outputTypes[ie].shape],Se=P.outputTypes[ie].dtype,Qe=cn(ze),It=Dt(Se);he[ie]=new It(Math.max(Qe,1)),le[ie]=ze}let Ie=[...z,...U,...he],G=l(()=>he.map((ie,ze)=>tn(ie,le[ze],P.outputTypes[ze].dtype,R.device)),"build"),K=y(v.bwdResult,E,Ie);return K?K.then(G):G()}l(b,"_executeSeparateBackward");function w(v){for(let O=0;O<u.length;O++){let R=u[O];R.shapeEnv.bindInputShapes(v);let{passed:B}=R.shapeEnv.evaluateGuards();if(B)return R}return null}l(w,"_findCachedEntry");function x(v,O){if(c=v,v.mode==="joint")return k(v,O);let R=_(v,O);return __(R)?R.then(B=>(p=B,B.results)):(p=R,R.results)}l(x,"_forwardWith");function S(v){let O=w(v);if(O)return x(O,v);let R=f(v);return __(R)?R.then(B=>(u.push(B),B.shapeEnv.bindInputShapes(v),x(B,v))):(u.push(R),R.shapeEnv.bindInputShapes(v),x(R,v))}l(S,"_runForward");function T(...v){return h?h.then(()=>S(v)):S(v)}l(T,"compiledForward");function k(v,O){let B=v.result.listKernels()[0],E=O.length>0?O[0].device:"cpu",z=O.map(K=>Je(K)),J=v.capturedParams.map(K=>Je(K)),j=v.jointFunc,U=j.outputTypes.length,P=new Array(U),V=new Array(U);for(let K=0;K<U;K++){let ie=[...j.outputTypes[K].shape],ze=j.outputTypes[K].dtype,Se=cn(ie),Qe=Dt(ze);P[K]=new Qe(Math.max(Se,1)),V[K]=ie}let he=new Array(v.numForwardOutputs);for(let K=0;K<v.numForwardOutputs;K++){let ie=v.outputTypes[K],ze=cn(ie.shape),Se=Dt(ie.dtype);he[K]=new Se(Math.max(ze,1))}p={inputArrays:z,paramArrays:J,gradOutputArrays:he,outputArrays:P,outputShapes:V,device:E,compiled:v};let le=[...z,...J,...he,...P],Ie=l(()=>{let K=[];for(let ie=0;ie<v.numForwardOutputs;ie++)K.push(tn(P[ie],V[ie],j.outputTypes[ie].dtype,E));return K.length===1?K[0]:K},"build"),G=y(v.result,B,le);return G?G.then(Ie):Ie()}l(k,"_executeJointForward");let A=T;A.backward=function(...v){if(!c||!p)throw new Error("Must run forward before backward");return c.mode==="joint"?N(c,v,p):b(c,v,p)};function N(v,O,R){let{inputArrays:B,paramArrays:E,outputArrays:z,outputShapes:D,device:J}=R,j=O.map(K=>Je(K));for(let K=0;K<j.length;K++)R.gradOutputArrays[K].set(j[K]);let U=v.jointFunc,P=U.outputTypes.length,V=new Array(P),he=new Array(P);for(let K=0;K<P;K++){let ie=[...U.outputTypes[K].shape],ze=U.outputTypes[K].dtype,Se=cn(ie),Qe=Dt(ze);V[K]=new Qe(Math.max(Se,1)),he[K]=ie}let le=[...B,...E,...j,...V],Ie=l(()=>{let K=[];for(let ie=v.numForwardOutputs;ie<P;ie++)K.push(tn(V[ie],he[ie],U.outputTypes[ie].dtype,J));return K},"build"),G=y(v.result,v.result.listKernels()[0],le);return G?G.then(Ie):Ie()}if(l(N,"_executeJointBackward"),A.original=r,A.backwardGraph=()=>{if(u.length===0)return null;let v=u[0];return v.mode==="joint"?v.jointFunc:v.backwardFunc},A.forwardGraph=()=>{if(u.length===0)return null;let v=u[0];return v.mode==="joint"?v.jointFunc:v.forwardFunc},A.capturedParams=()=>u.length?u[0].capturedParams:[],e){let v=f(e);__(v)?h=v.then(O=>(u.push(O),h=null,O)):u.push(v)}return A}var w_=M(()=>{"use strict";Sx();u_();Ks();Yo();Af();es();xx();y_();CM();Kn();ts();jr();l(__,"_isThenable");l(b_,"compileWithBackward")});function La(r){return typeof r.then=="function"}function jz(){PM||(mM(),PM=!0)}function qz(r,e){if(!r)return new Array(e.length).fill(null);let t=new Array(e.length);for(let n=0;n<e.length;n++){let s=r[n];if(s===!0){let o=new Set;for(let i=0;i<e[n].shape.length;i++)o.add(i);t[n]=o}else s instanceof Set?t[n]=s:t[n]=null}return t}function T_(r,e,t){jz();let n=t?.name||r.name||"traced",s=new Eg(n),o=qz(t?.dynamicShapes,e);for(let h=0;h<e.length;h++)s.createInput(e[h].shape,e[h].dtype,o[h]);let i=e.length,a=s._initGraph();function u(h){Array.isArray(h)?s.markOutputs(h):s.markOutput(h),s.deactivate();let f=s.getGraphModule(),d=f.functions().next().value;return{graph:f,capturedParams:[...s.capturedParams],numUserInputs:i,outputTypes:d.outputTypes,shapeEnv:s.shapeEnv,outputSymShapes:s.outputSymShapes}}l(u,"_finalize"),s.activate();let p=Pr.fromKey(48),c=LS(p,()=>r(...a));if(La(c))return c.then(h=>u(h),h=>{throw s.deactivate(),h});try{return u(c)}catch(h){throw s.deactivate(),h}}function $M(r,e,t){let n=T_(r,e,t);return La(n)?n.then(s=>s.graph):n.graph}function Uz(r,e,t){let n=r.result.listKernels();if(n.length===0)throw new Error("No kernels compiled");let s=n[0],o=e.length>0?e[0].device:"cpu",i=new Array(e.length);for(let g=0;g<e.length;g++)i[g]=Je(e[g]);let a=r.capturedParams,u=new Array(a.length);for(let g=0;g<a.length;g++)u[g]=Je(a[g]);let p=r.outputTypes,c=r.outputSymShapes,h=new Array(p.length),f=new Array(p.length);for(let g=0;g<p.length;g++){let y=c&&t?t.resolveSymbolicShape(c[g]):p[g].shape,_=p[g].dtype,b=cn(y),w=Dt(_);h[g]=new w(Math.max(b,1)),f[g]=[...y]}let d=new Array(i.length+u.length+h.length),m=0;for(let g=0;g<i.length;g++)d[m++]=new to(i[g],e[g].shape,e[g].dtype);for(let g=0;g<u.length;g++){let y=new to(u[g],a[g].shape,a[g].dtype),_=a[g]._impl;_&&(y.resident={key:_.storage.rawData,version:_.version}),d[m++]=y}for(let g=0;g<h.length;g++)d[m++]=new to(h[g],f[g],p[g].dtype);return{funcName:s,device:o,outputTypes:p,outputArrays:h,outputShapes:f,allArgs:d}}function kx(r,e,t,n){if(e.length===1)return tn(t[0],n[0],e[0].dtype,r);let s=new Array(e.length);for(let o=0;o<e.length;o++)s[o]=tn(t[o],n[o],e[o].dtype,r);return s}function Kz(r,e,t){let{funcName:n,device:s,outputTypes:o,outputArrays:i,outputShapes:a,allArgs:u}=Uz(r,e,t),p=r.result.module.executionPlan;return p?r.result.module.runPlanAsync(p,u,{resident:!0}).then(()=>kx(s,o,i,a)):r.result.isAsync(n)?r.result.runAsync(n,...u).then(()=>kx(s,o,i,a)):(r.result.run(n,...u),kx(s,o,i,a))}function Wz(r,e){let t=[];for(let n of r.result.listKernels()){let s=r.result.getSource(n);s!==null&&t.push(`// ---- compiled entry ${e}, kernel ${n} ----
 ${s}`)}return t.length>0?t.join(`
 
 `):null}function Hz(r,e,t={}){if(t?.backward)return b_(r,e,t);let n=t?.target??Us(),s={target:n,verify:!1,...t},o=t?.dynamic_shapes||null,i=t?.shapeBuckets||null,a=t?.foldWeights??t?.quantization?.foldWeights??!1,u=[];function p(w,x,S){if(!w||typeof w!="object"||w.repro)return w;try{w.repro={name:r.constructor?.name||"compiled",phase:S,target:n?.name,inputs:(x||[]).map(T=>({shape:T.shape,dtype:T.dtype})),config:{fusion:s.fusion,scheduling:s.scheduling,optimization:s.optimization,quantization:s.quantization,dynamicShapes:!!o}}}catch{return w}return w}l(p,"_attachRepro");function c(w){let x=a?yM(w,Je):w;return{result:new Do(s).compile(x.graph),graph:x.graph,capturedParams:x.capturedParams,numUserInputs:x.numUserInputs,outputTypes:x.outputTypes,shapeEnv:x.shapeEnv,outputSymShapes:x.outputSymShapes}}l(c,"_finalize");function h(w,x){try{let S=T_((...T)=>r.forward(...T),w,{name:r.constructor.name||"compiled",dynamicShapes:x});return La(S)?S.then(c,T=>{throw p(T,w,"compile")}):c(S)}catch(S){throw p(S,w,"compile")}}l(h,"_compileWith");function f(w){return h(w,o)}l(f,"_compile");function d(w){return w.map((x,S)=>({shape:x,dtype:e[S].dtype}))}l(d,"_bucketInputs");function m(w){for(let x=0;x<u.length;x++){let S=u[x];S.shapeEnv.bindInputShapes(w);let{passed:T}=S.shapeEnv.evaluateGuards();if(T)return S}return null}l(m,"_findCachedEntry");function g(w,x){try{let S=Kz(w,x,w.shapeEnv);return La(S)?S.then(void 0,T=>{throw p(T,x,"run")}):S}catch(S){throw p(S,x,"run")}}l(g,"_execute");function y(...w){let x=m(w);if(!x){let S=f(w);if(La(S))return S.then(T=>(u.push(T),T.shapeEnv.bindInputShapes(w),g(T,w)));x=S,u.push(x),x.shapeEnv.bindInputShapes(w)}return g(x,w)}l(y,"compiledForward");let _=null;if(e){let w=[];if(i)for(let x of i)w.push(h(d(x),null));if(w.push(f(e)),w.some(x=>La(x)))_=Promise.all(w).then(x=>{for(let S of x)u.push(S)});else for(let x of w)La(x)||u.push(x)}let b=y;return b.original=r,b.graph=w=>$M((...S)=>r.forward(...S),w||e,{name:r.constructor.name||"compiled",dynamicShapes:o}),b.source=()=>u.length===0?null:Wz(u[0],0),b.kernels=()=>u.length===0?[]:u[0].result.listKernels(),b.result=()=>u.length>0?u[0].result:null,b._ready=_,b}var PM,Sx=M(()=>{"use strict";ya();gM();Nn();ab();u_();Ks();Kn();tm();ts();jr();_M();w_();PM=!1;l(La,"_isThenable");l(jz,"_ensureTracing");l(qz,"_normalizeDynamicShapes");l(T_,"_traceCore");l($M,"trace");l(Uz,"_prepareExecution");l(kx,"_wrapOutputs");l(Kz,"executeCompiled");l(Wz,"sourceForEntry");l(Hz,"compile")});var GM={};nt(GM,{beginEagerCapture:()=>n3,endEagerCapture:()=>r3,replay:()=>s3,syncStream:()=>o3});var x_,n3,r3,s3,o3,jM=M(()=>{"use strict";x_=l(r=>{throw new Error(`mlfw: CUDA eager-graph (${r}) is not available in the browser`)},"unavailable"),n3=l(()=>x_("beginEagerCapture"),"beginEagerCapture"),r3=l(()=>x_("endEagerCapture"),"endEagerCapture"),s3=l(()=>x_("replay"),"replay"),o3=l(()=>x_("syncStream"),"syncStream")});var qM={};nt(qM,{clearCapturePins:()=>c3,deviceBufferDptr:()=>u3,deviceBufferForInput:()=>a3,pinResident:()=>l3,setEagerDeferred:()=>i3});var If,i3,a3,u3,l3,c3,UM=M(()=>{"use strict";If=l(r=>{throw new Error(`mlfw: CUDA resident memory (${r}) is not available in the browser`)},"unavailable"),i3=l(()=>If("setEagerDeferred"),"setEagerDeferred"),a3=l(()=>If("deviceBufferForInput"),"deviceBufferForInput"),u3=l(()=>If("deviceBufferDptr"),"deviceBufferDptr"),l3=l(()=>If("pinResident"),"pinResident"),c3=l(()=>If("clearCapturePins"),"clearCapturePins")});var WM={};nt(WM,{copyDeviceToHost:()=>p3,copyHostToDeviceAsync:()=>f3});var KM,p3,f3,HM=M(()=>{"use strict";KM=l(r=>{throw new Error(`mlfw: CUDA memory (${r}) is not available in the browser`)},"unavailable"),p3=l(()=>KM("copyDeviceToHost"),"copyDeviceToHost"),f3=l(()=>KM("copyHostToDeviceAsync"),"copyHostToDeviceAsync")});gs();Nn();Kn();km();jn();Oo();ea();ar();jr();ts();function Qu(r,e){let t=xn(r),n=En.allocate(0,e,Nu),s=new Bt(n,0,r,t,e,Nu);return new We(s)}l(Qu,"_metaTensor");function KC(r,e,t){let n=Au(e.shape,t.shape);if(!n)throw new Error(`Incompatible shapes: [${e.shape}] vs [${t.shape}]`);let s=Bd(e.dtype,t.dtype);return Qu(n,s)}l(KC,"_metaBinary");function WC(r,e){return Qu([...e.shape],e.dtype)}l(WC,"_metaUnary");function HC(r,e,t,n){let s=[...e.shape],o=t!=null?Array.isArray(t)?t:[t]:Array.from({length:s.length},(u,p)=>p);if(o.length===s.length||t===void 0)return Qu(n?s.map(()=>1):[],e.dtype);let i=[],a=new Set(o.map(u=>u<0?s.length+u:u));for(let u=0;u<s.length;u++)a.has(u)?n&&i.push(1):i.push(s[u]);return Qu(i,e.dtype)}l(HC,"_metaReduction");function YC(r,e,t){let n=Od(e.shape,t.shape);if(n===null)throw new Error("metaMatmul: unsupported shapes");return Qu(n,e.dtype)}l(YC,"metaMatmul");function JC(r,e){return Qu([...e.shape],e.dtype)}l(JC,"metaClone");function _0(r,e){return Object.fromEntries(r.map(t=>[t,e]))}l(_0,"mapKernel");var LA=Object.freeze({..._0(["add","sub","mul","div","pow","rem","maximum","minimum"],KC),..._0(["neg","exp","log","sqrt","rsqrt","abs","sin","cos","tanh","erf","erfc","lgamma","gamma","sigmoid","relu","gelu","silu","sign","floor","ceil"],WC),..._0(["sum","mean","max","min","prod"],HC),matmul:YC,clone:JC});jn();Oo();ea();ts();jr();function CA(r,e,t,n){let s=r.length,o=t<0?s+t:t,i=n<0?s+n:n,a=[...r],u=[...e];return a[o]=r[i],a[i]=r[o],u[o]=e[i],u[i]=e[o],{sizes:a,strides:u}}l(CA,"computeTranspose");function DA(r,e,t){let n=r.length,s=new Array(n),o=new Array(n);for(let i=0;i<n;i++){let a=t[i],u=a<0?n+a:a;s[i]=r[u],o[i]=e[u]}return{sizes:s,strides:o}}l(DA,"computePermute");function PA(r,e,t){let n=r.length,s=t.length,o=s-n,i=new Array(s),a=new Array(s);for(let u=0;u<s;u++){let p=u-o;if(p<0)a[u]=t[u],i[u]=0;else if(r[p]===1&&t[u]!==1)a[u]=t[u],i[u]=0;else if(r[p]===t[u]||t[u]===-1)a[u]=r[p],i[u]=e[p];else throw new Error(`Cannot expand size ${r[p]} to ${t[u]} at dim ${u}`)}return{sizes:a,strides:i}}l(PA,"computeExpand");function $A(r,e,t,n,s,o){let i=r.length,a=t<0?i+t:t,u=r[a],p=n??0,c=s??u,h=o??1;p<0&&(p+=u),c<0&&(c+=u),p=Math.max(0,Math.min(p,u)),c=Math.max(0,Math.min(c,u));let f=Math.max(0,Math.ceil((c-p)/h)),d=p*e[a],m=[...r],g=[...e];return m[a]=f,g[a]=e[a]*h,{sizes:m,strides:g,offsetDelta:d}}l($A,"computeSlice");function FA(r,e,t){let n=r.length,s=t<0?n+1+t:t,o=[...r],i=[...e],a=s<n?r[s]*e[s]:1;return o.splice(s,0,1),i.splice(s,0,a),{sizes:o,strides:i}}l(FA,"computeUnsqueeze");function zA(r,e,t){if(t!=null){let o=r.length,i=t<0?o+t:t;if(r[i]!==1)return{sizes:[...r],strides:[...e]};let a=[...r],u=[...e];return a.splice(i,1),u.splice(i,1),{sizes:a,strides:u}}let n=[],s=[];for(let o=0;o<r.length;o++)r[o]!==1&&(n.push(r[o]),s.push(e[o]));return{sizes:n,strides:s}}l(zA,"computeSqueeze");function VA(r,e,t,n,s){let o=r.length,i=t<0?o+t:t,a=[...r];a[i]=s;let u=n*e[i];return{sizes:a,strides:[...e],offsetDelta:u}}l(VA,"computeNarrow");function GA(r,e,t,n){let s=r.length,o=t<0?s+t:t,i=n<0?r[o]+n:n,a=[],u=[];for(let c=0;c<s;c++)c!==o&&(a.push(r[c]),u.push(e[c]));let p=i*e[o];return{sizes:a,strides:u,offsetDelta:p}}l(GA,"computeSelect");function Ms(r,e,t,n){let s=new Bt(r._impl.storage,r._impl.storageOffset+(n||0),e,t,r._impl.dtype,r._impl.device),o=r._impl.autogradMeta;return o&&s.setAutogradMeta(o),new We(s)}l(Ms,"makeView");function ZC(r,e){let t=Uv(r.shape,r.strides,e);if(!t)throw new Error(`Cannot reshape tensor of shape [${r.shape}] to [${e}]`);if(!t.needsCopy)return Ms(r,t.sizes,t.strides,0);let n=r.isContiguous?r:qA(r);return Ms(n,t.sizes,t.strides,0)}l(ZC,"reshapeKernel");function QC(r,e,t){let{sizes:n,strides:s}=CA(r.shape,r.strides,e,t);return Ms(r,n,s,0)}l(QC,"transposeKernel");function XC(r,e){let{sizes:t,strides:n}=DA(r.shape,r.strides,e);return Ms(r,t,n,0)}l(XC,"permuteKernel");function eD(r,e){let{sizes:t,strides:n}=PA(r.shape,r.strides,e);return Ms(r,t,n,0)}l(eD,"expandKernel");function tD(r,e,t){let n=r.shape,s=r.strides,o=[...e],i=new Array(e.length).fill(0);for(let a=0;a<t.length;a++){let u=t[a];i[u]=n[a]===1&&e[u]!==1?0:s[a]}return Ms(r,o,i,0)}l(tD,"broadcastInDimKernel");function nD(r,e,t,n,s){let{sizes:o,strides:i,offsetDelta:a}=$A(r.shape,r.strides,e,t,n,s);return Ms(r,o,i,a)}l(nD,"sliceKernel");function rD(r,e){let{sizes:t,strides:n}=FA(r.shape,r.strides,e);return Ms(r,t,n,0)}l(rD,"unsqueezeKernel");function sD(r,e){let{sizes:t,strides:n}=zA(r.shape,r.strides,e);return Ms(r,t,n,0)}l(sD,"squeezeKernel");function oD(r,e,t,n){let{sizes:s,strides:o,offsetDelta:i}=VA(r.shape,r.strides,e,t,n);return Ms(r,s,o,i)}l(oD,"narrowKernel");function iD(r,e,t){let{sizes:n,strides:s,offsetDelta:o}=GA(r.shape,r.strides,e,t);return Ms(r,n,s,o)}l(iD,"selectKernel");var jA=null;function qA(r){let e=r._impl;return r.isContiguous&&e.storageOffset===0&&e.storage.rawData.length===r.numel?r:aD(r)}l(qA,"contiguousKernel");var UA=Object.freeze({reshape:l((r,e,t)=>ZC(e,t),"reshape"),transpose:l((r,e,t,n)=>QC(e,t,n),"transpose"),permute:l((r,e,t)=>XC(e,t),"permute"),broadcast_in_dim:l((r,e,t,n)=>tD(e,t,n),"broadcast_in_dim"),expand:l((r,e,t)=>eD(e,t),"expand"),slice:l((r,e,t,n,s,o)=>nD(e,t,n,s,o),"slice"),unsqueeze:l((r,e,t)=>rD(e,t),"unsqueeze"),squeeze:l((r,e,t)=>sD(e,t),"squeeze"),narrow:l((r,e,t,n,s)=>oD(e,t,n,s),"narrow"),select:l((r,e,t,n)=>iD(e,t,n),"select"),contiguous:l((r,e)=>qA(e),"contiguous")});function aD(r){if(jA){let g=jA(r);if(g)return g}let e=r.shape,t=r.strides,n=r._impl.storage.data,s=r._impl.storageOffset,o=r.numel,i=xn(e),a=r._impl.dtype,u=En.allocate(o*Xi(a),a,r._impl.device),p=u.data,c=e.length,h=new Int32Array(c),f=s;for(let g=0;g<o;g++){p[g]=n[f];for(let y=c-1;y>=0&&(h[y]++,f+=t[y],!(h[y]<e[y]));y--)f-=h[y]*t[y],h[y]=0}let d=new Bt(u,0,e,i,r._impl.dtype,r._impl.device),m=r._impl.autogradMeta;return m&&d.setAutogradMeta(m),new We(d)}l(aD,"copyContiguous");be();_t();qr();jr();function uD(r,e,t,n){let s=$e(r.shape,{dtype:r.dtype}),o=ra(s,e,t,n),i=ra(s,e,t,Ht(n.shape,{dtype:r.dtype}));return qn(Mu(i,$e(i.shape,{dtype:i.dtype})),o,r)}l(uD,"scatterKernel");function lD(r,e){let t=r.shape,n=t.length;if(e.length<n)throw new Error("repeat: reps length must be >= tensor rank");let s=e.length-n,o=s>0?[...Array(s).fill(1),...t]:t.slice(),i=[],a=[],u=[];for(let p=0;p<o.length;p++)i.push(1,o[p]),a.push(e[p],o[p]),u.push(e[p]*o[p]);return Cn(Eu(Cn(r,i),a),u)}l(lD,"repeatKernel");function cD(r,e){let t=r.shape.length,n=e.length<t?[...Array(t-e.length).fill(1),...e]:e;return Cu(r,n)}l(cD,"tileKernel");function pD(r,e,t){let n=[],s=r.shape.length,o=t<0?s+t:t,i=0;for(let a of e)n.push(Un(r,o,i,a)),i+=a;return n}l(pD,"splitKernel");function fD(r,e,t){let n=r.shape.length,s=t<0?n+t:t,o=r.shape[s],i=Math.ceil(o/e);return Bo(r,i,s)}l(fD,"chunkKernel");function hD(r,e,t=0){let n=r.shape.length,s=vs(t,n),o=r.shape[s],i=(e%o+o)%o;return i===0?r:sa([Un(r,s,o-i,i),Un(r,s,0,o-i)],s)}l(hD,"rollKernel");function dD(r,e){let t=Array.isArray(e)?e:[e],n=r.shape.length,s=r;for(let o of t){let i=vs(o,n),a=s.shape[i],u=new Array(a);for(let p=0;p<a;p++)u[p]=a-1-p;s=li(s,i,Pt(u,{dtype:"i32"}))}return s}l(dD,"flipKernel");function mD(r,e=0){let t=r.shape.length,n=vs(e,t),s=r.shape[n],o=r;for(let i=1;i<s;i*=2){let a=[...o.shape];a[n]=i;let u=sa([$e(a,{dtype:o.dtype}),Un(o,n,0,s-i)],n);o=ae(o,u)}return o}l(mD,"cumsumKernel");function gD(r){let e=1;for(;e<r;)e<<=1;return e}l(gD,"nextPow2");function Rm(r,e,t){let n=r.shape.length,s=n-1,o=r.shape[s],i=gD(o),a=e?-1/0:1/0,u=r;if(i>o){let h=new Array(n).fill(0),f=new Array(n).fill(0);f[s]=i-o,u=na(r,h,f,a)}let p=new Array(n).fill(1);p[s]=i;let c=null;if(t){let h=new Array(i);for(let f=0;f<i;f++)h[f]=f;c=Cn(Pt(h,{dtype:"i32"}),p)}for(let h=2;h<=i;h<<=1)for(let f=h>>1;f>=1;f>>=1){let d=new Array(i),m=new Array(i);for(let x=0;x<i;x++){let S=x^f;d[x]=S;let T=(x&h)===0,k=e?!T:T;S>x?m[x]=k?1:0:m[x]=k?0:1}let g=Pt(d,{dtype:"i32"}),y=li(u,s,g),_=Ed(u,y),b=ai(u,y),w=Cn(Pt(m,{dtype:"f32"}),p);if(t){let x=li(c,s,g),S=qn(w,Pc(_,u),Pc(b,u));c=qn(S,c,x)}u=qn(w,_,b)}return i>o&&(u=Un(u,s,0,o),t&&(c=Un(c,s,0,o))),t?{values:u,indices:c}:u}l(Rm,"bitonicLastDim");function yD(r,e=-1,t=!1){let n=r.shape.length,s=vs(e,n);if(s===n-1)return Rm(r,t,!1);let o=gt(r,s,n-1),i=Rm(o,t,!1);return gt(i,s,n-1)}l(yD,"sortKernel");function JA(r,e,t){let n=r.shape.length,s=vs(e,n);if(s===n-1)return Rm(r,t,!0);let o=gt(r,s,n-1),i=Rm(o,t,!0);return{values:gt(i.values,s,n-1),indices:gt(i.indices,s,n-1)}}l(JA,"sortWithIndices");function _D(r,e=-1,t=!1){return JA(r,e,t).indices}l(_D,"argsortKernel");function bD(r,e,t=-1,n=!0){let s=r.shape.length,o=vs(t,s),{values:i,indices:a}=JA(r,o,n);return[Un(i,o,0,e),Un(a,o,0,e)]}l(bD,"topkKernel");var ZA=Object.freeze({scatter:l((r,e,t,n,s)=>uD(e,t,n,s),"scatter"),repeat:l((r,e,t)=>lD(e,t),"repeat"),tile:l((r,e,t)=>cD(e,t),"tile"),split:l((r,e,t,n)=>pD(e,t,n),"split"),chunk:l((r,e,t,n)=>fD(e,t,n),"chunk"),roll:l((r,e,t,n)=>hD(e,t,n),"roll"),flip:l((r,e,t)=>dD(e,t),"flip"),cumsum:l((r,e,t)=>mD(e,t),"cumsum"),sort:l((r,e,t,n)=>yD(e,t,n),"sort"),argsort:l((r,e,t,n)=>_D(e,t,n),"argsort"),topk:l((r,e,t,n,s)=>bD(e,t,n,s),"topk")});var wD=3,QA=!1,TD=[0,1,2,3,5];function XA(){if(QA)return;QA=!0;let r=new Xt("mlc","DEF");for(let t of kA())r.def(t);let e=new Xt("mlc","IMPL");for(let[t,n]of Object.entries(LA))e.impl(t,wD,n);for(let[t,n]of Object.entries({...UA,...ZA}))for(let s of TD)e.impl(t,s,n);y0()}l(XA,"registerNativeOps");gs();Nn();gs();var xD=["svd(Tensor input) -> (Tensor, Tensor, Tensor)","eigh(Tensor input) -> (Tensor, Tensor)","cholesky(Tensor input) -> Tensor","inv(Tensor input) -> Tensor","pinv(Tensor input) -> Tensor","det(Tensor input) -> Tensor","cov(Tensor input) -> Tensor","solve(Tensor a, Tensor b) -> Tensor","lstsq(Tensor a, Tensor b) -> Tensor"],e1=!1;function Mm(){if(e1)return;e1=!0;let r=new Xt("mlc","DEF");for(let e of xD)r.def(e)}l(Mm,"ensureLinalgSchemas");qr();Kn();ar();var SD=new Set(["cpu","wasm"]);function w0(r){let e=new Float64Array(r.length);for(let t=0;t<r.length;t++)e[t]=Number(r[t]);return e}l(w0,"float64From");function t1(r){if(!SD.has(r.device.type))throw new Error(`linalg/ml: host-readable device required (cpu or wasm), got '${r.device.type}'`)}l(t1,"requireHost");function Ue(r){if(r.ndim!==2)throw new Error(`linalg/ml: expected a 2-D matrix, got ${r.ndim}-D`);t1(r);let[e,t]=r.shape;return{data:w0(Je(r)),rows:e,cols:t}}l(Ue,"hostMatrix");function Rt(r){if(t1(r),r.ndim===1)return{data:w0(Je(r)),rows:r.shape[0],cols:1,wasVector:!0};if(r.ndim===2)return{data:w0(Je(r)),rows:r.shape[0],cols:r.shape[1],wasVector:!1};throw new Error(`linalg/ml: expected a 1-D or 2-D right-hand side, got ${r.ndim}-D`)}l(Rt,"hostColumns");function Q(r,e,t,n=ks){return Pt(r,{shape:e,dtype:t,device:n})}l(Q,"toHostTensor");qr();function n1(r,e){let t=Float64Array.from(r),n=new Int32Array(e);for(let o=0;o<e;o++)n[o]=o;let s=1;for(let o=0;o<e;o++){let i=o,a=Math.abs(t[o*e+o]);for(let p=o+1;p<e;p++){let c=Math.abs(t[p*e+o]);c>a&&(a=c,i=p)}if(a===0)throw new Error("linalg: matrix is singular");if(i!==o){for(let c=0;c<e;c++){let h=t[o*e+c];t[o*e+c]=t[i*e+c],t[i*e+c]=h}let p=n[o];n[o]=n[i],n[i]=p,s=-s}let u=t[o*e+o];for(let p=o+1;p<e;p++){let c=t[p*e+o]/u;t[p*e+o]=c;for(let h=o+1;h<e;h++)t[p*e+h]-=c*t[o*e+h]}}return{lu:t,piv:n,sign:s}}l(n1,"luFactor");function kD(r,e,t,n,s){let o=new Float64Array(t*s);for(let i=0;i<t;i++)for(let a=0;a<s;a++)o[i*s+a]=n[e[i]*s+a];for(let i=0;i<t;i++)for(let a=0;a<s;a++){let u=o[i*s+a];for(let p=0;p<i;p++)u-=r[i*t+p]*o[p*s+a];o[i*s+a]=u}for(let i=t-1;i>=0;i--){let a=r[i*t+i];for(let u=0;u<s;u++){let p=o[i*s+u];for(let c=i+1;c<t;c++)p-=r[i*t+c]*o[c*s+u];o[i*s+u]=p/a}}return o}l(kD,"luSolve");function T0(r,e,t,n){let{lu:s,piv:o}=n1(r,e);return kD(s,o,e,t,n)}l(T0,"solveHost");function r1(r,e){let t;try{t=n1(r,e)}catch{return 0}let n=t.sign;for(let s=0;s<e;s++)n*=t.lu[s*e+s];return n}l(r1,"detHost");function s1(r,e){let t=new Float64Array(e*e);for(let n=0;n<e;n++)for(let s=0;s<=n;s++){let o=r[n*e+s];for(let i=0;i<s;i++)o-=t[n*e+i]*t[s*e+i];if(n===s){if(o<=0)throw new Error("linalg.cholesky: matrix is not positive definite");t[n*e+s]=Math.sqrt(o)}else t[n*e+s]=o/t[s*e+s]}return t}l(s1,"choleskyHost");function Xc(r,e,t){let n=t?.tol??1e-12,s=t?.maxSweeps??100,o=Float64Array.from(r),i=new Float64Array(e*e);for(let c=0;c<e;c++)i[c*e+c]=1;for(let c=0;c<s;c++){let h=0;for(let f=0;f<e;f++)for(let d=f+1;d<e;d++)h+=o[f*e+d]*o[f*e+d];if(Math.sqrt(h)<n)break;for(let f=0;f<e;f++)for(let d=f+1;d<e;d++){let m=o[f*e+d];if(Math.abs(m)<1e-300)continue;let g=(o[d*e+d]-o[f*e+f])/(2*m),y=Math.sign(g||1)/(Math.abs(g)+Math.sqrt(g*g+1)),_=1/Math.sqrt(y*y+1),b=y*_;for(let w=0;w<e;w++){let x=o[w*e+f],S=o[w*e+d];o[w*e+f]=_*x-b*S,o[w*e+d]=b*x+_*S}for(let w=0;w<e;w++){let x=o[f*e+w],S=o[d*e+w];o[f*e+w]=_*x-b*S,o[d*e+w]=b*x+_*S}for(let w=0;w<e;w++){let x=i[w*e+f],S=i[w*e+d];i[w*e+f]=_*x-b*S,i[w*e+d]=b*x+_*S}}}let a=Array.from({length:e},(c,h)=>h).sort((c,h)=>o[c*e+c]-o[h*e+h]),u=new Float64Array(e),p=new Float64Array(e*e);for(let c=0;c<e;c++){let h=a[c];u[c]=o[h*e+h];for(let f=0;f<e;f++)p[f*e+c]=i[f*e+h]}return{values:u,vectors:p}}l(Xc,"eighHost");function AD(r,e,t,n){if(n){let o=new Float64Array(t*t);for(let i=0;i<t;i++)for(let a=i;a<t;a++){let u=0;for(let p=0;p<e;p++)u+=r[p*t+i]*r[p*t+a];o[i*t+a]=u,o[a*t+i]=u}return o}let s=new Float64Array(e*e);for(let o=0;o<e;o++)for(let i=o;i<e;i++){let a=0;for(let u=0;u<t;u++)a+=r[o*t+u]*r[i*t+u];s[o*e+i]=a,s[i*e+o]=a}return s}l(AD,"gram");function OD(r,e,t,n,s){let o=new Float64Array(e*s);for(let i=0;i<e;i++)for(let a=0;a<s;a++){let u=0;for(let p=0;p<t;p++)u+=r[i*t+p]*n[a*t+p];o[i*s+a]=u}return o}l(OD,"matmulRows");function da(r,e,t,n,s=AD,o=OD){let i=n?.tol??1e-12,a=Math.min(e,t),u=new Float64Array(e*a),p=new Float64Array(a),c=new Float64Array(t*a);if(t<=e){let{values:h,vectors:f}=Xc(s(r,e,t,!0),t,n),d=new Float64Array(a*t);for(let g=0;g<a;g++){let y=t-1-g;p[g]=Math.sqrt(Math.max(h[y],0));for(let _=0;_<t;_++){let b=f[_*t+y];c[_*a+g]=b,d[g*t+_]=b}}let m=o(r,e,t,d,a);for(let g=0;g<a;g++)if(p[g]>i)for(let y=0;y<e;y++)u[y*a+g]=m[y*a+g]/p[g]}else{let{values:h,vectors:f}=Xc(s(r,e,t,!1),e,n),d=new Float64Array(a*e);for(let y=0;y<a;y++){let _=e-1-y;p[y]=Math.sqrt(Math.max(h[_],0));for(let b=0;b<e;b++){let w=f[b*e+_];u[b*a+y]=w,d[y*e+b]=w}}let m=new Float64Array(t*e);for(let y=0;y<e;y++)for(let _=0;_<t;_++)m[_*e+y]=r[y*t+_];let g=o(m,t,e,d,a);for(let y=0;y<a;y++)if(p[y]>i)for(let _=0;_<t;_++)c[_*a+y]=g[_*a+y]/p[y]}return{U:u,S:p,V:c,k:a}}l(da,"svdHost");function i1(r,e,t,n,s,o){let i=o?.rcond??1e-12,{U:a,S:u,V:p,k:c}=da(r,e,t,o),h=i*(u.length?u[0]:0),f=new Float64Array(c*s);for(let m=0;m<c;m++){let g=u[m];for(let y=0;y<s;y++){let _=0;for(let b=0;b<e;b++)_+=a[b*c+m]*n[b*s+y];f[m*s+y]=g>h?_/g:0}}let d=new Float64Array(t*s);for(let m=0;m<t;m++)for(let g=0;g<s;g++){let y=0;for(let _=0;_<c;_++)y+=p[m*c+_]*f[_*s+g];d[m*s+g]=y}return d}l(i1,"lstsqHost");function a1(r,e,t,n){let s=n?.rcond??1e-12,{U:o,S:i,V:a,k:u}=da(r,e,t,n),p=s*(i.length?i[0]:0),c=new Float64Array(t*e);for(let h=0;h<t;h++)for(let f=0;f<e;f++){let d=0;for(let m=0;m<u;m++){let g=i[m];g>p&&(d+=a[h*u+m]*(1/g)*o[f*u+m])}c[h*e+f]=d}return c}l(a1,"pinvHost");function u1(r,e,t){let n=new Float64Array(t);for(let i=0;i<t;i++){let a=0;for(let u=0;u<e;u++)a+=r[u*t+i];n[i]=a/e}let s=e>1?e-1:1,o=new Float64Array(t*t);for(let i=0;i<t;i++)for(let a=i;a<t;a++){let u=0;for(let c=0;c<e;c++)u+=(r[c*t+i]-n[i])*(r[c*t+a]-n[a]);let p=u/s;o[i*t+a]=p,o[a*t+i]=p}return o}l(u1,"covHost");function ep(r,e){let t=Ue(r);if(t.rows!==t.cols)throw new Error(`linalg.${e}: matrix must be square`);return t}l(ep,"squareMatrix");function l1(r,e){let{data:t,rows:n}=ep(e,"cholesky");return Q(s1(t,n),[n,n],e.dtype,e.device)}l(l1,"cpuCholesky");function c1(r,e,t){let{data:n,rows:s}=ep(e,"solve"),o=Rt(t);if(o.rows!==s)throw new Error("linalg.solve: right-hand side rows must match matrix");let i=T0(n,s,o.data,o.cols);return Q(i,o.wasVector?[s]:[s,o.cols],e.dtype,e.device)}l(c1,"cpuSolve");function p1(r,e,t){let{data:n,rows:s,cols:o}=Ue(e),i=Rt(t);if(i.rows!==s)throw new Error("linalg.lstsq: right-hand side rows must match matrix");let a=i1(n,s,o,i.data,i.cols);return Q(a,i.wasVector?[o]:[o,i.cols],e.dtype,e.device)}l(p1,"cpuLstsq");function f1(r,e){let{data:t,rows:n}=ep(e,"inv"),s=new Float64Array(n*n);for(let i=0;i<n;i++)s[i*n+i]=1;let o=T0(t,n,s,n);return Q(o,[n,n],e.dtype,e.device)}l(f1,"cpuInv");function h1(r,e){let{data:t,rows:n,cols:s}=Ue(e);return Q(a1(t,n,s),[s,n],e.dtype,e.device)}l(h1,"cpuPinv");function d1(r,e){let{data:t,rows:n}=ep(e,"det");return Cc(r1(t,n),{dtype:e.dtype})}l(d1,"cpuDet");function m1(r,e){let{data:t,rows:n,cols:s}=Ue(e);return Q(u1(t,n,s),[s,s],e.dtype,e.device)}l(m1,"cpuCov");function g1(r,e){let{data:t,rows:n}=ep(e,"eigh"),{values:s,vectors:o}=Xc(t,n);return[Q(s,[n],e.dtype,e.device),Q(o,[n,n],e.dtype,e.device)]}l(g1,"cpuEigh");function Im(r,e){let{data:t,rows:n,cols:s}=Ue(e),{U:o,S:i,V:a,k:u}=da(t,n,s);return[Q(o,[n,u],e.dtype,e.device),Q(i,[u],e.dtype,e.device),Q(a,[s,u],e.dtype,e.device)]}l(Im,"cpuSvd");var BD={svd:Im,eigh:g1,cholesky:l1,inv:f1,pinv:h1,det:d1,cov:m1,solve:c1,lstsq:p1},RD=[0,2],y1=!1;function _1(){if(y1)return;y1=!0,Mm();let r=new Xt("mlc","IMPL");for(let[e,t]of Object.entries(BD))for(let n of RD)r.impl(e,n,t)}l(_1,"registerCpuLinalg");gs();Nn();gs();var MD=["kmeans(Tensor x, int n_clusters, int max_iter, int n_init, int seed) -> (Tensor, Tensor, Tensor)","kmeans_predict(Tensor x, Tensor centers) -> Tensor","knn_predict(Tensor x_train, Tensor y_train, Tensor x_query, int n_neighbors, bool classify) -> Tensor","gaussian_nb_fit(Tensor x, Tensor y) -> (Tensor, Tensor, Tensor, Tensor)","gaussian_nb_predict(Tensor x, Tensor means, Tensor variances, Tensor priors, Tensor classes) -> Tensor","elastic_net(Tensor x, Tensor y, float alpha, float l1_ratio, int max_iter, float tol, bool fit_intercept) -> (Tensor, Tensor)","decision_tree_fit(Tensor x, Tensor y, int max_depth, int min_split, int min_leaf, int max_features, bool classify, int seed) -> (Tensor, Tensor, Tensor, Tensor, Tensor)","decision_tree_predict(Tensor x, Tensor feature, Tensor threshold, Tensor left, Tensor right, Tensor value) -> Tensor"],b1=!1;function Em(){if(b1)return;b1=!0;let r=new Xt("mlc","DEF");for(let e of MD)r.def(e)}l(Em,"ensureMlSchemas");function cr(r){let e=(r??2654435769)>>>0;return l(function(){e|=0,e=e+1831565813|0;let n=Math.imul(e^e>>>15,1|e);return n=n+Math.imul(n^n>>>7,61|n)^n,((n^n>>>14)>>>0)/4294967296},"next")}l(cr,"makeRng");function Xu(r,e){return Math.floor(r()*e)}l(Xu,"randInt");function S0(r,e){let t=Array.from({length:r},(n,s)=>s);for(let n=r-1;n>0;n--){let s=Xu(e,n+1),o=t[n];t[n]=t[s],t[s]=o}return t}l(S0,"shuffledIndices");function k0(r,e,t,n,s){let o=0;for(let i=0;i<s;i++){let a=r[e*s+i]-t[n*s+i];o+=a*a}return o}l(k0,"sqDist");function v0(r,e,t,n,s){let o=new Float64Array(n*t),i=Xu(s,e);for(let u=0;u<t;u++)o[u]=r[i*t+u];let a=new Float64Array(e).fill(1/0);for(let u=1;u<n;u++){let p=0;for(let f=0;f<e;f++){let d=k0(r,f,o,u-1,t);d<a[f]&&(a[f]=d),p+=a[f]}let c=s()*p,h=e-1;for(let f=0;f<e;f++)if(c-=a[f],c<=0){h=f;break}for(let f=0;f<t;f++)o[u*t+f]=r[h*t+f]}return o}l(v0,"kmeansPlusPlus");function w1(r,e,t,n,s,o){let i=0;for(let a=0;a<e;a++){let u=0,p=1/0;for(let c=0;c<s;c++){let h=k0(r,a,n,c,t);h<p&&(p=h,u=c)}o[a]=u,i+=p}return i}l(w1,"assign");function N0(r,e,t,n,s,o){let i=new Float64Array(n*t),a=new Int32Array(n);for(let p=0;p<e;p++){let c=s[p];a[c]++;for(let h=0;h<t;h++)i[c*t+h]+=r[p*t+h]}let u=!1;for(let p=0;p<n;p++)if(a[p]!==0)for(let c=0;c<t;c++){let h=i[p*t+c]/a[p];h!==o[p*t+c]&&(u=!0),o[p*t+c]=h}return u}l(N0,"updateCenters");function ID(r,e,t,n,s,o){let i=v0(r,e,t,n,o),a=new Int32Array(e),u=1/0;for(let p=0;p<s&&(u=w1(r,e,t,i,n,a),!!N0(r,e,t,n,a,i));p++);return u=w1(r,e,t,i,n,a),{centers:i,labels:a,inertia:u}}l(ID,"lloyd");function Lm(r,e,t,n,s,o){let{data:i,rows:a,cols:u}=Ue(e),p=null;for(let h=0;h<s;h++){let f=cr(o+h*2654435769),d=ID(i,a,u,t,n,f);(!p||d.inertia<p.inertia)&&(p=d)}let c=new Float64Array(a);for(let h=0;h<a;h++)c[h]=p.labels[h];return[Q(p.centers,[t,u],e.dtype,e.device),Q(c,[a],e.dtype,e.device),Q(new Float64Array([p.inertia]),[1],e.dtype,e.device)]}l(Lm,"cpuKmeans");function Cm(r,e,t){let{data:n,rows:s,cols:o}=Ue(e),i=Ue(t),a=new Float64Array(s);for(let u=0;u<s;u++){let p=0,c=1/0;for(let h=0;h<i.rows;h++){let f=k0(n,u,i.data,h,o);f<c&&(c=f,p=h)}a[u]=p}return Q(a,[s],e.dtype,e.device)}l(Cm,"cpuKmeansPredict");function A0(r,e,t,n){for(let s=0;s<t;s++)e[s]=s;for(let s=0;s<n;s++){let o=s;for(let a=s+1;a<t;a++)r[e[a]]<r[e[o]]&&(o=a);let i=e[s];e[s]=e[o],e[o]=i}}l(A0,"topKSelect");function O0(r,e,t,n){if(n){let o=new Map,i=r[e[0]],a=0;for(let u=0;u<t;u++){let p=r[e[u]],c=(o.get(p)||0)+1;o.set(p,c),c>a&&(a=c,i=p)}return i}let s=0;for(let o=0;o<t;o++)s+=r[e[o]];return s/t}l(O0,"knnVote");function Dm(r,e,t,n,s,o){let i=Ue(e),a=Ue(n),u=Rt(t),p=i.cols,c=Math.min(s,i.rows),h=new Float64Array(a.rows),f=new Float64Array(i.rows),d=new Int32Array(i.rows);for(let m=0;m<a.rows;m++){for(let g=0;g<i.rows;g++){let y=0;for(let _=0;_<p;_++){let b=a.data[m*p+_]-i.data[g*p+_];y+=b*b}f[g]=y}A0(f,d,i.rows,c),h[m]=O0(u.data,d,c,o)}return Q(h,[a.rows],n.dtype,n.device)}l(Dm,"cpuKnnPredict");be();qr();function ns(r,e){let t=Pt(Int32Array.from(e),{shape:[e.length],dtype:"i32",device:r.device});return ot("index_select",r,t,0)}l(ns,"takeRows");function el(r){return Ue(r)}l(el,"matrixOf");function zt(r){let e=Rt(r);if(!e.wasVector&&e.cols!==1)throw new Error("ml: expected a 1-D target or single-column matrix");return{data:e.data,n:e.rows}}l(zt,"vectorOf");function tp(r,e,t,n){return Q(r,[e,t],n??"f32")}l(tp,"matrix");function B0(r,e,t){return Q(r,[e],t??"f32")}l(B0,"vector");function rs(r,e){let t=[],n=new Map,s=new Int32Array(e);for(let o=0;o<e;o++){let i=r[o],a=n.get(i);a===void 0&&(a=t.length,n.set(i,a),t.push(i)),s[o]=a}return{y:s,classes:t}}l(rs,"encodeLabels");var ED=1e-9;function T1(r,e,t){let{data:n,rows:s,cols:o}=Ue(e),i=Rt(t),{y:a,classes:u}=rs(i.data,i.rows),p=u.length,c=new Float64Array(p*o),h=new Float64Array(p*o),f=new Float64Array(p),d=new Int32Array(p);for(let _=0;_<s;_++){let b=a[_];d[b]++;for(let w=0;w<o;w++)c[b*o+w]+=n[_*o+w]}for(let _=0;_<p;_++){for(let b=0;b<o;b++)c[_*o+b]/=d[_];f[_]=d[_]/s}for(let _=0;_<s;_++){let b=a[_];for(let w=0;w<o;w++){let x=n[_*o+w]-c[b*o+w];h[b*o+w]+=x*x}}let m=0;for(let _=0;_<p;_++)for(let b=0;b<o;b++)h[_*o+b]/=d[_],h[_*o+b]>m&&(m=h[_*o+b]);let g=ED*m;for(let _=0;_<p*o;_++)h[_]+=g;let y=new Float64Array(p);for(let _=0;_<p;_++)y[_]=u[_];return[Q(c,[p,o],e.dtype,e.device),Q(h,[p,o],e.dtype,e.device),Q(f,[p],e.dtype,e.device),Q(y,[p],e.dtype,e.device)]}l(T1,"cpuGaussianNbFit");function x1(r,e,t,n,s,o){let{data:i,rows:a,cols:u}=Ue(e),p=Ue(t),c=Ue(n),h=Rt(s),f=Rt(o),d=p.rows,m=new Float64Array(a);for(let g=0;g<a;g++){let y=0,_=-1/0;for(let b=0;b<d;b++){let w=Math.log(h.data[b]);for(let x=0;x<u;x++){let S=c.data[b*u+x],T=i[g*u+x]-p.data[b*u+x];w+=-.5*(Math.log(2*Math.PI*S)+T*T/S)}w>_&&(_=w,y=b)}m[g]=f.data[y]}return Q(m,[a],e.dtype,e.device)}l(x1,"cpuGaussianNbPredict");function LD(r,e){return r>e?r-e:r<-e?r+e:0}l(LD,"softThreshold");function R0(r,e,t,n,s){let o=new Float64Array(t),i=0;if(s){for(let c=0;c<t;c++){let h=0;for(let f=0;f<e;f++)h+=r[f*t+c];o[c]=h/e}for(let c=0;c<e;c++)i+=n[c];i/=e}let a=new Float64Array(e*t);for(let c=0;c<e;c++)for(let h=0;h<t;h++)a[c*t+h]=r[c*t+h]-o[h];let u=new Float64Array(e);for(let c=0;c<e;c++)u[c]=n[c]-i;let p=new Float64Array(t);for(let c=0;c<t;c++){let h=0;for(let f=0;f<e;f++)h+=a[f*t+c]*a[f*t+c];p[c]=h/e}return{xc:a,r:u,z:p,meanX:o,meanY:i}}l(R0,"elasticNetPrep");function M0(r,e,t,n){let s=e;for(let o=0;o<n;o++)s-=r[o]*t[o];return s}l(M0,"elasticNetIntercept");function CD(r,e,t,n,s,o,i,a,u,p){for(let c=0;c<u;c++){let h=0;for(let f=0;f<t;f++){if(s[f]===0)continue;let d=0;for(let _=0;_<e;_++)d+=r[_*t+f]*n[_];let m=d/e+o[f]*s[f],g=LD(m,i)/(s[f]+a),y=g-o[f];if(y!==0){for(let _=0;_<e;_++)n[_]-=y*r[_*t+f];o[f]=g,Math.abs(y)>h&&(h=Math.abs(y))}}if(h<p)break}}l(CD,"coordinateDescent");function Pm(r,e,t,n,s,o,i,a){let{data:u,rows:p,cols:c}=Ue(e),h=Rt(t),{xc:f,r:d,z:m,meanX:g,meanY:y}=R0(u,p,c,h.data,a),_=new Float64Array(c),b=n*s,w=n*(1-s);CD(f,p,c,d,m,_,b,w,o,i);let x=M0(g,y,_,c);return[Q(_,[c],e.dtype,e.device),Q(new Float64Array([x]),[1],e.dtype,e.device)]}l(Pm,"cpuElasticNet");function DD(r,e,t){if(e<=0||e>=r)return Array.from({length:r},(s,o)=>o);let n=Array.from({length:r},(s,o)=>o);for(let s=r-1;s>r-1-e;s--){let o=Math.floor(t()*(s+1)),i=n[s];n[s]=n[o],n[o]=i}return n.slice(r-e)}l(DD,"chooseFeatures");function PD(r,e,t,n,s,o,i){let a=t.length,u=null;for(let p of n){let c=t.slice().sort((m,g)=>r[m*e+p]-r[g*e+p]),h=new Float64Array(s),f=new Float64Array(s);for(let m of c)f[o[m]]++;let d=0;for(let m=0;m<a-1;m++){let g=o[c[m]];h[g]++,f[g]--,d++;let y=a-d,_=r[c[m]*e+p],b=r[c[m+1]*e+p];if(_===b||d<i||y<i)continue;let w=1,x=1;for(let T=0;T<s;T++){let k=h[T]/d,A=f[T]/y;w-=k*k,x-=A*A}let S=(d*w+y*x)/a;(!u||S<u.impurity)&&(u={feature:p,threshold:(_+b)/2,impurity:S})}}return u}l(PD,"bestSplitClassify");function $D(r,e,t,n,s,o){let i=t.length,a=null;for(let u of n){let p=t.slice().sort((g,y)=>r[g*e+u]-r[y*e+u]),c=0,h=0;for(let g of p)c+=s[g],h+=s[g]*s[g];let f=0,d=0,m=0;for(let g=0;g<i-1;g++){let y=s[p[g]];f+=y,d+=y*y,m++;let _=i-m,b=r[p[g]*e+u],w=r[p[g+1]*e+u];if(b===w||m<o||_<o)continue;let x=d-f*f/m,S=c-f,k=h-d-S*S/_,A=x+k;(!a||A<a.impurity)&&(a={feature:u,threshold:(b+w)/2,impurity:A})}}return a}l($D,"bestSplitRegress");function S1(r,e,t,n,s,o,i,a,u){let p=Ue(e),c=p.data,h=p.cols,f=Rt(t),d=cr(u),m=null,g=null,y=0;if(a){let N=rs(f.data,f.rows);m=N.y,g=N.classes,y=g.length}let _=[],b=[],w=[],x=[],S=[],T=l(N=>{if(a){let O=new Float64Array(y);for(let B of N)O[m[B]]++;let R=0;for(let B=1;B<y;B++)O[B]>O[R]&&(R=B);return g[R]}let v=0;for(let O of N)v+=f.data[O];return v/N.length},"leafValue"),k=l(N=>{let v=a?m:f.data,O=v[N[0]];for(let R of N)if(v[R]!==O)return!1;return!0},"isPure"),A=l((N,v)=>{let O=_.length;if(_.push(-1),b.push(0),w.push(-1),x.push(-1),S.push(0),v>=n||N.length<s||k(N))return S[O]=T(N),O;let R=DD(h,i,d),B=a?PD(c,h,N,R,y,m,o):$D(c,h,N,R,f.data,o);if(!B)return S[O]=T(N),O;let E=[],z=[];for(let D of N)c[D*h+B.feature]<=B.threshold?E.push(D):z.push(D);return _[O]=B.feature,b[O]=B.threshold,w[O]=A(E,v+1),x[O]=A(z,v+1),O},"build");return A(Array.from({length:p.rows},(N,v)=>v),0),[Q(Float64Array.from(_),[_.length],e.dtype,e.device),Q(Float64Array.from(b),[b.length],e.dtype,e.device),Q(Float64Array.from(w),[w.length],e.dtype,e.device),Q(Float64Array.from(x),[x.length],e.dtype,e.device),Q(Float64Array.from(S),[S.length],e.dtype,e.device)]}l(S1,"cpuDecisionTreeFit");function k1(r,e,t,n,s,o,i){let a=Ue(e),u=Rt(t).data,p=Rt(n).data,c=Rt(s).data,h=Rt(o).data,f=Rt(i).data,d=new Float64Array(a.rows),m=a.cols;for(let g=0;g<a.rows;g++){let y=0;for(;u[y]!==-1;)a.data[g*m+u[y]]<=p[y]?y=c[y]:y=h[y];d[g]=f[y]}return Q(d,[a.rows],e.dtype,e.device)}l(k1,"cpuDecisionTreePredict");var FD={kmeans:Lm,kmeans_predict:Cm,knn_predict:Dm,gaussian_nb_fit:T1,gaussian_nb_predict:x1,elastic_net:Pm,decision_tree_fit:S1,decision_tree_predict:k1},zD=[0,2],VD=[1,5];function GD(r){return()=>{throw new Error(`ml.${r}: scalar-iterative algorithm runs on CPU/WASM only; GPU/WebGPU not supported (no performance benefit)`)}}l(GD,"unsupported");var v1=!1;function N1(){if(v1)return;v1=!0,Em();let r=new Xt("mlc","IMPL");for(let[e,t]of Object.entries(FD)){for(let n of zD)r.impl(e,n,t);for(let n of VD)r.impl(e,n,GD(e))}}l(N1,"registerCpuMl");gs();Nn();gs();var jD=["qr(Tensor input) -> (Tensor, Tensor)","fft(Tensor input) -> Tensor","ifft(Tensor input) -> Tensor"],A1=!1;function O1(){if(A1)return;A1=!0;let r=new Xt("mlc","DEF");for(let e of jD)r.def(e)}l(O1,"ensureNumericSchemas");Kn();function B1(r,e,t){let n=Math.min(e,t),s=Float64Array.from(r),o=[];for(let u=0;u<n;u++){let p=0;for(let d=u;d<e;d++)p+=s[d*t+u]*s[d*t+u];if(p=Math.sqrt(p),p===0){o.push(null);continue}let c=s[u*t+u]>=0?-p:p,h=new Float64Array(e-u);h[0]=s[u*t+u]-c;for(let d=u+1;d<e;d++)h[d-u]=s[d*t+u];let f=0;for(let d=0;d<h.length;d++)f+=h[d]*h[d];if(f===0){o.push(null);continue}o.push(h);for(let d=u;d<t;d++){let m=0;for(let y=0;y<h.length;y++)m+=h[y]*s[(u+y)*t+d];let g=2*m/f;for(let y=0;y<h.length;y++)s[(u+y)*t+d]-=g*h[y]}}let i=new Float64Array(e*n);for(let u=0;u<n;u++)i[u*n+u]=1;for(let u=n-1;u>=0;u--){let p=o[u];if(!p)continue;let c=0;for(let h=0;h<p.length;h++)c+=p[h]*p[h];for(let h=0;h<n;h++){let f=0;for(let m=0;m<p.length;m++)f+=p[m]*i[(u+m)*n+h];let d=2*f/c;for(let m=0;m<p.length;m++)i[(u+m)*n+h]-=d*p[m]}}let a=new Float64Array(n*t);for(let u=0;u<n;u++)for(let p=u;p<t;p++)a[u*t+p]=s[u*t+p];return{Q:i,R:a,k:n}}l(B1,"qrHost");function qD(r,e){let t=r.length;for(let n=1,s=0;n<t;n++){let o=t>>1;for(;s&o;o>>=1)s^=o;s^=o,n<s&&([r[n],r[s]]=[r[s],r[n]],[e[n],e[s]]=[e[s],e[n]])}}l(qD,"bitReversePermute");function $m(r,e,t){let n=r.length;qD(r,e);for(let s=2;s<=n;s<<=1){let o=(t?1:-1)*2*Math.PI/s,i=Math.cos(o),a=Math.sin(o);for(let u=0;u<n;u+=s){let p=1,c=0,h=s>>1;for(let f=0;f<h;f++){let d=r[u+f],m=e[u+f],g=r[u+f+h]*p-e[u+f+h]*c,y=r[u+f+h]*c+e[u+f+h]*p;r[u+f]=d+g,e[u+f]=m+y,r[u+f+h]=d-g,e[u+f+h]=m-y;let _=p*i-c*a;c=p*a+c*i,p=_}}}if(t)for(let s=0;s<n;s++)r[s]/=n,e[s]/=n}l($m,"fftRadix2");function UD(r){let e=1;for(;e<r;)e<<=1;return e}l(UD,"nextPow2");function KD(r,e,t){let n=r.length,s=UD(2*n-1),o=t?1:-1,i=new Float64Array(n),a=new Float64Array(n);for(let f=0;f<n;f++){let d=f*f%(2*n),m=o*Math.PI*d/n;i[f]=Math.cos(m),a[f]=Math.sin(m)}let u=new Float64Array(s),p=new Float64Array(s);for(let f=0;f<n;f++)u[f]=r[f]*i[f]-e[f]*a[f],p[f]=r[f]*a[f]+e[f]*i[f];let c=new Float64Array(s),h=new Float64Array(s);c[0]=i[0],h[0]=-a[0];for(let f=1;f<n;f++)c[f]=i[f],h[f]=-a[f],c[s-f]=i[f],h[s-f]=-a[f];$m(u,p,!1),$m(c,h,!1);for(let f=0;f<s;f++){let d=u[f]*c[f]-p[f]*h[f];p[f]=u[f]*h[f]+p[f]*c[f],u[f]=d}$m(u,p,!0);for(let f=0;f<n;f++)r[f]=u[f]*i[f]-p[f]*a[f],e[f]=u[f]*a[f]+p[f]*i[f];if(t)for(let f=0;f<n;f++)r[f]/=n,e[f]/=n}l(KD,"bluestein");function R1(r,e,t){let n=r.length;n!==0&&((n&n-1)===0?$m(r,e,t):KD(r,e,t))}l(R1,"fftHost");function M1(r,e){let{data:t,rows:n,cols:s}=Ue(e),{Q:o,R:i,k:a}=B1(t,n,s);return[Q(o,[n,a],e.dtype,e.device),Q(i,[a,s],e.dtype,e.device)]}l(M1,"cpuQr");function WD(r){let e=Je(r);if(r.ndim===1){let t=r.shape[0];return{re:Float64Array.from(e),im:new Float64Array(t),n:t}}if(r.ndim===2&&r.shape[1]===2){let t=r.shape[0],n=new Float64Array(t),s=new Float64Array(t);for(let o=0;o<t;o++)n[o]=e[2*o],s[o]=e[2*o+1];return{re:n,im:s,n:t}}throw new Error(`fft: expected a 1-D real signal or [n, 2] complex tensor, got shape [${r.shape}]`)}l(WD,"splitComplex");function HD(r,e,t){let n=new Float64Array(2*t);for(let s=0;s<t;s++)n[2*s]=r[s],n[2*s+1]=e[s];return n}l(HD,"interleave");function I1(r,e){let{re:t,im:n,n:s}=WD(r);return R1(t,n,e),Q(HD(t,n,s),[s,2],r.dtype,r.device)}l(I1,"transform");function E1(r,e){return I1(e,!1)}l(E1,"cpuFft");function L1(r,e){return I1(e,!0)}l(L1,"cpuIfft");var YD={qr:M1,fft:E1,ifft:L1},JD=[0,2],C1=!1;function D1(){if(C1)return;C1=!0,O1();let r=new Xt("mlc","IMPL");for(let[e,t]of Object.entries(YD))for(let n of JD)r.impl(e,n,t)}l(D1,"registerCpuNumeric");gs();Nn();nw();var it=8,E0=4,tl=2,ZD=tl*it,ma=Math.log2(tl),QD=65536,P1=ZD,XD=64*1024*1024;function $1(r,e){return Math.ceil(r/e)*e}l($1,"alignUp");var I0=class{static{l(this,"SimdModule")}constructor(e,t){this._module=e,this._exportName=t,this._instantiate()}_instantiate(){this._instance=new WebAssembly.Instance(this._module,{}),this._memory=this._instance.exports.memory,this._fn=this._instance.exports[this._exportName],this._top=0}reset(){this._top=0}reclaim(){this._memory.buffer.byteLength>XD?this._instantiate():this._top=0}_ensure(e){let t=this._memory.buffer.byteLength;e<=t||this._memory.grow(Math.ceil((e-t)/QD))}alloc(e){let t=$1(this._top,P1);return this._top=t+$1(e,P1),this._ensure(this._top),t}allocF64(e){return this.alloc(e*it)}allocI32(e){return this.alloc(e*E0)}f64(e,t){return new Float64Array(this._memory.buffer,e,t)}i32(e,t){return new Int32Array(this._memory.buffer,e,t)}writeF64(e,t){this.f64(e,t.length).set(t)}run(...e){this._fn(...e)}},F1=new Map;function so(r,e,t){let n=F1.get(r);return n||(n=new I0(new WebAssembly.Module(kd(e())),t),F1.set(r,n)),n.reclaim(),n}l(so,"simdModule");var eP=1e308,tP=1,nP=32768;function np(r,e,t,n){let s=e.map(([i,a])=>`(param $${i} ${a})`).join(" "),o=t.map(([i,a])=>`(local $${i} ${a})`).join(" ");return`(module (memory (export "memory") ${tP} ${nP})