doppler-gpu 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +45 -29
  2. package/assets/doppler-runtime-map.svg +82 -0
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/client/runtime/model-session.d.ts +1 -0
  11. package/src/client/runtime/model-session.js +9 -0
  12. package/src/config/conversion/README.md +4 -0
  13. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  14. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  26. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  29. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  30. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  32. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  33. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  34. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  36. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  39. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  41. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  42. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  43. package/src/config/execution-contract-check.js +14 -10
  44. package/src/config/merge-contract-check.js +1 -0
  45. package/src/config/merge.js +30 -0
  46. package/src/config/param-validator.d.ts +5 -0
  47. package/src/config/param-validator.js +42 -8
  48. package/src/config/required-inference-fields-contract-check.js +14 -0
  49. package/src/config/schema/debug.schema.d.ts +2 -0
  50. package/src/config/schema/debug.schema.js +1 -0
  51. package/src/config/schema/doppler.schema.d.ts +5 -1
  52. package/src/config/schema/doppler.schema.js +82 -292
  53. package/src/config/schema/inference.schema.d.ts +17 -3
  54. package/src/config/schema/kvcache.schema.d.ts +19 -3
  55. package/src/config/schema/kvcache.schema.js +2 -1
  56. package/src/config/schema/manifest.schema.d.ts +38 -1
  57. package/src/config/schema/manifest.schema.js +18 -0
  58. package/src/config/schema/policy-schema-registry.json +5 -0
  59. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  60. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  61. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  62. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  63. package/src/config/transforms/execution-graph-transforms.js +8 -0
  64. package/src/converter/conversion-plan.js +2 -0
  65. package/src/converter/parsers/transformer.js +27 -1
  66. package/src/converter/quantization-info.js +16 -4
  67. package/src/converter/rope-config.js +55 -3
  68. package/src/experimental/training/distillation/student-fixture.js +1 -1
  69. package/src/experimental/training/lora-pipeline.js +1 -1
  70. package/src/experimental/training/lora.js +2 -2
  71. package/src/experimental/training/suite.js +1 -1
  72. package/src/gpu/device.js +54 -11
  73. package/src/gpu/kernels/README.md +8 -1
  74. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  75. package/src/gpu/kernels/dequant.d.ts +6 -6
  76. package/src/gpu/kernels/dequant.js +17 -6
  77. package/src/gpu/kernels/linear-attention-core.js +1 -6
  78. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  79. package/src/gpu/kernels/matmul-selection.js +9 -2
  80. package/src/gpu/kernels/matmul.d.ts +1 -1
  81. package/src/gpu/kernels/matmul.js +63 -9
  82. package/src/gpu/kernels/moe.d.ts +5 -1
  83. package/src/gpu/kernels/moe.js +5 -2
  84. package/src/gpu/kernels/softmax.d.ts +3 -3
  85. package/src/gpu/kernels/softmax.js +9 -2
  86. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  87. package/src/gpu/kernels/types.d.ts +1 -1
  88. package/src/gpu/profiler.js +11 -1
  89. package/src/inference/browser-harness-model-helpers.js +4 -1
  90. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  91. package/src/inference/browser-harness-text-helpers.js +357 -116
  92. package/src/inference/browser-harness.d.ts +3 -3
  93. package/src/inference/browser-harness.js +266 -11
  94. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  95. package/src/inference/kv-cache/quantized.js +16 -2
  96. package/src/inference/kv-cache/tiered.js +50 -15
  97. package/src/inference/moe-router.js +38 -7
  98. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  99. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  100. package/src/inference/pipelines/text/config.d.ts +5 -1
  101. package/src/inference/pipelines/text/config.js +135 -112
  102. package/src/inference/pipelines/text/embed.d.ts +1 -0
  103. package/src/inference/pipelines/text/embed.js +39 -12
  104. package/src/inference/pipelines/text/execution-plan.js +8 -2
  105. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  106. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  107. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  108. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  109. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  110. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  111. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  112. package/src/inference/pipelines/text/generator-steps.js +3 -0
  113. package/src/inference/pipelines/text/generator.d.ts +1 -0
  114. package/src/inference/pipelines/text/generator.js +14 -1
  115. package/src/inference/pipelines/text/init-rope.js +54 -6
  116. package/src/inference/pipelines/text/init.js +63 -15
  117. package/src/inference/pipelines/text/layer.js +32 -5
  118. package/src/inference/pipelines/text/linear-attention.js +12 -8
  119. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  120. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  121. package/src/inference/pipelines/text/logits/index.js +44 -87
  122. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  123. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  124. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  125. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  126. package/src/inference/pipelines/text/logits/utils.js +11 -0
  127. package/src/inference/pipelines/text/lora-apply.js +6 -4
  128. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  129. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  130. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  131. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  132. package/src/inference/pipelines/text/ops.d.ts +3 -0
  133. package/src/inference/pipelines/text/ops.js +26 -1
  134. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  135. package/src/inference/pipelines/text/sampling.js +52 -10
  136. package/src/inference/pipelines/text/weights.js +7 -2
  137. package/src/inference/pipelines/text.js +24 -4
  138. package/src/inference/tokenizers/behavior-flags.js +23 -2
  139. package/src/loader/final-weights-loader.js +12 -1
  140. package/src/loader/tensors/tensor-loader.js +37 -3
  141. package/src/rules/execution-rules-contract-check.js +12 -3
  142. package/src/rules/inference/capability-transforms.rules.json +14 -2
  143. package/src/rules/inference/config.rules.json +26 -2
  144. package/src/rules/inference/dtype.rules.json +12 -5
  145. package/src/rules/inference/execution.rules.json +5 -19
  146. package/src/rules/inference/moe.rules.json +97 -0
  147. package/src/rules/loader/weights.rules.json +1 -0
  148. package/src/tooling/command-api-constants.js +2 -2
  149. package/src/tooling/command-api-family-normalizers.js +2 -1
  150. package/src/tooling/command-api.d.ts +1 -1
  151. package/src/tooling/command-runner-shared.d.ts +1 -1
  152. package/src/tooling/command-runner-shared.js +3 -1
  153. package/src/tooling/node-converter.js +20 -0
  154. package/src/tooling/node-source-runtime.js +5 -1
  155. package/src/version.js +1 -1
@@ -1,4 +1,5 @@
1
1
  import { getDevice } from '../../../gpu/device.js';
2
+ import { getWeightDtype, isWeightBuffer } from '../../../gpu/weight-buffer.js';
2
3
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
3
4
  import { createTensor } from '../../../gpu/tensor.js';
4
5
  import { runMatmul, runSiLU, runGeLU } from '../../../gpu/kernel-selector.js';
@@ -8,6 +9,20 @@ import { ensureExpertLoaded, gatherTokens } from './moe-helpers.js';
8
9
  import { selectRuleValue } from '../../../rules/rule-registry.js';
9
10
  import { runGptOssExpertCPU } from './moe-cpu-gptoss.js';
10
11
 
12
+ function requireMixtralExpertWeightDtype(weights, role, layerIdx, expertIdx) {
13
+ const tensor = weights?.[role];
14
+ if (isWeightBuffer(tensor)) {
15
+ return getWeightDtype(tensor);
16
+ }
17
+ const declared = weights?.[`${role}Dtype`] ?? weights?.weightsDtype ?? tensor?.dtype;
18
+ if (declared === 'f16' || declared === 'f32' || declared === 'q4k') {
19
+ return declared;
20
+ }
21
+ throw new Error(
22
+ `[MoE] Expert layer_${layerIdx}_expert_${expertIdx}.${role} requires explicit weight dtype.`
23
+ );
24
+ }
25
+
11
26
  export async function moeFeedForwardCPU(
12
27
  hiddenStates,
13
28
  numTokens,
@@ -72,6 +87,10 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
72
87
  return new Float32Array(input.length);
73
88
  }
74
89
 
90
+ const gateDtype = requireMixtralExpertWeightDtype(weights, 'gate', layerIdx, expertIdx);
91
+ const upDtype = requireMixtralExpertWeightDtype(weights, 'up', layerIdx, expertIdx);
92
+ const downDtype = requireMixtralExpertWeightDtype(weights, 'down', layerIdx, expertIdx);
93
+
75
94
  const inputBuffer = acquireBuffer(input.byteLength, undefined, 'expert_input');
76
95
  device.queue.writeBuffer(inputBuffer, 0, input);
77
96
  const inputTensor = createTensor(inputBuffer, 'f32', [numTokens, hiddenSize], 'expert_input');
@@ -84,12 +103,16 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
84
103
  transposeB: 'auto',
85
104
  role: 'moe_gate',
86
105
  kernelPath,
106
+ bDtype: gateDtype,
107
+ outputDtype: 'f32',
87
108
  });
88
109
 
89
110
  upOutput = await runMatmul(inputTensor, weights.up, numTokens, intermediateSize, hiddenSize, {
90
111
  transposeB: 'auto',
91
112
  role: 'moe_up',
92
113
  kernelPath,
114
+ bDtype: upDtype,
115
+ outputDtype: 'f32',
93
116
  });
94
117
 
95
118
  const activationFn = {
@@ -106,6 +129,8 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
106
129
  transposeB: 'auto',
107
130
  role: 'moe_down',
108
131
  kernelPath,
132
+ bDtype: downDtype,
133
+ outputDtype: 'f32',
109
134
  });
110
135
 
111
136
  const outputData = await readBuffer(output.buffer, input.byteLength);
@@ -28,9 +28,10 @@ import { selectRuleValue } from '../../../rules/rule-registry.js';
28
28
  import { getRuntimeConfig } from '../../../config/runtime.js';
29
29
  import {
30
30
  validateMoeShape,
31
+ resolveMoeExecutionProfile,
32
+ resolveMoeIntermediateSize,
31
33
  resolveMoeVendorProfile,
32
- resolveGptOssKernelPathProfile,
33
- resolveMixtralKernelPathProfile,
34
+ resolveMoeKernelPathProfile,
34
35
  } from './moe-shape-validator.js';
35
36
  import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
36
37
  import { QK_K, Q4K_BLOCK_BYTES } from '../../../config/schema/index.js';
@@ -128,6 +129,92 @@ function resolvePerExpertScaleBuffer(device, value) {
128
129
  return { buffer, ownedBuffer: null };
129
130
  }
130
131
 
132
+ const MOE_ROUTE_EXECUTORS = Object.freeze({
133
+ 'gemma4-route': runGemma4RouteExperts,
134
+ });
135
+
136
+ const MOE_EXPERT_EXECUTORS = Object.freeze({
137
+ 'gpt-oss': runGptOssProfileExpert,
138
+ 'gemma4-packed': runGemma4ProfileExpert,
139
+ mixtral: runMixtralProfileExpert,
140
+ });
141
+
142
+ function requireMoeExecutor(registry, id, label) {
143
+ const executor = registry[id];
144
+ if (typeof executor !== 'function') {
145
+ throw new Error(`[MoE] Unknown ${label} "${String(id)}".`);
146
+ }
147
+ return executor;
148
+ }
149
+
150
+ function assertMoeExpertWeights(moeProfile, weights, expertKey) {
151
+ if (moeProfile.expertExecutor === 'gemma4-packed' && !weights.gateUp) {
152
+ throw new Error(`[MoE] Missing Gemma-style packed weights for ${expertKey}`);
153
+ }
154
+ if (moeProfile.expertExecutor === 'gemma4-packed' && !weights.down) {
155
+ throw new Error(`[MoE] Missing Gemma-style packed weights for ${expertKey}`);
156
+ }
157
+ if (moeProfile.expertExecutor === 'mixtral' && (!weights.gate || !weights.up || !weights.down)) {
158
+ throw new Error(`[MoE] Missing Mixtral weights for ${expertKey}`);
159
+ }
160
+ }
161
+
162
+ async function runGptOssProfileExpert(args) {
163
+ return runGptOssExpert(
164
+ args.gathered,
165
+ args.expertOutputs,
166
+ args.weights,
167
+ args.layerIdx,
168
+ args.expertIdx,
169
+ args.count,
170
+ args.inputOffset,
171
+ args.outputOffset,
172
+ args.hiddenSize,
173
+ args.intermediateSize,
174
+ args.numExperts,
175
+ args.activationDtype,
176
+ args.swigluLimit,
177
+ args.kernelPath,
178
+ args.executionPolicies,
179
+ args.modelType,
180
+ args.vendorProfile,
181
+ args.moeKernelPathProfile
182
+ );
183
+ }
184
+
185
+ async function runGemma4ProfileExpert(args) {
186
+ return runGemma4Expert(
187
+ args.gathered,
188
+ args.expertOutputs,
189
+ args.weights,
190
+ args.count,
191
+ args.inputOffset,
192
+ args.outputOffset,
193
+ args.hiddenSize,
194
+ args.intermediateSize,
195
+ args.activationDtype,
196
+ args.swigluLimit,
197
+ args.kernelPath
198
+ );
199
+ }
200
+
201
+ async function runMixtralProfileExpert(args) {
202
+ return runMixtralExpert(
203
+ args.gathered,
204
+ args.expertOutputs,
205
+ args.weights,
206
+ args.count,
207
+ args.inputOffset,
208
+ args.outputOffset,
209
+ args.hiddenSize,
210
+ args.intermediateSize,
211
+ args.hiddenActivation,
212
+ args.activationDtype,
213
+ args.swigluLimit,
214
+ args.kernelPath
215
+ );
216
+ }
217
+
131
218
  export async function moeFeedForwardGPU(
132
219
  inputBuffer,
133
220
  numTokens,
@@ -143,9 +230,6 @@ export async function moeFeedForwardGPU(
143
230
 
144
231
  const { hiddenSize, numExperts, moeTopK, hiddenActivation } = config;
145
232
  const expertFormat = config.expertFormat;
146
- const intermediateSize = expertFormat === 'gemma4'
147
- ? config.expertIntermediateSize
148
- : config.intermediateSize;
149
233
  const swigluLimit = config.swigluLimit;
150
234
  const kernelPath = config.kernelPath ?? null;
151
235
  if (!expertFormat) {
@@ -154,9 +238,6 @@ export async function moeFeedForwardGPU(
154
238
  if (swigluLimit === undefined) {
155
239
  throw new Error('MoE swigluLimit must be explicitly set (null or number).');
156
240
  }
157
- if (!Number.isFinite(intermediateSize) || intermediateSize <= 0) {
158
- throw new Error(`[MoE] Invalid expert intermediate size for ${expertFormat}: ${String(intermediateSize)}.`);
159
- }
160
241
  const topK = moeTopK ?? moeRouter.topK;
161
242
  if (topK == null) {
162
243
  throw new Error('MoE topK is required in config.');
@@ -165,15 +246,17 @@ export async function moeFeedForwardGPU(
165
246
  throw new Error('MoE config.modelType is required; got null/undefined.');
166
247
  }
167
248
  const modelType = config.modelType;
249
+ const moeProfile = resolveMoeExecutionProfile(config, { modelType });
250
+ const intermediateSize = resolveMoeIntermediateSize(config, moeProfile);
168
251
  validateMoeShape(
169
252
  { hiddenSize, intermediateSize, moeTopK: topK, numExperts, expertFormat },
170
- { modelType }
253
+ { modelType, moeProfile }
171
254
  );
172
- const vendorProfile = resolveMoeVendorProfile(modelType);
255
+ const vendorProfile = resolveMoeVendorProfile(moeProfile);
173
256
  const caps = getKernelCapabilities();
174
- if (modelType === 'gpt-oss' && !caps.hasF16) {
257
+ if (moeProfile.requiresShaderF16 && !caps.hasF16) {
175
258
  throw new Error(
176
- '[MoE] GPT-OSS routing requires shader-f16 support. ' +
259
+ `[MoE] ${moeProfile.label} requires shader-f16 support. ` +
177
260
  `Adapter: ${caps.adapterInfo?.vendor ?? 'unknown'} ${caps.adapterInfo?.architecture ?? ''}`.trim()
178
261
  );
179
262
  }
@@ -224,19 +307,19 @@ export async function moeFeedForwardGPU(
224
307
  }
225
308
 
226
309
  try {
227
- const needsGemmaRouterScale = modelType === 'diffusion_gemma'
310
+ const needsRouterScale = moeProfile.routerScaleMode === 'required'
228
311
  || layerRouter?.scale != null
229
312
  || layerRouter?.perExpertScale != null;
230
313
  let routerInputTensor = routerSourceTensor;
231
- if (needsGemmaRouterScale) {
314
+ if (needsRouterScale) {
232
315
  if (!layerRouter?.scale) {
233
- throw new Error(`[MoE] DiffusionGemma router scale missing for layer ${layerIdx}.`);
316
+ throw new Error(`[MoE] ${moeProfile.label} router scale missing for layer ${layerIdx}.`);
234
317
  }
235
318
  if (!layerRouter?.perExpertScale) {
236
- throw new Error(`[MoE] DiffusionGemma per-expert router scale missing for layer ${layerIdx}.`);
319
+ throw new Error(`[MoE] ${moeProfile.label} per-expert router scale missing for layer ${layerIdx}.`);
237
320
  }
238
321
  if (!Number.isFinite(config.rmsNormEps) || config.rmsNormEps <= 0) {
239
- throw new Error(`[MoE] DiffusionGemma router RMSNorm eps is invalid: ${String(config.rmsNormEps)}.`);
322
+ throw new Error(`[MoE] ${moeProfile.label} router RMSNorm eps is invalid: ${String(config.rmsNormEps)}.`);
240
323
  }
241
324
  routerNormTensor = await runRMSNorm(
242
325
  inputTensor,
@@ -295,32 +378,16 @@ export async function moeFeedForwardGPU(
295
378
  trace.buffers(`MoE L${layerIdx} router_logits`, { min, max, nanCount, dtype: logitsDtype });
296
379
  }
297
380
 
298
- // Profile resolution: routerTopK/dequantExpert are resolved for tracing and
299
- // forward validation. Actual kernel dispatch uses the generic softmax.rules.json
300
- // topkVariant rules (keyed by modelType) and format-specific dequant paths.
301
- // GPT-OSS: dequantTileShape actively steers MXFP4 dequant; routerTopK is trace-only.
302
- // Mixtral: expert weights are pre-loaded (no runtime dequant); both fields are trace-only.
303
- let gptOssKernelPathProfile = null;
304
- let mixtralKernelPathProfile = null;
305
- if (modelType === 'gpt-oss') {
306
- gptOssKernelPathProfile = await resolveGptOssKernelPathProfile({
307
- hasF16: caps.hasF16,
308
- hasSubgroups: caps.hasSubgroups,
309
- routerDtype: logitsDtype,
310
- weightsDtype: activationDtype,
311
- outputDtype: activationDtype,
312
- groupSize: 32,
313
- tileShape: vendorProfile.dequantTileShape,
314
- });
315
- } else if (modelType === 'mixtral') {
316
- mixtralKernelPathProfile = await resolveMixtralKernelPathProfile({
317
- hasF16: caps.hasF16,
318
- hasSubgroups: caps.hasSubgroups,
319
- routerDtype: logitsDtype,
320
- weightsDtype: activationDtype,
321
- outputDtype: activationDtype,
322
- });
323
- }
381
+ const moeKernelPathProfile = await resolveMoeKernelPathProfile(moeProfile, {
382
+ hasF16: caps.hasF16,
383
+ hasSubgroups: caps.hasSubgroups,
384
+ routerDtype: logitsDtype,
385
+ inputDtype: logitsDtype,
386
+ weightsDtype: activationDtype,
387
+ outputDtype: activationDtype,
388
+ groupSize: 32,
389
+ tileShape: vendorProfile.dequantTileShape,
390
+ });
324
391
 
325
392
  stepStart = perfMark();
326
393
  ({ indices: indicesBuffer, weights: weightsBuffer } = await runSoftmaxTopK(
@@ -338,7 +405,7 @@ export async function moeFeedForwardGPU(
338
405
  perfLog(`MoE L${layerIdx} topk`, stepStart, {
339
406
  topK,
340
407
  modelType,
341
- routerTopKKernel: gptOssKernelPathProfile?.routerTopK ?? mixtralKernelPathProfile?.routerTopK ?? null,
408
+ routerTopKKernel: moeKernelPathProfile?.routerTopK ?? null,
342
409
  });
343
410
 
344
411
  if (isTraceEnabled('buffers')) {
@@ -389,28 +456,31 @@ export async function moeFeedForwardGPU(
389
456
 
390
457
  const activeExpertSelection = resolveMoEActiveExpertSelection();
391
458
  if (activeExpertSelection === 'topk-route') {
392
- if (expertFormat !== 'gemma4') {
393
- throw new Error(`[MoE] topk-route active expert selection requires gemma4 expert format, got ${expertFormat}.`);
459
+ if (moeProfile.topkRouteExecutor == null) {
460
+ throw new Error(`[MoE] topk-route active expert selection is not supported by profile "${moeProfile.id}".`);
394
461
  }
462
+ const routeExecutor = requireMoeExecutor(MOE_ROUTE_EXECUTORS, moeProfile.topkRouteExecutor, 'route executor');
395
463
  stepStart = perfMark();
396
464
  await ensureExpertLoaded(layerIdx, 0, expertWeights, expertLoader);
397
465
  const routeWeights = expertWeights.get(`layer_${layerIdx}_expert_0`);
398
466
  perfLog(`MoE L${layerIdx} route_weight_load`, stepStart, { expertFormat, topK });
399
467
  stepStart = perfMark();
400
- outputTensor = await runGemma4RouteExperts(
468
+ outputTensor = await routeExecutor({
401
469
  inputTensor,
402
470
  indicesBuffer,
403
471
  weightsBuffer,
404
472
  layerRouter,
405
- routeWeights,
473
+ weights: routeWeights,
474
+ expectedExpertFormat: expertFormat,
475
+ profile: moeProfile,
406
476
  layerIdx,
407
477
  numTokens,
408
478
  topK,
409
479
  hiddenSize,
410
480
  intermediateSize,
411
481
  activationDtype,
412
- swigluLimit
413
- );
482
+ swigluLimit,
483
+ });
414
484
  perfLog(`MoE L${layerIdx} route_experts`, stepStart, {
415
485
  numTokens,
416
486
  topK,
@@ -503,6 +573,7 @@ export async function moeFeedForwardGPU(
503
573
  if (!weights.expertFormat) {
504
574
  throw new Error(`[MoE] Expert ${expertKey} missing expertFormat.`);
505
575
  }
576
+ assertMoeExpertWeights(moeProfile, weights, expertKey);
506
577
 
507
578
  const inputOffset = expertIdx * expertStrideBytes;
508
579
  const outputOffset = expertIdx * expertStrideBytes;
@@ -515,8 +586,8 @@ export async function moeFeedForwardGPU(
515
586
  );
516
587
  }
517
588
 
518
- if (expertFormat === 'gpt-oss') {
519
- await runGptOssExpert(
589
+ const expertExecutor = requireMoeExecutor(MOE_EXPERT_EXECUTORS, moeProfile.expertExecutor, 'expert executor');
590
+ await expertExecutor({
520
591
  gathered,
521
592
  expertOutputs,
522
593
  weights,
@@ -531,44 +602,12 @@ export async function moeFeedForwardGPU(
531
602
  activationDtype,
532
603
  swigluLimit,
533
604
  kernelPath,
605
+ executionPolicies: config.executionPolicies ?? null,
534
606
  modelType,
535
607
  vendorProfile,
536
- gptOssKernelPathProfile
537
- );
538
- } else if (expertFormat === 'gemma4' && weights.gateUp && weights.down) {
539
- await runGemma4Expert(
540
- gathered,
541
- expertOutputs,
542
- weights,
543
- count,
544
- inputOffset,
545
- outputOffset,
546
- hiddenSize,
547
- intermediateSize,
548
- activationDtype,
549
- swigluLimit,
550
- kernelPath
551
- );
552
- } else if (expertFormat === 'gemma4') {
553
- throw new Error(`[MoE] Missing Gemma-style packed weights for ${expertKey}`);
554
- } else if (expertFormat === 'mixtral' && weights.gate && weights.up && weights.down) {
555
- await runMixtralExpert(
556
- gathered,
557
- expertOutputs,
558
- weights,
559
- count,
560
- inputOffset,
561
- outputOffset,
562
- hiddenSize,
563
- intermediateSize,
608
+ moeKernelPathProfile,
564
609
  hiddenActivation,
565
- activationDtype,
566
- swigluLimit,
567
- kernelPath
568
- );
569
- } else if (expertFormat === 'mixtral') {
570
- throw new Error(`[MoE] Missing Mixtral weights for ${expertKey}`);
571
- }
610
+ });
572
611
  perfLog(`MoE L${layerIdx} expert_exec`, stepStart, { expertIdx, count });
573
612
  }
574
613
 
@@ -666,28 +705,39 @@ function resolveMatrixStorageStrideBytes(buffer, rows, cols, label) {
666
705
  return alignTo4(rows * cols * bytesPerElement);
667
706
  }
668
707
 
669
- async function runGemma4RouteExperts(
708
+ async function runGemma4RouteExperts({
670
709
  inputTensor,
671
710
  indicesBuffer,
672
711
  weightsBuffer,
673
712
  layerRouter,
674
713
  weights,
714
+ expectedExpertFormat,
715
+ profile,
675
716
  layerIdx,
676
717
  numTokens,
677
718
  topK,
678
719
  hiddenSize,
679
720
  intermediateSize,
680
721
  activationDtype,
681
- swigluLimit
682
- ) {
722
+ swigluLimit,
723
+ }) {
724
+ const profileLabel = typeof profile?.label === 'string' && profile.label.length > 0
725
+ ? profile.label
726
+ : 'unknown MoE profile';
683
727
  if (activationDtype !== 'f16') {
684
- throw new Error(`[MoE] topk-route Gemma4 expert path requires f16 activations, got ${activationDtype}.`);
728
+ throw new Error(`[MoE] topk-route ${profileLabel} path requires f16 activations, got ${activationDtype}.`);
685
729
  }
686
- if (!weights?.gateUp || !weights?.down || weights.expertFormat !== 'gemma4') {
687
- throw new Error(`[MoE] topk-route Gemma4 expert path missing packed weights for layer ${layerIdx}.`);
730
+ if (!weights?.gateUp || !weights?.down) {
731
+ throw new Error(`[MoE] topk-route ${profileLabel} path missing packed weights for layer ${layerIdx}.`);
732
+ }
733
+ if (weights.expertFormat !== expectedExpertFormat) {
734
+ throw new Error(
735
+ `[MoE] topk-route ${profileLabel} expert format mismatch for layer ${layerIdx}: ` +
736
+ `weights=${weights.expertFormat}, config=${expectedExpertFormat}`
737
+ );
688
738
  }
689
739
  if (!layerRouter?.perExpertScale) {
690
- throw new Error(`[MoE] topk-route Gemma4 expert path requires per-expert router scale for layer ${layerIdx}.`);
740
+ throw new Error(`[MoE] topk-route ${profileLabel} path requires per-expert router scale for layer ${layerIdx}.`);
691
741
  }
692
742
 
693
743
  const device = getDevice();
@@ -775,9 +825,10 @@ async function runGptOssExpert(
775
825
  activationDtype,
776
826
  swigluLimit,
777
827
  kernelPath,
828
+ executionPolicies,
778
829
  modelType,
779
830
  vendorProfile,
780
- gptOssKernelPathProfile
831
+ moeKernelPathProfile
781
832
  ) {
782
833
  const perfEnabled = isTraceEnabled('perf');
783
834
  const perfMark = () => (perfEnabled ? performance.now() : 0);
@@ -849,7 +900,7 @@ async function runGptOssExpert(
849
900
  perfLog(`MoE L${layerIdx} expert ${expertIdx} dequant`, stepStart, {
850
901
  hit: false,
851
902
  dequantTileShape: vendorProfile.dequantTileShape,
852
- dequantKernel: gptOssKernelPathProfile?.dequantExpert ?? null,
903
+ dequantKernel: moeKernelPathProfile?.dequantExpert ?? null,
853
904
  });
854
905
  }
855
906
 
@@ -875,7 +926,7 @@ async function runGptOssExpert(
875
926
  let biasTemp = null;
876
927
  if (biasTensor.dtype !== activationDtype) {
877
928
  assertImplicitDtypeTransitionAllowed({
878
- executionPolicies: config.executionPolicies ?? null,
929
+ executionPolicies,
879
930
  fromDtype: biasTensor.dtype,
880
931
  toDtype: activationDtype,
881
932
  op: 'moe_gate_up_bias',
@@ -1,6 +1,6 @@
1
1
  export interface MoEVendorProfile {
2
2
  preferVec4Dequant: boolean;
3
- dequantTileShape: 'vec4' | 'scalar';
3
+ dequantTileShape: 'vec4' | 'scalar' | null;
4
4
  routerWorkgroupSize: number;
5
5
  maxTokensPerExpertScale: number;
6
6
  }
@@ -14,10 +14,35 @@ export interface MoEShapeConfig {
14
14
  }
15
15
 
16
16
  export interface ValidateMoeShapeOptions {
17
- modelType?: string;
17
+ modelType: string;
18
+ moeProfile?: MoEExecutionProfile;
18
19
  }
19
20
 
20
- export declare function resolveMoeVendorProfile(modelType: string): MoEVendorProfile;
21
+ export interface MoEExecutionProfile {
22
+ id: string;
23
+ label: string;
24
+ expertExecutor: 'gpt-oss' | 'gemma4-packed' | 'mixtral' | string;
25
+ intermediateSizeSource: 'architecture' | 'expert' | string;
26
+ requiresShaderF16: boolean;
27
+ routerScaleMode: 'none' | 'optional' | 'required';
28
+ topkRouteExecutor: 'gemma4-route' | string | null;
29
+ vendorProfile: Record<string, unknown>;
30
+ kernelPathProfileResolver: 'gpt-oss' | 'mixtral' | string | null;
31
+ kernelRuleModelType: string | null;
32
+ shapePolicy: Record<string, unknown> | null;
33
+ }
34
+
35
+ export declare function resolveMoeExecutionProfile(
36
+ config: Record<string, unknown>,
37
+ options?: Record<string, unknown>
38
+ ): MoEExecutionProfile;
39
+
40
+ export declare function resolveMoeIntermediateSize(
41
+ config: Record<string, unknown>,
42
+ moeProfile: MoEExecutionProfile
43
+ ): number;
44
+
45
+ export declare function resolveMoeVendorProfile(moeProfile: MoEExecutionProfile): MoEVendorProfile;
21
46
 
22
47
  export declare function validateMoeShape(config: MoEShapeConfig, options?: ValidateMoeShapeOptions): void;
23
48
 
@@ -38,3 +63,8 @@ export interface MixtralKernelPathProfile {
38
63
  export declare function resolveMixtralKernelPathProfile(
39
64
  context: Record<string, unknown>
40
65
  ): Promise<MixtralKernelPathProfile>;
66
+
67
+ export declare function resolveMoeKernelPathProfile(
68
+ moeProfile: MoEExecutionProfile,
69
+ context: Record<string, unknown>
70
+ ): Promise<GptOssKernelPathProfile | MixtralKernelPathProfile | null>;