doppler-gpu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -31
- package/assets/doppler-runtime-map.svg +1 -3
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +4 -1
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.js +6 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
- package/assets/doppler-readme-hero.svg +0 -84
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getDevice } from '../../../gpu/device.js';
|
|
2
|
+
import { getWeightDtype, isWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
2
3
|
import { acquireBuffer, releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
|
|
3
4
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
4
5
|
import { runMatmul, runSiLU, runGeLU } from '../../../gpu/kernel-selector.js';
|
|
@@ -8,6 +9,20 @@ import { ensureExpertLoaded, gatherTokens } from './moe-helpers.js';
|
|
|
8
9
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
9
10
|
import { runGptOssExpertCPU } from './moe-cpu-gptoss.js';
|
|
10
11
|
|
|
12
|
+
function requireMixtralExpertWeightDtype(weights, role, layerIdx, expertIdx) {
|
|
13
|
+
const tensor = weights?.[role];
|
|
14
|
+
if (isWeightBuffer(tensor)) {
|
|
15
|
+
return getWeightDtype(tensor);
|
|
16
|
+
}
|
|
17
|
+
const declared = weights?.[`${role}Dtype`] ?? weights?.weightsDtype ?? tensor?.dtype;
|
|
18
|
+
if (declared === 'f16' || declared === 'f32' || declared === 'q4k') {
|
|
19
|
+
return declared;
|
|
20
|
+
}
|
|
21
|
+
throw new Error(
|
|
22
|
+
`[MoE] Expert layer_${layerIdx}_expert_${expertIdx}.${role} requires explicit weight dtype.`
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
|
|
11
26
|
export async function moeFeedForwardCPU(
|
|
12
27
|
hiddenStates,
|
|
13
28
|
numTokens,
|
|
@@ -72,6 +87,10 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
|
|
|
72
87
|
return new Float32Array(input.length);
|
|
73
88
|
}
|
|
74
89
|
|
|
90
|
+
const gateDtype = requireMixtralExpertWeightDtype(weights, 'gate', layerIdx, expertIdx);
|
|
91
|
+
const upDtype = requireMixtralExpertWeightDtype(weights, 'up', layerIdx, expertIdx);
|
|
92
|
+
const downDtype = requireMixtralExpertWeightDtype(weights, 'down', layerIdx, expertIdx);
|
|
93
|
+
|
|
75
94
|
const inputBuffer = acquireBuffer(input.byteLength, undefined, 'expert_input');
|
|
76
95
|
device.queue.writeBuffer(inputBuffer, 0, input);
|
|
77
96
|
const inputTensor = createTensor(inputBuffer, 'f32', [numTokens, hiddenSize], 'expert_input');
|
|
@@ -84,12 +103,16 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
|
|
|
84
103
|
transposeB: 'auto',
|
|
85
104
|
role: 'moe_gate',
|
|
86
105
|
kernelPath,
|
|
106
|
+
bDtype: gateDtype,
|
|
107
|
+
outputDtype: 'f32',
|
|
87
108
|
});
|
|
88
109
|
|
|
89
110
|
upOutput = await runMatmul(inputTensor, weights.up, numTokens, intermediateSize, hiddenSize, {
|
|
90
111
|
transposeB: 'auto',
|
|
91
112
|
role: 'moe_up',
|
|
92
113
|
kernelPath,
|
|
114
|
+
bDtype: upDtype,
|
|
115
|
+
outputDtype: 'f32',
|
|
93
116
|
});
|
|
94
117
|
|
|
95
118
|
const activationFn = {
|
|
@@ -106,6 +129,8 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
|
|
|
106
129
|
transposeB: 'auto',
|
|
107
130
|
role: 'moe_down',
|
|
108
131
|
kernelPath,
|
|
132
|
+
bDtype: downDtype,
|
|
133
|
+
outputDtype: 'f32',
|
|
109
134
|
});
|
|
110
135
|
|
|
111
136
|
const outputData = await readBuffer(output.buffer, input.byteLength);
|
|
@@ -28,9 +28,10 @@ import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
|
28
28
|
import { getRuntimeConfig } from '../../../config/runtime.js';
|
|
29
29
|
import {
|
|
30
30
|
validateMoeShape,
|
|
31
|
+
resolveMoeExecutionProfile,
|
|
32
|
+
resolveMoeIntermediateSize,
|
|
31
33
|
resolveMoeVendorProfile,
|
|
32
|
-
|
|
33
|
-
resolveMixtralKernelPathProfile,
|
|
34
|
+
resolveMoeKernelPathProfile,
|
|
34
35
|
} from './moe-shape-validator.js';
|
|
35
36
|
import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
|
|
36
37
|
import { QK_K, Q4K_BLOCK_BYTES } from '../../../config/schema/index.js';
|
|
@@ -128,6 +129,92 @@ function resolvePerExpertScaleBuffer(device, value) {
|
|
|
128
129
|
return { buffer, ownedBuffer: null };
|
|
129
130
|
}
|
|
130
131
|
|
|
132
|
+
const MOE_ROUTE_EXECUTORS = Object.freeze({
|
|
133
|
+
'gemma4-route': runGemma4RouteExperts,
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const MOE_EXPERT_EXECUTORS = Object.freeze({
|
|
137
|
+
'gpt-oss': runGptOssProfileExpert,
|
|
138
|
+
'gemma4-packed': runGemma4ProfileExpert,
|
|
139
|
+
mixtral: runMixtralProfileExpert,
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
function requireMoeExecutor(registry, id, label) {
|
|
143
|
+
const executor = registry[id];
|
|
144
|
+
if (typeof executor !== 'function') {
|
|
145
|
+
throw new Error(`[MoE] Unknown ${label} "${String(id)}".`);
|
|
146
|
+
}
|
|
147
|
+
return executor;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function assertMoeExpertWeights(moeProfile, weights, expertKey) {
|
|
151
|
+
if (moeProfile.expertExecutor === 'gemma4-packed' && !weights.gateUp) {
|
|
152
|
+
throw new Error(`[MoE] Missing Gemma-style packed weights for ${expertKey}`);
|
|
153
|
+
}
|
|
154
|
+
if (moeProfile.expertExecutor === 'gemma4-packed' && !weights.down) {
|
|
155
|
+
throw new Error(`[MoE] Missing Gemma-style packed weights for ${expertKey}`);
|
|
156
|
+
}
|
|
157
|
+
if (moeProfile.expertExecutor === 'mixtral' && (!weights.gate || !weights.up || !weights.down)) {
|
|
158
|
+
throw new Error(`[MoE] Missing Mixtral weights for ${expertKey}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async function runGptOssProfileExpert(args) {
|
|
163
|
+
return runGptOssExpert(
|
|
164
|
+
args.gathered,
|
|
165
|
+
args.expertOutputs,
|
|
166
|
+
args.weights,
|
|
167
|
+
args.layerIdx,
|
|
168
|
+
args.expertIdx,
|
|
169
|
+
args.count,
|
|
170
|
+
args.inputOffset,
|
|
171
|
+
args.outputOffset,
|
|
172
|
+
args.hiddenSize,
|
|
173
|
+
args.intermediateSize,
|
|
174
|
+
args.numExperts,
|
|
175
|
+
args.activationDtype,
|
|
176
|
+
args.swigluLimit,
|
|
177
|
+
args.kernelPath,
|
|
178
|
+
args.executionPolicies,
|
|
179
|
+
args.modelType,
|
|
180
|
+
args.vendorProfile,
|
|
181
|
+
args.moeKernelPathProfile
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async function runGemma4ProfileExpert(args) {
|
|
186
|
+
return runGemma4Expert(
|
|
187
|
+
args.gathered,
|
|
188
|
+
args.expertOutputs,
|
|
189
|
+
args.weights,
|
|
190
|
+
args.count,
|
|
191
|
+
args.inputOffset,
|
|
192
|
+
args.outputOffset,
|
|
193
|
+
args.hiddenSize,
|
|
194
|
+
args.intermediateSize,
|
|
195
|
+
args.activationDtype,
|
|
196
|
+
args.swigluLimit,
|
|
197
|
+
args.kernelPath
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
async function runMixtralProfileExpert(args) {
|
|
202
|
+
return runMixtralExpert(
|
|
203
|
+
args.gathered,
|
|
204
|
+
args.expertOutputs,
|
|
205
|
+
args.weights,
|
|
206
|
+
args.count,
|
|
207
|
+
args.inputOffset,
|
|
208
|
+
args.outputOffset,
|
|
209
|
+
args.hiddenSize,
|
|
210
|
+
args.intermediateSize,
|
|
211
|
+
args.hiddenActivation,
|
|
212
|
+
args.activationDtype,
|
|
213
|
+
args.swigluLimit,
|
|
214
|
+
args.kernelPath
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
|
|
131
218
|
export async function moeFeedForwardGPU(
|
|
132
219
|
inputBuffer,
|
|
133
220
|
numTokens,
|
|
@@ -143,9 +230,6 @@ export async function moeFeedForwardGPU(
|
|
|
143
230
|
|
|
144
231
|
const { hiddenSize, numExperts, moeTopK, hiddenActivation } = config;
|
|
145
232
|
const expertFormat = config.expertFormat;
|
|
146
|
-
const intermediateSize = expertFormat === 'gemma4'
|
|
147
|
-
? config.expertIntermediateSize
|
|
148
|
-
: config.intermediateSize;
|
|
149
233
|
const swigluLimit = config.swigluLimit;
|
|
150
234
|
const kernelPath = config.kernelPath ?? null;
|
|
151
235
|
if (!expertFormat) {
|
|
@@ -154,9 +238,6 @@ export async function moeFeedForwardGPU(
|
|
|
154
238
|
if (swigluLimit === undefined) {
|
|
155
239
|
throw new Error('MoE swigluLimit must be explicitly set (null or number).');
|
|
156
240
|
}
|
|
157
|
-
if (!Number.isFinite(intermediateSize) || intermediateSize <= 0) {
|
|
158
|
-
throw new Error(`[MoE] Invalid expert intermediate size for ${expertFormat}: ${String(intermediateSize)}.`);
|
|
159
|
-
}
|
|
160
241
|
const topK = moeTopK ?? moeRouter.topK;
|
|
161
242
|
if (topK == null) {
|
|
162
243
|
throw new Error('MoE topK is required in config.');
|
|
@@ -165,15 +246,17 @@ export async function moeFeedForwardGPU(
|
|
|
165
246
|
throw new Error('MoE config.modelType is required; got null/undefined.');
|
|
166
247
|
}
|
|
167
248
|
const modelType = config.modelType;
|
|
249
|
+
const moeProfile = resolveMoeExecutionProfile(config, { modelType });
|
|
250
|
+
const intermediateSize = resolveMoeIntermediateSize(config, moeProfile);
|
|
168
251
|
validateMoeShape(
|
|
169
252
|
{ hiddenSize, intermediateSize, moeTopK: topK, numExperts, expertFormat },
|
|
170
|
-
{ modelType }
|
|
253
|
+
{ modelType, moeProfile }
|
|
171
254
|
);
|
|
172
|
-
const vendorProfile = resolveMoeVendorProfile(
|
|
255
|
+
const vendorProfile = resolveMoeVendorProfile(moeProfile);
|
|
173
256
|
const caps = getKernelCapabilities();
|
|
174
|
-
if (
|
|
257
|
+
if (moeProfile.requiresShaderF16 && !caps.hasF16) {
|
|
175
258
|
throw new Error(
|
|
176
|
-
|
|
259
|
+
`[MoE] ${moeProfile.label} requires shader-f16 support. ` +
|
|
177
260
|
`Adapter: ${caps.adapterInfo?.vendor ?? 'unknown'} ${caps.adapterInfo?.architecture ?? ''}`.trim()
|
|
178
261
|
);
|
|
179
262
|
}
|
|
@@ -224,19 +307,19 @@ export async function moeFeedForwardGPU(
|
|
|
224
307
|
}
|
|
225
308
|
|
|
226
309
|
try {
|
|
227
|
-
const
|
|
310
|
+
const needsRouterScale = moeProfile.routerScaleMode === 'required'
|
|
228
311
|
|| layerRouter?.scale != null
|
|
229
312
|
|| layerRouter?.perExpertScale != null;
|
|
230
313
|
let routerInputTensor = routerSourceTensor;
|
|
231
|
-
if (
|
|
314
|
+
if (needsRouterScale) {
|
|
232
315
|
if (!layerRouter?.scale) {
|
|
233
|
-
throw new Error(`[MoE]
|
|
316
|
+
throw new Error(`[MoE] ${moeProfile.label} router scale missing for layer ${layerIdx}.`);
|
|
234
317
|
}
|
|
235
318
|
if (!layerRouter?.perExpertScale) {
|
|
236
|
-
throw new Error(`[MoE]
|
|
319
|
+
throw new Error(`[MoE] ${moeProfile.label} per-expert router scale missing for layer ${layerIdx}.`);
|
|
237
320
|
}
|
|
238
321
|
if (!Number.isFinite(config.rmsNormEps) || config.rmsNormEps <= 0) {
|
|
239
|
-
throw new Error(`[MoE]
|
|
322
|
+
throw new Error(`[MoE] ${moeProfile.label} router RMSNorm eps is invalid: ${String(config.rmsNormEps)}.`);
|
|
240
323
|
}
|
|
241
324
|
routerNormTensor = await runRMSNorm(
|
|
242
325
|
inputTensor,
|
|
@@ -295,32 +378,16 @@ export async function moeFeedForwardGPU(
|
|
|
295
378
|
trace.buffers(`MoE L${layerIdx} router_logits`, { min, max, nanCount, dtype: logitsDtype });
|
|
296
379
|
}
|
|
297
380
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
hasSubgroups: caps.hasSubgroups,
|
|
309
|
-
routerDtype: logitsDtype,
|
|
310
|
-
weightsDtype: activationDtype,
|
|
311
|
-
outputDtype: activationDtype,
|
|
312
|
-
groupSize: 32,
|
|
313
|
-
tileShape: vendorProfile.dequantTileShape,
|
|
314
|
-
});
|
|
315
|
-
} else if (modelType === 'mixtral') {
|
|
316
|
-
mixtralKernelPathProfile = await resolveMixtralKernelPathProfile({
|
|
317
|
-
hasF16: caps.hasF16,
|
|
318
|
-
hasSubgroups: caps.hasSubgroups,
|
|
319
|
-
routerDtype: logitsDtype,
|
|
320
|
-
weightsDtype: activationDtype,
|
|
321
|
-
outputDtype: activationDtype,
|
|
322
|
-
});
|
|
323
|
-
}
|
|
381
|
+
const moeKernelPathProfile = await resolveMoeKernelPathProfile(moeProfile, {
|
|
382
|
+
hasF16: caps.hasF16,
|
|
383
|
+
hasSubgroups: caps.hasSubgroups,
|
|
384
|
+
routerDtype: logitsDtype,
|
|
385
|
+
inputDtype: logitsDtype,
|
|
386
|
+
weightsDtype: activationDtype,
|
|
387
|
+
outputDtype: activationDtype,
|
|
388
|
+
groupSize: 32,
|
|
389
|
+
tileShape: vendorProfile.dequantTileShape,
|
|
390
|
+
});
|
|
324
391
|
|
|
325
392
|
stepStart = perfMark();
|
|
326
393
|
({ indices: indicesBuffer, weights: weightsBuffer } = await runSoftmaxTopK(
|
|
@@ -338,7 +405,7 @@ export async function moeFeedForwardGPU(
|
|
|
338
405
|
perfLog(`MoE L${layerIdx} topk`, stepStart, {
|
|
339
406
|
topK,
|
|
340
407
|
modelType,
|
|
341
|
-
routerTopKKernel:
|
|
408
|
+
routerTopKKernel: moeKernelPathProfile?.routerTopK ?? null,
|
|
342
409
|
});
|
|
343
410
|
|
|
344
411
|
if (isTraceEnabled('buffers')) {
|
|
@@ -389,28 +456,31 @@ export async function moeFeedForwardGPU(
|
|
|
389
456
|
|
|
390
457
|
const activeExpertSelection = resolveMoEActiveExpertSelection();
|
|
391
458
|
if (activeExpertSelection === 'topk-route') {
|
|
392
|
-
if (
|
|
393
|
-
throw new Error(`[MoE] topk-route active expert selection
|
|
459
|
+
if (moeProfile.topkRouteExecutor == null) {
|
|
460
|
+
throw new Error(`[MoE] topk-route active expert selection is not supported by profile "${moeProfile.id}".`);
|
|
394
461
|
}
|
|
462
|
+
const routeExecutor = requireMoeExecutor(MOE_ROUTE_EXECUTORS, moeProfile.topkRouteExecutor, 'route executor');
|
|
395
463
|
stepStart = perfMark();
|
|
396
464
|
await ensureExpertLoaded(layerIdx, 0, expertWeights, expertLoader);
|
|
397
465
|
const routeWeights = expertWeights.get(`layer_${layerIdx}_expert_0`);
|
|
398
466
|
perfLog(`MoE L${layerIdx} route_weight_load`, stepStart, { expertFormat, topK });
|
|
399
467
|
stepStart = perfMark();
|
|
400
|
-
outputTensor = await
|
|
468
|
+
outputTensor = await routeExecutor({
|
|
401
469
|
inputTensor,
|
|
402
470
|
indicesBuffer,
|
|
403
471
|
weightsBuffer,
|
|
404
472
|
layerRouter,
|
|
405
|
-
routeWeights,
|
|
473
|
+
weights: routeWeights,
|
|
474
|
+
expectedExpertFormat: expertFormat,
|
|
475
|
+
profile: moeProfile,
|
|
406
476
|
layerIdx,
|
|
407
477
|
numTokens,
|
|
408
478
|
topK,
|
|
409
479
|
hiddenSize,
|
|
410
480
|
intermediateSize,
|
|
411
481
|
activationDtype,
|
|
412
|
-
swigluLimit
|
|
413
|
-
);
|
|
482
|
+
swigluLimit,
|
|
483
|
+
});
|
|
414
484
|
perfLog(`MoE L${layerIdx} route_experts`, stepStart, {
|
|
415
485
|
numTokens,
|
|
416
486
|
topK,
|
|
@@ -503,6 +573,7 @@ export async function moeFeedForwardGPU(
|
|
|
503
573
|
if (!weights.expertFormat) {
|
|
504
574
|
throw new Error(`[MoE] Expert ${expertKey} missing expertFormat.`);
|
|
505
575
|
}
|
|
576
|
+
assertMoeExpertWeights(moeProfile, weights, expertKey);
|
|
506
577
|
|
|
507
578
|
const inputOffset = expertIdx * expertStrideBytes;
|
|
508
579
|
const outputOffset = expertIdx * expertStrideBytes;
|
|
@@ -515,8 +586,8 @@ export async function moeFeedForwardGPU(
|
|
|
515
586
|
);
|
|
516
587
|
}
|
|
517
588
|
|
|
518
|
-
|
|
519
|
-
|
|
589
|
+
const expertExecutor = requireMoeExecutor(MOE_EXPERT_EXECUTORS, moeProfile.expertExecutor, 'expert executor');
|
|
590
|
+
await expertExecutor({
|
|
520
591
|
gathered,
|
|
521
592
|
expertOutputs,
|
|
522
593
|
weights,
|
|
@@ -531,44 +602,12 @@ export async function moeFeedForwardGPU(
|
|
|
531
602
|
activationDtype,
|
|
532
603
|
swigluLimit,
|
|
533
604
|
kernelPath,
|
|
605
|
+
executionPolicies: config.executionPolicies ?? null,
|
|
534
606
|
modelType,
|
|
535
607
|
vendorProfile,
|
|
536
|
-
|
|
537
|
-
);
|
|
538
|
-
} else if (expertFormat === 'gemma4' && weights.gateUp && weights.down) {
|
|
539
|
-
await runGemma4Expert(
|
|
540
|
-
gathered,
|
|
541
|
-
expertOutputs,
|
|
542
|
-
weights,
|
|
543
|
-
count,
|
|
544
|
-
inputOffset,
|
|
545
|
-
outputOffset,
|
|
546
|
-
hiddenSize,
|
|
547
|
-
intermediateSize,
|
|
548
|
-
activationDtype,
|
|
549
|
-
swigluLimit,
|
|
550
|
-
kernelPath
|
|
551
|
-
);
|
|
552
|
-
} else if (expertFormat === 'gemma4') {
|
|
553
|
-
throw new Error(`[MoE] Missing Gemma-style packed weights for ${expertKey}`);
|
|
554
|
-
} else if (expertFormat === 'mixtral' && weights.gate && weights.up && weights.down) {
|
|
555
|
-
await runMixtralExpert(
|
|
556
|
-
gathered,
|
|
557
|
-
expertOutputs,
|
|
558
|
-
weights,
|
|
559
|
-
count,
|
|
560
|
-
inputOffset,
|
|
561
|
-
outputOffset,
|
|
562
|
-
hiddenSize,
|
|
563
|
-
intermediateSize,
|
|
608
|
+
moeKernelPathProfile,
|
|
564
609
|
hiddenActivation,
|
|
565
|
-
|
|
566
|
-
swigluLimit,
|
|
567
|
-
kernelPath
|
|
568
|
-
);
|
|
569
|
-
} else if (expertFormat === 'mixtral') {
|
|
570
|
-
throw new Error(`[MoE] Missing Mixtral weights for ${expertKey}`);
|
|
571
|
-
}
|
|
610
|
+
});
|
|
572
611
|
perfLog(`MoE L${layerIdx} expert_exec`, stepStart, { expertIdx, count });
|
|
573
612
|
}
|
|
574
613
|
|
|
@@ -666,28 +705,39 @@ function resolveMatrixStorageStrideBytes(buffer, rows, cols, label) {
|
|
|
666
705
|
return alignTo4(rows * cols * bytesPerElement);
|
|
667
706
|
}
|
|
668
707
|
|
|
669
|
-
async function runGemma4RouteExperts(
|
|
708
|
+
async function runGemma4RouteExperts({
|
|
670
709
|
inputTensor,
|
|
671
710
|
indicesBuffer,
|
|
672
711
|
weightsBuffer,
|
|
673
712
|
layerRouter,
|
|
674
713
|
weights,
|
|
714
|
+
expectedExpertFormat,
|
|
715
|
+
profile,
|
|
675
716
|
layerIdx,
|
|
676
717
|
numTokens,
|
|
677
718
|
topK,
|
|
678
719
|
hiddenSize,
|
|
679
720
|
intermediateSize,
|
|
680
721
|
activationDtype,
|
|
681
|
-
swigluLimit
|
|
682
|
-
) {
|
|
722
|
+
swigluLimit,
|
|
723
|
+
}) {
|
|
724
|
+
const profileLabel = typeof profile?.label === 'string' && profile.label.length > 0
|
|
725
|
+
? profile.label
|
|
726
|
+
: 'unknown MoE profile';
|
|
683
727
|
if (activationDtype !== 'f16') {
|
|
684
|
-
throw new Error(`[MoE] topk-route
|
|
728
|
+
throw new Error(`[MoE] topk-route ${profileLabel} path requires f16 activations, got ${activationDtype}.`);
|
|
685
729
|
}
|
|
686
|
-
if (!weights?.gateUp || !weights?.down
|
|
687
|
-
throw new Error(`[MoE] topk-route
|
|
730
|
+
if (!weights?.gateUp || !weights?.down) {
|
|
731
|
+
throw new Error(`[MoE] topk-route ${profileLabel} path missing packed weights for layer ${layerIdx}.`);
|
|
732
|
+
}
|
|
733
|
+
if (weights.expertFormat !== expectedExpertFormat) {
|
|
734
|
+
throw new Error(
|
|
735
|
+
`[MoE] topk-route ${profileLabel} expert format mismatch for layer ${layerIdx}: ` +
|
|
736
|
+
`weights=${weights.expertFormat}, config=${expectedExpertFormat}`
|
|
737
|
+
);
|
|
688
738
|
}
|
|
689
739
|
if (!layerRouter?.perExpertScale) {
|
|
690
|
-
throw new Error(`[MoE] topk-route
|
|
740
|
+
throw new Error(`[MoE] topk-route ${profileLabel} path requires per-expert router scale for layer ${layerIdx}.`);
|
|
691
741
|
}
|
|
692
742
|
|
|
693
743
|
const device = getDevice();
|
|
@@ -775,9 +825,10 @@ async function runGptOssExpert(
|
|
|
775
825
|
activationDtype,
|
|
776
826
|
swigluLimit,
|
|
777
827
|
kernelPath,
|
|
828
|
+
executionPolicies,
|
|
778
829
|
modelType,
|
|
779
830
|
vendorProfile,
|
|
780
|
-
|
|
831
|
+
moeKernelPathProfile
|
|
781
832
|
) {
|
|
782
833
|
const perfEnabled = isTraceEnabled('perf');
|
|
783
834
|
const perfMark = () => (perfEnabled ? performance.now() : 0);
|
|
@@ -849,7 +900,7 @@ async function runGptOssExpert(
|
|
|
849
900
|
perfLog(`MoE L${layerIdx} expert ${expertIdx} dequant`, stepStart, {
|
|
850
901
|
hit: false,
|
|
851
902
|
dequantTileShape: vendorProfile.dequantTileShape,
|
|
852
|
-
dequantKernel:
|
|
903
|
+
dequantKernel: moeKernelPathProfile?.dequantExpert ?? null,
|
|
853
904
|
});
|
|
854
905
|
}
|
|
855
906
|
|
|
@@ -875,7 +926,7 @@ async function runGptOssExpert(
|
|
|
875
926
|
let biasTemp = null;
|
|
876
927
|
if (biasTensor.dtype !== activationDtype) {
|
|
877
928
|
assertImplicitDtypeTransitionAllowed({
|
|
878
|
-
executionPolicies
|
|
929
|
+
executionPolicies,
|
|
879
930
|
fromDtype: biasTensor.dtype,
|
|
880
931
|
toDtype: activationDtype,
|
|
881
932
|
op: 'moe_gate_up_bias',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export interface MoEVendorProfile {
|
|
2
2
|
preferVec4Dequant: boolean;
|
|
3
|
-
dequantTileShape: 'vec4' | 'scalar';
|
|
3
|
+
dequantTileShape: 'vec4' | 'scalar' | null;
|
|
4
4
|
routerWorkgroupSize: number;
|
|
5
5
|
maxTokensPerExpertScale: number;
|
|
6
6
|
}
|
|
@@ -14,10 +14,35 @@ export interface MoEShapeConfig {
|
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
export interface ValidateMoeShapeOptions {
|
|
17
|
-
modelType
|
|
17
|
+
modelType: string;
|
|
18
|
+
moeProfile?: MoEExecutionProfile;
|
|
18
19
|
}
|
|
19
20
|
|
|
20
|
-
export
|
|
21
|
+
export interface MoEExecutionProfile {
|
|
22
|
+
id: string;
|
|
23
|
+
label: string;
|
|
24
|
+
expertExecutor: 'gpt-oss' | 'gemma4-packed' | 'mixtral' | string;
|
|
25
|
+
intermediateSizeSource: 'architecture' | 'expert' | string;
|
|
26
|
+
requiresShaderF16: boolean;
|
|
27
|
+
routerScaleMode: 'none' | 'optional' | 'required';
|
|
28
|
+
topkRouteExecutor: 'gemma4-route' | string | null;
|
|
29
|
+
vendorProfile: Record<string, unknown>;
|
|
30
|
+
kernelPathProfileResolver: 'gpt-oss' | 'mixtral' | string | null;
|
|
31
|
+
kernelRuleModelType: string | null;
|
|
32
|
+
shapePolicy: Record<string, unknown> | null;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export declare function resolveMoeExecutionProfile(
|
|
36
|
+
config: Record<string, unknown>,
|
|
37
|
+
options?: Record<string, unknown>
|
|
38
|
+
): MoEExecutionProfile;
|
|
39
|
+
|
|
40
|
+
export declare function resolveMoeIntermediateSize(
|
|
41
|
+
config: Record<string, unknown>,
|
|
42
|
+
moeProfile: MoEExecutionProfile
|
|
43
|
+
): number;
|
|
44
|
+
|
|
45
|
+
export declare function resolveMoeVendorProfile(moeProfile: MoEExecutionProfile): MoEVendorProfile;
|
|
21
46
|
|
|
22
47
|
export declare function validateMoeShape(config: MoEShapeConfig, options?: ValidateMoeShapeOptions): void;
|
|
23
48
|
|
|
@@ -38,3 +63,8 @@ export interface MixtralKernelPathProfile {
|
|
|
38
63
|
export declare function resolveMixtralKernelPathProfile(
|
|
39
64
|
context: Record<string, unknown>
|
|
40
65
|
): Promise<MixtralKernelPathProfile>;
|
|
66
|
+
|
|
67
|
+
export declare function resolveMoeKernelPathProfile(
|
|
68
|
+
moeProfile: MoEExecutionProfile,
|
|
69
|
+
context: Record<string, unknown>
|
|
70
|
+
): Promise<GptOssKernelPathProfile | MixtralKernelPathProfile | null>;
|