doppler-gpu 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -29
- package/assets/doppler-runtime-map.svg +82 -0
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/client/runtime/model-session.d.ts +1 -0
- package/src/client/runtime/model-session.js +9 -0
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/device.js +54 -11
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +63 -9
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.d.ts +1 -0
- package/src/inference/pipelines/text/generator.js +14 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/sampling.js +52 -10
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/tensors/tensor-loader.js +37 -3
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/loader/weights.rules.json +1 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
|
@@ -28,6 +28,15 @@ function asNumberArray(value) {
|
|
|
28
28
|
return normalized.map((entry) => Math.trunc(entry));
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
function asPositiveNumberArray(value) {
|
|
32
|
+
if (!Array.isArray(value)) return null;
|
|
33
|
+
const normalized = value.map((entry) => asFiniteNumber(entry));
|
|
34
|
+
if (normalized.some((entry) => entry == null || entry <= 0)) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
return normalized.map((entry) => Number(entry));
|
|
38
|
+
}
|
|
39
|
+
|
|
31
40
|
function normalizeRoPEType(value) {
|
|
32
41
|
if (typeof value !== 'string') return null;
|
|
33
42
|
const normalized = value.trim().toLowerCase();
|
|
@@ -90,6 +99,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
|
|
|
90
99
|
yarnBetaFast: null,
|
|
91
100
|
yarnBetaSlow: null,
|
|
92
101
|
yarnOriginalMaxPos: null,
|
|
102
|
+
longropeShortFactor: null,
|
|
103
|
+
longropeLongFactor: null,
|
|
104
|
+
longropeOriginalMaxPos: null,
|
|
93
105
|
};
|
|
94
106
|
}
|
|
95
107
|
|
|
@@ -98,6 +110,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
|
|
|
98
110
|
let yarnBetaFast = null;
|
|
99
111
|
let yarnBetaSlow = null;
|
|
100
112
|
let yarnOriginalMaxPos = null;
|
|
113
|
+
let longropeShortFactor = null;
|
|
114
|
+
let longropeLongFactor = null;
|
|
115
|
+
let longropeOriginalMaxPos = null;
|
|
101
116
|
|
|
102
117
|
if (ropeScalingType == null) {
|
|
103
118
|
if (factor != null && factor > 0 && factor !== 1.0) {
|
|
@@ -123,6 +138,23 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
|
|
|
123
138
|
yarnBetaSlow = betaSlow;
|
|
124
139
|
yarnOriginalMaxPos = origMaxPos;
|
|
125
140
|
}
|
|
141
|
+
if (ropeScalingType === 'longrope') {
|
|
142
|
+
const shortFactor = asPositiveNumberArray(ropeScalingConfig.short_factor);
|
|
143
|
+
const longFactor = asPositiveNumberArray(ropeScalingConfig.long_factor);
|
|
144
|
+
const origMaxPos = asFiniteNumber(ropeScalingConfig.original_max_position_embeddings);
|
|
145
|
+
if (!shortFactor || !longFactor || origMaxPos == null) {
|
|
146
|
+
throw new Error(
|
|
147
|
+
'LongRoPE scaling detected but required params missing in HF config. ' +
|
|
148
|
+
'LongRoPE requires short_factor, long_factor, and original_max_position_embeddings. ' +
|
|
149
|
+
`Got: short_factor=${Array.isArray(ropeScalingConfig.short_factor) ? ropeScalingConfig.short_factor.length : ropeScalingConfig.short_factor}, ` +
|
|
150
|
+
`long_factor=${Array.isArray(ropeScalingConfig.long_factor) ? ropeScalingConfig.long_factor.length : ropeScalingConfig.long_factor}, ` +
|
|
151
|
+
`original_max_position_embeddings=${origMaxPos}`
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
longropeShortFactor = shortFactor;
|
|
155
|
+
longropeLongFactor = longFactor;
|
|
156
|
+
longropeOriginalMaxPos = origMaxPos;
|
|
157
|
+
}
|
|
126
158
|
|
|
127
159
|
return {
|
|
128
160
|
ropeScalingType,
|
|
@@ -130,6 +162,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
|
|
|
130
162
|
yarnBetaFast,
|
|
131
163
|
yarnBetaSlow,
|
|
132
164
|
yarnOriginalMaxPos,
|
|
165
|
+
longropeShortFactor,
|
|
166
|
+
longropeLongFactor,
|
|
167
|
+
longropeOriginalMaxPos,
|
|
133
168
|
};
|
|
134
169
|
}
|
|
135
170
|
|
|
@@ -140,7 +175,9 @@ function hasScalingDirective(ropeScalingConfig) {
|
|
|
140
175
|
|| ropeScalingConfig.factor != null
|
|
141
176
|
|| ropeScalingConfig.beta_fast != null
|
|
142
177
|
|| ropeScalingConfig.beta_slow != null
|
|
143
|
-
|| ropeScalingConfig.original_max_position_embeddings != null
|
|
178
|
+
|| ropeScalingConfig.original_max_position_embeddings != null
|
|
179
|
+
|| ropeScalingConfig.short_factor != null
|
|
180
|
+
|| ropeScalingConfig.long_factor != null;
|
|
144
181
|
}
|
|
145
182
|
|
|
146
183
|
function hasMeaningfulScalingConfig(resolvedScaling) {
|
|
@@ -149,7 +186,10 @@ function hasMeaningfulScalingConfig(resolvedScaling) {
|
|
|
149
186
|
|| resolvedScaling.ropeScalingFactor !== DEFAULT_MANIFEST_INFERENCE.rope.ropeScalingFactor
|
|
150
187
|
|| resolvedScaling.yarnBetaFast != null
|
|
151
188
|
|| resolvedScaling.yarnBetaSlow != null
|
|
152
|
-
|| resolvedScaling.yarnOriginalMaxPos != null
|
|
189
|
+
|| resolvedScaling.yarnOriginalMaxPos != null
|
|
190
|
+
|| resolvedScaling.longropeShortFactor != null
|
|
191
|
+
|| resolvedScaling.longropeLongFactor != null
|
|
192
|
+
|| resolvedScaling.longropeOriginalMaxPos != null;
|
|
153
193
|
}
|
|
154
194
|
|
|
155
195
|
function isSameScalingConfig(left, right) {
|
|
@@ -157,7 +197,10 @@ function isSameScalingConfig(left, right) {
|
|
|
157
197
|
&& left.ropeScalingFactor === right.ropeScalingFactor
|
|
158
198
|
&& left.yarnBetaFast === right.yarnBetaFast
|
|
159
199
|
&& left.yarnBetaSlow === right.yarnBetaSlow
|
|
160
|
-
&& left.yarnOriginalMaxPos === right.yarnOriginalMaxPos
|
|
200
|
+
&& left.yarnOriginalMaxPos === right.yarnOriginalMaxPos
|
|
201
|
+
&& JSON.stringify(left.longropeShortFactor ?? null) === JSON.stringify(right.longropeShortFactor ?? null)
|
|
202
|
+
&& JSON.stringify(left.longropeLongFactor ?? null) === JSON.stringify(right.longropeLongFactor ?? null)
|
|
203
|
+
&& left.longropeOriginalMaxPos === right.longropeOriginalMaxPos;
|
|
161
204
|
}
|
|
162
205
|
|
|
163
206
|
function failOnConflictingScaling(sourceLabel, canonicalScaling, candidateScaling) {
|
|
@@ -210,6 +253,9 @@ export function buildRoPEConfig(converterInference, config) {
|
|
|
210
253
|
yarnBetaFast: configuredRoPE.yarnBetaFast ?? null,
|
|
211
254
|
yarnBetaSlow: configuredRoPE.yarnBetaSlow ?? null,
|
|
212
255
|
yarnOriginalMaxPos: configuredRoPE.yarnOriginalMaxPos ?? null,
|
|
256
|
+
longropeShortFactor: configuredRoPE.longropeShortFactor ?? null,
|
|
257
|
+
longropeLongFactor: configuredRoPE.longropeLongFactor ?? null,
|
|
258
|
+
longropeOriginalMaxPos: configuredRoPE.longropeOriginalMaxPos ?? null,
|
|
213
259
|
};
|
|
214
260
|
|
|
215
261
|
if (ropeScaling) {
|
|
@@ -253,6 +299,9 @@ export function buildRoPEConfig(converterInference, config) {
|
|
|
253
299
|
yarnBetaFast: configuredRoPE.ropeLocalYarnBetaFast ?? globalScaling.yarnBetaFast,
|
|
254
300
|
yarnBetaSlow: configuredRoPE.ropeLocalYarnBetaSlow ?? globalScaling.yarnBetaSlow,
|
|
255
301
|
yarnOriginalMaxPos: configuredRoPE.ropeLocalYarnOriginalMaxPos ?? globalScaling.yarnOriginalMaxPos,
|
|
302
|
+
longropeShortFactor: globalScaling.longropeShortFactor,
|
|
303
|
+
longropeLongFactor: globalScaling.longropeLongFactor,
|
|
304
|
+
longropeOriginalMaxPos: globalScaling.longropeOriginalMaxPos,
|
|
256
305
|
}
|
|
257
306
|
: { ...globalScaling };
|
|
258
307
|
if (ropeScaling) {
|
|
@@ -323,6 +372,9 @@ export function buildRoPEConfig(converterInference, config) {
|
|
|
323
372
|
yarnBetaFast: globalScaling.yarnBetaFast,
|
|
324
373
|
yarnBetaSlow: globalScaling.yarnBetaSlow,
|
|
325
374
|
yarnOriginalMaxPos: globalScaling.yarnOriginalMaxPos,
|
|
375
|
+
longropeShortFactor: globalScaling.longropeShortFactor,
|
|
376
|
+
longropeLongFactor: globalScaling.longropeLongFactor,
|
|
377
|
+
longropeOriginalMaxPos: globalScaling.longropeOriginalMaxPos,
|
|
326
378
|
ropeLocalScalingType: localScaling.ropeScalingType,
|
|
327
379
|
ropeLocalScalingFactor: localScaling.ropeScalingFactor,
|
|
328
380
|
ropeLocalYarnBetaFast: localScaling.yarnBetaFast,
|
|
@@ -421,7 +421,7 @@ function createDistillStudentProjectionModelFixture(overrides = {}, options = {}
|
|
|
421
421
|
const rows = Number.isFinite(inputTensor?.shape?.[0]) ? inputTensor.shape[0] : 1;
|
|
422
422
|
return tape.record(
|
|
423
423
|
OpType.MATMUL,
|
|
424
|
-
(a, b) => runMatmul(a, b, rows, outputDim, embeddingDim, { transposeB: false }),
|
|
424
|
+
(a, b) => runMatmul(a, b, rows, outputDim, embeddingDim, { transposeB: false, outputDtype: 'f32' }),
|
|
425
425
|
[inputTensor, projectionWeight],
|
|
426
426
|
{ M: rows, N: outputDim, K: embeddingDim, transposeB: false }
|
|
427
427
|
);
|
|
@@ -400,7 +400,7 @@ function createToyLoraModel(workload) {
|
|
|
400
400
|
const batchSize = Number.isInteger(inputTensor?.shape?.[0]) ? inputTensor.shape[0] : 1;
|
|
401
401
|
const baseLogits = await tape.record(
|
|
402
402
|
OpType.MATMUL,
|
|
403
|
-
(a, b) => runMatmul(a, b, batchSize, 2, 3, { transposeB: false }),
|
|
403
|
+
(a, b) => runMatmul(a, b, batchSize, 2, 3, { transposeB: false, outputDtype: 'f32' }),
|
|
404
404
|
[inputTensor, baseWeight],
|
|
405
405
|
{ M: batchSize, N: 2, K: 3, transposeB: false }
|
|
406
406
|
);
|
|
@@ -68,13 +68,13 @@ export class LoraAdapter {
|
|
|
68
68
|
const [tokens] = input.shape;
|
|
69
69
|
const down = await tape.record(
|
|
70
70
|
OpType.MATMUL,
|
|
71
|
-
(a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0]),
|
|
71
|
+
(a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0], { outputDtype: input.dtype }),
|
|
72
72
|
[input, this.A],
|
|
73
73
|
{ M: tokens, N: this.rank, K: this.A.shape[0] }
|
|
74
74
|
);
|
|
75
75
|
const up = await tape.record(
|
|
76
76
|
OpType.MATMUL,
|
|
77
|
-
(a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank),
|
|
77
|
+
(a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank, { outputDtype: down.dtype }),
|
|
78
78
|
[down, this.B],
|
|
79
79
|
{ M: tokens, N: this.B.shape[1], K: this.rank }
|
|
80
80
|
);
|
|
@@ -821,7 +821,7 @@ export function createToyModelFixture(overrides = {}) {
|
|
|
821
821
|
async forward(inputTensor, tape) {
|
|
822
822
|
return tape.record(
|
|
823
823
|
OpType.MATMUL,
|
|
824
|
-
(a, b) => runMatmul(a, b, 2, 2, 3, { transposeB: false }),
|
|
824
|
+
(a, b) => runMatmul(a, b, 2, 2, 3, { transposeB: false, outputDtype: 'f32' }),
|
|
825
825
|
[inputTensor, baseWeight],
|
|
826
826
|
{ M: 2, N: 2, K: 3, transposeB: false }
|
|
827
827
|
);
|
package/src/gpu/device.js
CHANGED
|
@@ -369,6 +369,59 @@ function buildFeatureRequests(available) {
|
|
|
369
369
|
return requested;
|
|
370
370
|
}
|
|
371
371
|
|
|
372
|
+
function addFeatureRequestCandidate(candidates, seen, features) {
|
|
373
|
+
const unique = features.filter((feature, index) => features.indexOf(feature) === index);
|
|
374
|
+
const key = unique.join('|');
|
|
375
|
+
if (seen.has(key)) return;
|
|
376
|
+
seen.add(key);
|
|
377
|
+
candidates.push(unique);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
function buildFeatureRequestCandidates(requestedFeatures) {
|
|
381
|
+
const requested = Array.isArray(requestedFeatures) ? requestedFeatures : [];
|
|
382
|
+
const candidates = [];
|
|
383
|
+
const seen = new Set();
|
|
384
|
+
addFeatureRequestCandidate(candidates, seen, requested);
|
|
385
|
+
addFeatureRequestCandidate(
|
|
386
|
+
candidates,
|
|
387
|
+
seen,
|
|
388
|
+
requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY)
|
|
389
|
+
);
|
|
390
|
+
addFeatureRequestCandidate(
|
|
391
|
+
candidates,
|
|
392
|
+
seen,
|
|
393
|
+
requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY && feature !== FEATURES.SUBGROUPS)
|
|
394
|
+
);
|
|
395
|
+
addFeatureRequestCandidate(
|
|
396
|
+
candidates,
|
|
397
|
+
seen,
|
|
398
|
+
requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY && feature !== FEATURES.SHADER_F16)
|
|
399
|
+
);
|
|
400
|
+
addFeatureRequestCandidate(candidates, seen, []);
|
|
401
|
+
return candidates;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
async function requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits) {
|
|
405
|
+
const candidates = buildFeatureRequestCandidates(requestedFeatures);
|
|
406
|
+
let lastError = null;
|
|
407
|
+
for (const features of candidates) {
|
|
408
|
+
try {
|
|
409
|
+
if (features.length === 0) {
|
|
410
|
+
return await adapter.requestDevice();
|
|
411
|
+
}
|
|
412
|
+
return await adapter.requestDevice({
|
|
413
|
+
requiredFeatures: features,
|
|
414
|
+
requiredLimits: limits,
|
|
415
|
+
});
|
|
416
|
+
} catch (error) {
|
|
417
|
+
lastError = error;
|
|
418
|
+
const label = features.length > 0 ? features.join(', ') : 'none';
|
|
419
|
+
log.warn('GPU', 'Failed to request device with features [' + label + ']: ' + error.message);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
throw lastError || new Error('Failed to request WebGPU device');
|
|
423
|
+
}
|
|
424
|
+
|
|
372
425
|
|
|
373
426
|
function buildLimits(adapter) {
|
|
374
427
|
const adapterLimits = adapter.limits;
|
|
@@ -457,17 +510,7 @@ export async function initDevice() {
|
|
|
457
510
|
// Get adapter info (adapter.info is synchronous in modern WebGPU)
|
|
458
511
|
const adapterInfo = adapter.info || { vendor: 'unknown', architecture: 'unknown', device: 'unknown', description: '' };
|
|
459
512
|
|
|
460
|
-
|
|
461
|
-
gpuDevice = await adapter.requestDevice({
|
|
462
|
-
requiredFeatures: requestedFeatures,
|
|
463
|
-
requiredLimits: limits,
|
|
464
|
-
});
|
|
465
|
-
} catch (e) {
|
|
466
|
-
// Fallback: request device without optional features
|
|
467
|
-
const lostFeatures = requestedFeatures.length > 0 ? requestedFeatures.join(', ') : 'none';
|
|
468
|
-
log.warn('GPU', 'Failed to request device with features [' + lostFeatures + '], trying minimal config: ' + (e).message);
|
|
469
|
-
gpuDevice = await adapter.requestDevice();
|
|
470
|
-
}
|
|
513
|
+
gpuDevice = await requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits);
|
|
471
514
|
|
|
472
515
|
if (!gpuDevice) {
|
|
473
516
|
throw createDopplerError(ERROR_CODES.GPU_DEVICE_FAILED, 'Failed to create WebGPU device');
|
|
@@ -50,7 +50,10 @@ Three ways to make kernels flexible:
|
|
|
50
50
|
| Change per dispatch | select different | recompile | yes |
|
|
51
51
|
| Code duplication | high | minimal | none |
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
Use the topology test from the WGSL style guide. Entry points are for algorithm
|
|
54
|
+
or synchronization changes. Override constants are for model/load-time
|
|
55
|
+
parameters. Uniforms are for per-dispatch values. The registry records which
|
|
56
|
+
choice a variant made; WGSL files must not become a second policy layer.
|
|
54
57
|
|
|
55
58
|
## Entry Points
|
|
56
59
|
|
|
@@ -123,6 +126,10 @@ Kernel operation IDs, variant IDs, WGSL filenames, entry points, feature
|
|
|
123
126
|
requirements, bindings, uniforms, and metadata are defined in
|
|
124
127
|
`src/config/kernels/registry.json`.
|
|
125
128
|
|
|
129
|
+
The management front door is
|
|
130
|
+
[`docs/developer-guides/config-source-of-truth.md`](../../../docs/developer-guides/config-source-of-truth.md#wgsl-kernel-management).
|
|
131
|
+
Do not add separate kernel inventories in docs, scripts, or runtime helpers.
|
|
132
|
+
|
|
126
133
|
Use `npm run kernels:registry:check` to audit the on-disk WGSL inventory
|
|
127
134
|
against the registry. This README must not define separate filename suffix
|
|
128
135
|
rules.
|
|
@@ -141,12 +141,14 @@ async function runAttentionBackwardCore(
|
|
|
141
141
|
dV = await runMatmul(sTransposed, dHead.buffer, seqLen, headDim, seqLen, {
|
|
142
142
|
transposeB: false,
|
|
143
143
|
bDtype: 'f32',
|
|
144
|
+
outputDtype: 'f32',
|
|
144
145
|
});
|
|
145
146
|
|
|
146
147
|
vTransposed = await runTranspose(vHead, seqLen, headDim);
|
|
147
148
|
dS = await runMatmul(dHead, vTransposed.buffer, seqLen, seqLen, headDim, {
|
|
148
149
|
transposeB: false,
|
|
149
150
|
bDtype: 'f32',
|
|
151
|
+
outputDtype: 'f32',
|
|
150
152
|
});
|
|
151
153
|
dQK = causal
|
|
152
154
|
? await runBackwardKernel(
|
|
@@ -166,12 +168,14 @@ async function runAttentionBackwardCore(
|
|
|
166
168
|
transposeB: false,
|
|
167
169
|
alpha: scale,
|
|
168
170
|
bDtype: 'f32',
|
|
171
|
+
outputDtype: 'f32',
|
|
169
172
|
});
|
|
170
173
|
dQKTransposed = await runTranspose(dQK, seqLen, seqLen);
|
|
171
174
|
dK = await runMatmul(dQKTransposed, qHead.buffer, seqLen, headDim, seqLen, {
|
|
172
175
|
transposeB: false,
|
|
173
176
|
alpha: scale,
|
|
174
177
|
bDtype: 'f32',
|
|
178
|
+
outputDtype: 'f32',
|
|
175
179
|
});
|
|
176
180
|
|
|
177
181
|
const copyEncoder = getDevice().createCommandEncoder();
|
|
@@ -240,12 +244,14 @@ async function runAttentionBackwardCore(
|
|
|
240
244
|
dV = await recordMatmul(recorder, sTransposed, dHead.buffer, seqLen, headDim, seqLen, {
|
|
241
245
|
transposeB: false,
|
|
242
246
|
bDtype: 'f32',
|
|
247
|
+
outputDtype: 'f32',
|
|
243
248
|
});
|
|
244
249
|
|
|
245
250
|
vTransposed = await recordTranspose(recorder, vHead, seqLen, headDim);
|
|
246
251
|
dS = await recordMatmul(recorder, dHead, vTransposed.buffer, seqLen, seqLen, headDim, {
|
|
247
252
|
transposeB: false,
|
|
248
253
|
bDtype: 'f32',
|
|
254
|
+
outputDtype: 'f32',
|
|
249
255
|
});
|
|
250
256
|
dQK = causal
|
|
251
257
|
? await recordBackwardKernel(
|
|
@@ -266,12 +272,14 @@ async function runAttentionBackwardCore(
|
|
|
266
272
|
transposeB: false,
|
|
267
273
|
alpha: scale,
|
|
268
274
|
bDtype: 'f32',
|
|
275
|
+
outputDtype: 'f32',
|
|
269
276
|
});
|
|
270
277
|
dQKTransposed = await recordTranspose(recorder, dQK, seqLen, seqLen);
|
|
271
278
|
dK = await recordMatmul(recorder, dQKTransposed, qHead.buffer, seqLen, headDim, seqLen, {
|
|
272
279
|
transposeB: false,
|
|
273
280
|
alpha: scale,
|
|
274
281
|
bDtype: 'f32',
|
|
282
|
+
outputDtype: 'f32',
|
|
275
283
|
});
|
|
276
284
|
|
|
277
285
|
encoder.copyBufferToBuffer(dQ.buffer, 0, gradQBuf, qOffset, headBytes);
|
|
@@ -22,7 +22,7 @@ export interface DequantOptions extends OutputBufferOptions, OutputOffsetOptions
|
|
|
22
22
|
/**
|
|
23
23
|
* Select the best dequantization kernel variant
|
|
24
24
|
*/
|
|
25
|
-
export declare function selectDequantKernel(options
|
|
25
|
+
export declare function selectDequantKernel(options: DequantOptions): string;
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* Create bind group layout for dequant operation
|
|
@@ -35,7 +35,7 @@ export declare function createDequantBindGroupLayout(): GPUBindGroupLayout;
|
|
|
35
35
|
export declare function dequantize(
|
|
36
36
|
quantized: GPUBuffer,
|
|
37
37
|
numBlocks: number,
|
|
38
|
-
options
|
|
38
|
+
options: DequantOptions
|
|
39
39
|
): Promise<Tensor>;
|
|
40
40
|
|
|
41
41
|
/**
|
|
@@ -45,7 +45,7 @@ export declare function dequantizeRowwise(
|
|
|
45
45
|
quantized: GPUBuffer,
|
|
46
46
|
rows: number,
|
|
47
47
|
K: number,
|
|
48
|
-
options
|
|
48
|
+
options: OutputBufferOptions & OutputDtypeOptions
|
|
49
49
|
): Promise<Tensor>;
|
|
50
50
|
|
|
51
51
|
/**
|
|
@@ -69,7 +69,7 @@ export declare function dequantizeMXFP4Expert(
|
|
|
69
69
|
numExperts: number,
|
|
70
70
|
outDim: number,
|
|
71
71
|
numGroups: number,
|
|
72
|
-
options
|
|
72
|
+
options: DequantOptions
|
|
73
73
|
): Promise<Tensor>;
|
|
74
74
|
|
|
75
75
|
/**
|
|
@@ -78,7 +78,7 @@ export declare function dequantizeMXFP4Expert(
|
|
|
78
78
|
export declare function dequantizeQ6K(
|
|
79
79
|
quantized: GPUBuffer,
|
|
80
80
|
numBlocks: number,
|
|
81
|
-
options
|
|
81
|
+
options: DequantOptions
|
|
82
82
|
): Promise<Tensor>;
|
|
83
83
|
|
|
84
84
|
/**
|
|
@@ -88,5 +88,5 @@ export declare function recordDequantize(
|
|
|
88
88
|
recorder: CommandRecorder,
|
|
89
89
|
quantized: GPUBuffer,
|
|
90
90
|
numBlocks: number,
|
|
91
|
-
options
|
|
91
|
+
options: DequantOptions
|
|
92
92
|
): Promise<Tensor>;
|
|
@@ -11,10 +11,17 @@ import { releaseUniformBuffer } from '../uniform-cache.js';
|
|
|
11
11
|
import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
|
|
12
12
|
import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
|
|
13
13
|
|
|
14
|
+
function requireOutputDtype(value, label) {
|
|
15
|
+
if (value !== 'f16' && value !== 'f32') {
|
|
16
|
+
throw new Error(`[${label}] outputDtype is required and must be "f16" or "f32", got ${String(value)}.`);
|
|
17
|
+
}
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
14
20
|
|
|
15
21
|
export function selectDequantKernel(options = {}) {
|
|
16
22
|
const capabilities = getKernelCapabilities();
|
|
17
|
-
const { useVec4 = true
|
|
23
|
+
const { useVec4 = true } = options;
|
|
24
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'selectDequantKernel');
|
|
18
25
|
|
|
19
26
|
const wantsF16Out = outputDtype === 'f16' && capabilities.hasF16;
|
|
20
27
|
return selectKernelRuleValue(
|
|
@@ -90,8 +97,8 @@ export async function dequantize(
|
|
|
90
97
|
const {
|
|
91
98
|
outputOffset = 0,
|
|
92
99
|
outputBuffer = null,
|
|
93
|
-
outputDtype = 'f32',
|
|
94
100
|
} = options;
|
|
101
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'dequantize');
|
|
95
102
|
|
|
96
103
|
if (outputDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
97
104
|
throw new Error('[dequantize] f16 output requires shader-f16 support.');
|
|
@@ -160,7 +167,8 @@ export async function dequantizeRowwise(
|
|
|
160
167
|
) {
|
|
161
168
|
const device = getDevice();
|
|
162
169
|
const capabilities = getKernelCapabilities();
|
|
163
|
-
const { outputBuffer = null
|
|
170
|
+
const { outputBuffer = null } = options;
|
|
171
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeRowwise');
|
|
164
172
|
if (outputDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
165
173
|
throw new Error('[dequantizeRowwise] f16 output requires shader-f16 support.');
|
|
166
174
|
}
|
|
@@ -296,11 +304,11 @@ export async function dequantizeMXFP4Expert(
|
|
|
296
304
|
const device = getDevice();
|
|
297
305
|
const {
|
|
298
306
|
outputBuffer = null,
|
|
299
|
-
outputDtype = 'f32',
|
|
300
307
|
modelType = null,
|
|
301
308
|
groupSize = 32,
|
|
302
309
|
dequantTileShape = 'scalar',
|
|
303
310
|
} = options;
|
|
311
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeMXFP4Expert');
|
|
304
312
|
const caps = getKernelCapabilities();
|
|
305
313
|
|
|
306
314
|
const variant = selectKernelRuleValue('dequant', 'mxfp4ExpertVariant', {
|
|
@@ -375,8 +383,11 @@ export async function dequantizeQ6K(
|
|
|
375
383
|
const {
|
|
376
384
|
outputOffset = 0,
|
|
377
385
|
outputBuffer = null,
|
|
378
|
-
outputDtype = 'f16', // Q6_K always outputs f16 for now
|
|
379
386
|
} = options;
|
|
387
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeQ6K');
|
|
388
|
+
if (outputDtype !== 'f16') {
|
|
389
|
+
throw new Error(`[dequantizeQ6K] outputDtype must be "f16", got ${outputDtype}.`);
|
|
390
|
+
}
|
|
380
391
|
|
|
381
392
|
// Q6_K only has f16 output kernel currently
|
|
382
393
|
const pipeline = await getPipelineFast('dequant', 'q6k_f16out');
|
|
@@ -451,8 +462,8 @@ export async function recordDequantize(
|
|
|
451
462
|
const {
|
|
452
463
|
outputOffset = 0,
|
|
453
464
|
outputBuffer = null,
|
|
454
|
-
outputDtype = 'f32',
|
|
455
465
|
} = options;
|
|
466
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'recordDequantize');
|
|
456
467
|
|
|
457
468
|
// Select kernel
|
|
458
469
|
const variant = selectDequantKernel({ ...options, outputDtype });
|
|
@@ -374,12 +374,7 @@ function requireGpuBuffer(buffer, label) {
|
|
|
374
374
|
}
|
|
375
375
|
|
|
376
376
|
function resolveOutputDtype(outputDtype) {
|
|
377
|
-
const normalized = selectRuleValue(
|
|
378
|
-
'shared',
|
|
379
|
-
'dtype',
|
|
380
|
-
'f16OrF32FromDtype',
|
|
381
|
-
{ dtype: outputDtype === undefined ? 'f32' : outputDtype }
|
|
382
|
-
);
|
|
377
|
+
const normalized = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: outputDtype });
|
|
383
378
|
if (normalized === 'f16' || normalized === 'f32') {
|
|
384
379
|
return normalized;
|
|
385
380
|
}
|
|
@@ -26,10 +26,10 @@ export declare function isFusedQ4KDisabled(options?: {
|
|
|
26
26
|
|
|
27
27
|
export declare function toMatmulDtype(dtype: string | null | undefined): 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16';
|
|
28
28
|
|
|
29
|
-
export declare function selectMatmulKernel(options
|
|
29
|
+
export declare function selectMatmulKernel(options: {
|
|
30
30
|
preferF16?: boolean;
|
|
31
31
|
useVec4?: boolean;
|
|
32
|
-
outputDtype
|
|
32
|
+
outputDtype: TensorDtype | 'f16' | 'f32';
|
|
33
33
|
aDtype?: 'f16' | 'f32' | null;
|
|
34
34
|
bDtype?: 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16' | null;
|
|
35
35
|
isPrefill?: boolean;
|
|
@@ -129,13 +129,16 @@ export function selectMatmulKernel(options = {}) {
|
|
|
129
129
|
const {
|
|
130
130
|
preferF16 = true,
|
|
131
131
|
useVec4 = false,
|
|
132
|
-
outputDtype = 'f32',
|
|
133
132
|
aDtype = null,
|
|
134
133
|
bDtype = null,
|
|
135
134
|
isPrefill = false,
|
|
136
135
|
prefillRows = 0,
|
|
137
136
|
transposeB = true,
|
|
138
137
|
} = options;
|
|
138
|
+
const outputDtype = options.outputDtype;
|
|
139
|
+
if (outputDtype !== 'f16' && outputDtype !== 'f32') {
|
|
140
|
+
throw new Error(`[Matmul] selectMatmulKernel requires outputDtype "f16" or "f32", got ${String(outputDtype)}.`);
|
|
141
|
+
}
|
|
139
142
|
const { tiledPrefillMinRows } = getKernelThresholds().matmul;
|
|
140
143
|
|
|
141
144
|
const inputsAreF16 = aDtype === 'f16' && bDtype === 'f16';
|
|
@@ -334,7 +337,11 @@ function resolveMatmulOverride(
|
|
|
334
337
|
if (!weightDtypeOk) {
|
|
335
338
|
const overridePolicy = selectKernelRuleValue(
|
|
336
339
|
'matmul', 'weightDtypeMismatchPolicy',
|
|
337
|
-
{
|
|
340
|
+
{
|
|
341
|
+
requiredWeightDtype,
|
|
342
|
+
actualWeightDtype: bDtype,
|
|
343
|
+
hasSubgroups: capabilities?.hasSubgroups === true,
|
|
344
|
+
}
|
|
338
345
|
);
|
|
339
346
|
if (overridePolicy === 'fallthrough') {
|
|
340
347
|
return null;
|
|
@@ -56,7 +56,7 @@ export declare function isFusedQ4KDisabled(): boolean;
|
|
|
56
56
|
/**
|
|
57
57
|
* Select the best matmul kernel variant
|
|
58
58
|
*/
|
|
59
|
-
export declare function selectMatmulKernel(options
|
|
59
|
+
export declare function selectMatmulKernel(options: MatmulOptions): string;
|
|
60
60
|
|
|
61
61
|
/**
|
|
62
62
|
* Create bind group layout for matmul operation
|
|
@@ -212,7 +212,7 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
|
|
|
212
212
|
return entries;
|
|
213
213
|
}
|
|
214
214
|
|
|
215
|
-
function resolvePreferredWeightDtype(variant, hasQ4KMaterialization) {
|
|
215
|
+
function resolvePreferredWeightDtype(variant, hasQ4KMaterialization, capabilities) {
|
|
216
216
|
if (typeof variant !== 'string' || variant.length === 0) {
|
|
217
217
|
return null;
|
|
218
218
|
}
|
|
@@ -232,9 +232,51 @@ function resolvePreferredWeightDtype(variant, hasQ4KMaterialization) {
|
|
|
232
232
|
return selectKernelRuleValue('matmul', 'preferredWeightDtype', {
|
|
233
233
|
variantWeightDtype,
|
|
234
234
|
hasQ4KMaterialization,
|
|
235
|
+
hasSubgroups: capabilities?.hasSubgroups === true,
|
|
235
236
|
});
|
|
236
237
|
}
|
|
237
238
|
|
|
239
|
+
function buildF16CapabilityErrorDetail({
|
|
240
|
+
role,
|
|
241
|
+
layerIdx,
|
|
242
|
+
pathVariant,
|
|
243
|
+
preferredWeightDtype,
|
|
244
|
+
weightDtype,
|
|
245
|
+
weightLabel,
|
|
246
|
+
weightLayout,
|
|
247
|
+
weightShape,
|
|
248
|
+
hasQ4KMaterialization,
|
|
249
|
+
}) {
|
|
250
|
+
const parts = [];
|
|
251
|
+
if (role) parts.push(`role=${role}`);
|
|
252
|
+
if (Number.isFinite(layerIdx)) parts.push(`layer=${layerIdx}`);
|
|
253
|
+
if (pathVariant) parts.push(`variant=${pathVariant}`);
|
|
254
|
+
if (preferredWeightDtype) parts.push(`preferredWeightDtype=${preferredWeightDtype}`);
|
|
255
|
+
if (weightDtype) parts.push(`weightDtype=${weightDtype}`);
|
|
256
|
+
if (weightLabel) parts.push(`label=${weightLabel}`);
|
|
257
|
+
if (weightLayout) parts.push(`layout=${weightLayout}`);
|
|
258
|
+
if (weightShape) parts.push(`shape=${weightShape}`);
|
|
259
|
+
if (hasQ4KMaterialization) parts.push('q4kMaterialization=true');
|
|
260
|
+
return parts.length > 0 ? ` (${parts.join(', ')})` : '';
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function requireMatmulOutputDtype(dtype, opLabel) {
|
|
264
|
+
if (dtype === 'f16' || dtype === 'f32') {
|
|
265
|
+
return dtype;
|
|
266
|
+
}
|
|
267
|
+
throw new Error(`[${opLabel}] options.outputDtype is required and must be "f16" or "f32", got ${String(dtype)}.`);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function requireMatmulWeightDtype(dtype, opLabel, details = []) {
|
|
271
|
+
if (dtype != null && dtype !== '') {
|
|
272
|
+
return dtype;
|
|
273
|
+
}
|
|
274
|
+
const suffix = details.filter(Boolean).length > 0
|
|
275
|
+
? ` (${details.filter(Boolean).join(', ')})`
|
|
276
|
+
: '';
|
|
277
|
+
throw new Error(`[${opLabel}] B dtype is required for matmul dispatch${suffix}.`);
|
|
278
|
+
}
|
|
279
|
+
|
|
238
280
|
async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
239
281
|
const isRecord = Boolean(recorder);
|
|
240
282
|
const mode = isRecord ? 'record' : 'run';
|
|
@@ -254,7 +296,7 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
254
296
|
const phase = resolveMatmulPhase(M, options.phaseOverride ?? null);
|
|
255
297
|
const pathVariant = getKernelPathMatmulVariant(options.role, phase, options.layerIdx, options.kernelPath);
|
|
256
298
|
const hasQ4KMat = isWeightBuffer(B) && B.materializations?.q4k?.buffer != null;
|
|
257
|
-
const preferredWeightDtype = resolvePreferredWeightDtype(pathVariant, hasQ4KMat);
|
|
299
|
+
const preferredWeightDtype = resolvePreferredWeightDtype(pathVariant, hasQ4KMat, capabilities);
|
|
258
300
|
const resolvedWeight = resolveWeightBufferMaterialization(B, preferredWeightDtype);
|
|
259
301
|
const bBuffer = getBuffer(resolvedWeight);
|
|
260
302
|
const weightDtype = getWeightDtype(resolvedWeight);
|
|
@@ -278,20 +320,32 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
278
320
|
validateMatmulDimensions(opLabel, M, N, K);
|
|
279
321
|
|
|
280
322
|
const aDtype = toMatmulDtype(A.dtype);
|
|
281
|
-
const
|
|
282
|
-
|
|
323
|
+
const rawBDtype = requireMatmulWeightDtype(weightDtype ?? options.bDtype, opLabel, [
|
|
324
|
+
options.role ? `role=${options.role}` : null,
|
|
325
|
+
Number.isFinite(options.layerIdx) ? `layer=${options.layerIdx}` : null,
|
|
326
|
+
weightLabel ? `label=${weightLabel}` : null,
|
|
327
|
+
]);
|
|
328
|
+
const bDtype = toMatmulDtype(rawBDtype);
|
|
329
|
+
const requestedOutputDtype = requireMatmulOutputDtype(options.outputDtype, opLabel);
|
|
283
330
|
|
|
284
331
|
if (bDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
285
|
-
|
|
332
|
+
const detail = buildF16CapabilityErrorDetail({
|
|
333
|
+
role: options.role,
|
|
334
|
+
layerIdx: options.layerIdx,
|
|
335
|
+
pathVariant,
|
|
336
|
+
preferredWeightDtype,
|
|
337
|
+
weightDtype,
|
|
338
|
+
weightLabel,
|
|
339
|
+
weightLayout,
|
|
340
|
+
weightShape,
|
|
341
|
+
hasQ4KMaterialization: hasQ4KMat,
|
|
342
|
+
});
|
|
343
|
+
throw new Error(`[${opLabel}] f16 weights require shader-f16 support.${detail}`);
|
|
286
344
|
}
|
|
287
345
|
if (requestedOutputDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
288
346
|
throw new Error(`[${opLabel}] f16 output requires shader-f16 support.`);
|
|
289
347
|
}
|
|
290
348
|
|
|
291
|
-
if (!isRecord && isTraceEnabled('kernels') && !weightDtype && !options.bDtype && M <= 2) {
|
|
292
|
-
log.warn('Matmul', `runMatmul: B buffer dtype unknown! size=${bBuffer.size}, M=${M}, N=${N}, K=${K}. Assuming f32.`);
|
|
293
|
-
}
|
|
294
|
-
|
|
295
349
|
validateMatmulOffsets(opLabel, aOffset, bOffset, cOffset);
|
|
296
350
|
|
|
297
351
|
const runtimeSession = getRuntimeConfig().inference?.session;
|
package/src/gpu/kernels/moe.d.ts
CHANGED
|
@@ -19,6 +19,10 @@ export interface MoEOptions extends OutputBufferOptions {
|
|
|
19
19
|
perExpertScale?: GPUBuffer | null;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
export interface MoEScatterOptions extends MoEOptions {
|
|
23
|
+
weightsDtype: 'f16' | 'f32';
|
|
24
|
+
}
|
|
25
|
+
|
|
22
26
|
/** MoE gather result */
|
|
23
27
|
export interface MoEGatherResult {
|
|
24
28
|
gathered: Tensor;
|
|
@@ -105,5 +109,5 @@ export declare function runScatterAddDynamic(
|
|
|
105
109
|
numTokens: number,
|
|
106
110
|
hiddenSize: number,
|
|
107
111
|
topK: number,
|
|
108
|
-
options
|
|
112
|
+
options: MoEScatterOptions
|
|
109
113
|
): Promise<Tensor>;
|