doppler-gpu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -31
- package/assets/doppler-runtime-map.svg +1 -3
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +4 -1
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.js +6 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
- package/assets/doppler-readme-hero.svg +0 -84
|
@@ -28,6 +28,15 @@ function asNumberArray(value) {
|
|
|
28
28
|
return normalized.map((entry) => Math.trunc(entry));
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
function asPositiveNumberArray(value) {
|
|
32
|
+
if (!Array.isArray(value)) return null;
|
|
33
|
+
const normalized = value.map((entry) => asFiniteNumber(entry));
|
|
34
|
+
if (normalized.some((entry) => entry == null || entry <= 0)) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
return normalized.map((entry) => Number(entry));
|
|
38
|
+
}
|
|
39
|
+
|
|
31
40
|
function normalizeRoPEType(value) {
|
|
32
41
|
if (typeof value !== 'string') return null;
|
|
33
42
|
const normalized = value.trim().toLowerCase();
|
|
@@ -90,6 +99,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
|
|
|
90
99
|
yarnBetaFast: null,
|
|
91
100
|
yarnBetaSlow: null,
|
|
92
101
|
yarnOriginalMaxPos: null,
|
|
102
|
+
longropeShortFactor: null,
|
|
103
|
+
longropeLongFactor: null,
|
|
104
|
+
longropeOriginalMaxPos: null,
|
|
93
105
|
};
|
|
94
106
|
}
|
|
95
107
|
|
|
@@ -98,6 +110,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
|
|
|
98
110
|
let yarnBetaFast = null;
|
|
99
111
|
let yarnBetaSlow = null;
|
|
100
112
|
let yarnOriginalMaxPos = null;
|
|
113
|
+
let longropeShortFactor = null;
|
|
114
|
+
let longropeLongFactor = null;
|
|
115
|
+
let longropeOriginalMaxPos = null;
|
|
101
116
|
|
|
102
117
|
if (ropeScalingType == null) {
|
|
103
118
|
if (factor != null && factor > 0 && factor !== 1.0) {
|
|
@@ -123,6 +138,23 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
|
|
|
123
138
|
yarnBetaSlow = betaSlow;
|
|
124
139
|
yarnOriginalMaxPos = origMaxPos;
|
|
125
140
|
}
|
|
141
|
+
if (ropeScalingType === 'longrope') {
|
|
142
|
+
const shortFactor = asPositiveNumberArray(ropeScalingConfig.short_factor);
|
|
143
|
+
const longFactor = asPositiveNumberArray(ropeScalingConfig.long_factor);
|
|
144
|
+
const origMaxPos = asFiniteNumber(ropeScalingConfig.original_max_position_embeddings);
|
|
145
|
+
if (!shortFactor || !longFactor || origMaxPos == null) {
|
|
146
|
+
throw new Error(
|
|
147
|
+
'LongRoPE scaling detected but required params missing in HF config. ' +
|
|
148
|
+
'LongRoPE requires short_factor, long_factor, and original_max_position_embeddings. ' +
|
|
149
|
+
`Got: short_factor=${Array.isArray(ropeScalingConfig.short_factor) ? ropeScalingConfig.short_factor.length : ropeScalingConfig.short_factor}, ` +
|
|
150
|
+
`long_factor=${Array.isArray(ropeScalingConfig.long_factor) ? ropeScalingConfig.long_factor.length : ropeScalingConfig.long_factor}, ` +
|
|
151
|
+
`original_max_position_embeddings=${origMaxPos}`
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
longropeShortFactor = shortFactor;
|
|
155
|
+
longropeLongFactor = longFactor;
|
|
156
|
+
longropeOriginalMaxPos = origMaxPos;
|
|
157
|
+
}
|
|
126
158
|
|
|
127
159
|
return {
|
|
128
160
|
ropeScalingType,
|
|
@@ -130,6 +162,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
|
|
|
130
162
|
yarnBetaFast,
|
|
131
163
|
yarnBetaSlow,
|
|
132
164
|
yarnOriginalMaxPos,
|
|
165
|
+
longropeShortFactor,
|
|
166
|
+
longropeLongFactor,
|
|
167
|
+
longropeOriginalMaxPos,
|
|
133
168
|
};
|
|
134
169
|
}
|
|
135
170
|
|
|
@@ -140,7 +175,9 @@ function hasScalingDirective(ropeScalingConfig) {
|
|
|
140
175
|
|| ropeScalingConfig.factor != null
|
|
141
176
|
|| ropeScalingConfig.beta_fast != null
|
|
142
177
|
|| ropeScalingConfig.beta_slow != null
|
|
143
|
-
|| ropeScalingConfig.original_max_position_embeddings != null
|
|
178
|
+
|| ropeScalingConfig.original_max_position_embeddings != null
|
|
179
|
+
|| ropeScalingConfig.short_factor != null
|
|
180
|
+
|| ropeScalingConfig.long_factor != null;
|
|
144
181
|
}
|
|
145
182
|
|
|
146
183
|
function hasMeaningfulScalingConfig(resolvedScaling) {
|
|
@@ -149,7 +186,10 @@ function hasMeaningfulScalingConfig(resolvedScaling) {
|
|
|
149
186
|
|| resolvedScaling.ropeScalingFactor !== DEFAULT_MANIFEST_INFERENCE.rope.ropeScalingFactor
|
|
150
187
|
|| resolvedScaling.yarnBetaFast != null
|
|
151
188
|
|| resolvedScaling.yarnBetaSlow != null
|
|
152
|
-
|| resolvedScaling.yarnOriginalMaxPos != null
|
|
189
|
+
|| resolvedScaling.yarnOriginalMaxPos != null
|
|
190
|
+
|| resolvedScaling.longropeShortFactor != null
|
|
191
|
+
|| resolvedScaling.longropeLongFactor != null
|
|
192
|
+
|| resolvedScaling.longropeOriginalMaxPos != null;
|
|
153
193
|
}
|
|
154
194
|
|
|
155
195
|
function isSameScalingConfig(left, right) {
|
|
@@ -157,7 +197,10 @@ function isSameScalingConfig(left, right) {
|
|
|
157
197
|
&& left.ropeScalingFactor === right.ropeScalingFactor
|
|
158
198
|
&& left.yarnBetaFast === right.yarnBetaFast
|
|
159
199
|
&& left.yarnBetaSlow === right.yarnBetaSlow
|
|
160
|
-
&& left.yarnOriginalMaxPos === right.yarnOriginalMaxPos
|
|
200
|
+
&& left.yarnOriginalMaxPos === right.yarnOriginalMaxPos
|
|
201
|
+
&& JSON.stringify(left.longropeShortFactor ?? null) === JSON.stringify(right.longropeShortFactor ?? null)
|
|
202
|
+
&& JSON.stringify(left.longropeLongFactor ?? null) === JSON.stringify(right.longropeLongFactor ?? null)
|
|
203
|
+
&& left.longropeOriginalMaxPos === right.longropeOriginalMaxPos;
|
|
161
204
|
}
|
|
162
205
|
|
|
163
206
|
function failOnConflictingScaling(sourceLabel, canonicalScaling, candidateScaling) {
|
|
@@ -210,6 +253,9 @@ export function buildRoPEConfig(converterInference, config) {
|
|
|
210
253
|
yarnBetaFast: configuredRoPE.yarnBetaFast ?? null,
|
|
211
254
|
yarnBetaSlow: configuredRoPE.yarnBetaSlow ?? null,
|
|
212
255
|
yarnOriginalMaxPos: configuredRoPE.yarnOriginalMaxPos ?? null,
|
|
256
|
+
longropeShortFactor: configuredRoPE.longropeShortFactor ?? null,
|
|
257
|
+
longropeLongFactor: configuredRoPE.longropeLongFactor ?? null,
|
|
258
|
+
longropeOriginalMaxPos: configuredRoPE.longropeOriginalMaxPos ?? null,
|
|
213
259
|
};
|
|
214
260
|
|
|
215
261
|
if (ropeScaling) {
|
|
@@ -253,6 +299,9 @@ export function buildRoPEConfig(converterInference, config) {
|
|
|
253
299
|
yarnBetaFast: configuredRoPE.ropeLocalYarnBetaFast ?? globalScaling.yarnBetaFast,
|
|
254
300
|
yarnBetaSlow: configuredRoPE.ropeLocalYarnBetaSlow ?? globalScaling.yarnBetaSlow,
|
|
255
301
|
yarnOriginalMaxPos: configuredRoPE.ropeLocalYarnOriginalMaxPos ?? globalScaling.yarnOriginalMaxPos,
|
|
302
|
+
longropeShortFactor: globalScaling.longropeShortFactor,
|
|
303
|
+
longropeLongFactor: globalScaling.longropeLongFactor,
|
|
304
|
+
longropeOriginalMaxPos: globalScaling.longropeOriginalMaxPos,
|
|
256
305
|
}
|
|
257
306
|
: { ...globalScaling };
|
|
258
307
|
if (ropeScaling) {
|
|
@@ -323,6 +372,9 @@ export function buildRoPEConfig(converterInference, config) {
|
|
|
323
372
|
yarnBetaFast: globalScaling.yarnBetaFast,
|
|
324
373
|
yarnBetaSlow: globalScaling.yarnBetaSlow,
|
|
325
374
|
yarnOriginalMaxPos: globalScaling.yarnOriginalMaxPos,
|
|
375
|
+
longropeShortFactor: globalScaling.longropeShortFactor,
|
|
376
|
+
longropeLongFactor: globalScaling.longropeLongFactor,
|
|
377
|
+
longropeOriginalMaxPos: globalScaling.longropeOriginalMaxPos,
|
|
326
378
|
ropeLocalScalingType: localScaling.ropeScalingType,
|
|
327
379
|
ropeLocalScalingFactor: localScaling.ropeScalingFactor,
|
|
328
380
|
ropeLocalYarnBetaFast: localScaling.yarnBetaFast,
|
|
@@ -421,7 +421,7 @@ function createDistillStudentProjectionModelFixture(overrides = {}, options = {}
|
|
|
421
421
|
const rows = Number.isFinite(inputTensor?.shape?.[0]) ? inputTensor.shape[0] : 1;
|
|
422
422
|
return tape.record(
|
|
423
423
|
OpType.MATMUL,
|
|
424
|
-
(a, b) => runMatmul(a, b, rows, outputDim, embeddingDim, { transposeB: false }),
|
|
424
|
+
(a, b) => runMatmul(a, b, rows, outputDim, embeddingDim, { transposeB: false, outputDtype: 'f32' }),
|
|
425
425
|
[inputTensor, projectionWeight],
|
|
426
426
|
{ M: rows, N: outputDim, K: embeddingDim, transposeB: false }
|
|
427
427
|
);
|
|
@@ -400,7 +400,7 @@ function createToyLoraModel(workload) {
|
|
|
400
400
|
const batchSize = Number.isInteger(inputTensor?.shape?.[0]) ? inputTensor.shape[0] : 1;
|
|
401
401
|
const baseLogits = await tape.record(
|
|
402
402
|
OpType.MATMUL,
|
|
403
|
-
(a, b) => runMatmul(a, b, batchSize, 2, 3, { transposeB: false }),
|
|
403
|
+
(a, b) => runMatmul(a, b, batchSize, 2, 3, { transposeB: false, outputDtype: 'f32' }),
|
|
404
404
|
[inputTensor, baseWeight],
|
|
405
405
|
{ M: batchSize, N: 2, K: 3, transposeB: false }
|
|
406
406
|
);
|
|
@@ -68,13 +68,13 @@ export class LoraAdapter {
|
|
|
68
68
|
const [tokens] = input.shape;
|
|
69
69
|
const down = await tape.record(
|
|
70
70
|
OpType.MATMUL,
|
|
71
|
-
(a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0]),
|
|
71
|
+
(a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0], { outputDtype: input.dtype }),
|
|
72
72
|
[input, this.A],
|
|
73
73
|
{ M: tokens, N: this.rank, K: this.A.shape[0] }
|
|
74
74
|
);
|
|
75
75
|
const up = await tape.record(
|
|
76
76
|
OpType.MATMUL,
|
|
77
|
-
(a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank),
|
|
77
|
+
(a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank, { outputDtype: down.dtype }),
|
|
78
78
|
[down, this.B],
|
|
79
79
|
{ M: tokens, N: this.B.shape[1], K: this.rank }
|
|
80
80
|
);
|
|
@@ -821,7 +821,7 @@ export function createToyModelFixture(overrides = {}) {
|
|
|
821
821
|
async forward(inputTensor, tape) {
|
|
822
822
|
return tape.record(
|
|
823
823
|
OpType.MATMUL,
|
|
824
|
-
(a, b) => runMatmul(a, b, 2, 2, 3, { transposeB: false }),
|
|
824
|
+
(a, b) => runMatmul(a, b, 2, 2, 3, { transposeB: false, outputDtype: 'f32' }),
|
|
825
825
|
[inputTensor, baseWeight],
|
|
826
826
|
{ M: 2, N: 2, K: 3, transposeB: false }
|
|
827
827
|
);
|
|
@@ -50,7 +50,10 @@ Three ways to make kernels flexible:
|
|
|
50
50
|
| Change per dispatch | select different | recompile | yes |
|
|
51
51
|
| Code duplication | high | minimal | none |
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
Use the topology test from the WGSL style guide. Entry points are for algorithm
|
|
54
|
+
or synchronization changes. Override constants are for model/load-time
|
|
55
|
+
parameters. Uniforms are for per-dispatch values. The registry records which
|
|
56
|
+
choice a variant made; WGSL files must not become a second policy layer.
|
|
54
57
|
|
|
55
58
|
## Entry Points
|
|
56
59
|
|
|
@@ -123,6 +126,10 @@ Kernel operation IDs, variant IDs, WGSL filenames, entry points, feature
|
|
|
123
126
|
requirements, bindings, uniforms, and metadata are defined in
|
|
124
127
|
`src/config/kernels/registry.json`.
|
|
125
128
|
|
|
129
|
+
The management front door is
|
|
130
|
+
[`docs/developer-guides/config-source-of-truth.md`](../../../docs/developer-guides/config-source-of-truth.md#wgsl-kernel-management).
|
|
131
|
+
Do not add separate kernel inventories in docs, scripts, or runtime helpers.
|
|
132
|
+
|
|
126
133
|
Use `npm run kernels:registry:check` to audit the on-disk WGSL inventory
|
|
127
134
|
against the registry. This README must not define separate filename suffix
|
|
128
135
|
rules.
|
|
@@ -141,12 +141,14 @@ async function runAttentionBackwardCore(
|
|
|
141
141
|
dV = await runMatmul(sTransposed, dHead.buffer, seqLen, headDim, seqLen, {
|
|
142
142
|
transposeB: false,
|
|
143
143
|
bDtype: 'f32',
|
|
144
|
+
outputDtype: 'f32',
|
|
144
145
|
});
|
|
145
146
|
|
|
146
147
|
vTransposed = await runTranspose(vHead, seqLen, headDim);
|
|
147
148
|
dS = await runMatmul(dHead, vTransposed.buffer, seqLen, seqLen, headDim, {
|
|
148
149
|
transposeB: false,
|
|
149
150
|
bDtype: 'f32',
|
|
151
|
+
outputDtype: 'f32',
|
|
150
152
|
});
|
|
151
153
|
dQK = causal
|
|
152
154
|
? await runBackwardKernel(
|
|
@@ -166,12 +168,14 @@ async function runAttentionBackwardCore(
|
|
|
166
168
|
transposeB: false,
|
|
167
169
|
alpha: scale,
|
|
168
170
|
bDtype: 'f32',
|
|
171
|
+
outputDtype: 'f32',
|
|
169
172
|
});
|
|
170
173
|
dQKTransposed = await runTranspose(dQK, seqLen, seqLen);
|
|
171
174
|
dK = await runMatmul(dQKTransposed, qHead.buffer, seqLen, headDim, seqLen, {
|
|
172
175
|
transposeB: false,
|
|
173
176
|
alpha: scale,
|
|
174
177
|
bDtype: 'f32',
|
|
178
|
+
outputDtype: 'f32',
|
|
175
179
|
});
|
|
176
180
|
|
|
177
181
|
const copyEncoder = getDevice().createCommandEncoder();
|
|
@@ -240,12 +244,14 @@ async function runAttentionBackwardCore(
|
|
|
240
244
|
dV = await recordMatmul(recorder, sTransposed, dHead.buffer, seqLen, headDim, seqLen, {
|
|
241
245
|
transposeB: false,
|
|
242
246
|
bDtype: 'f32',
|
|
247
|
+
outputDtype: 'f32',
|
|
243
248
|
});
|
|
244
249
|
|
|
245
250
|
vTransposed = await recordTranspose(recorder, vHead, seqLen, headDim);
|
|
246
251
|
dS = await recordMatmul(recorder, dHead, vTransposed.buffer, seqLen, seqLen, headDim, {
|
|
247
252
|
transposeB: false,
|
|
248
253
|
bDtype: 'f32',
|
|
254
|
+
outputDtype: 'f32',
|
|
249
255
|
});
|
|
250
256
|
dQK = causal
|
|
251
257
|
? await recordBackwardKernel(
|
|
@@ -266,12 +272,14 @@ async function runAttentionBackwardCore(
|
|
|
266
272
|
transposeB: false,
|
|
267
273
|
alpha: scale,
|
|
268
274
|
bDtype: 'f32',
|
|
275
|
+
outputDtype: 'f32',
|
|
269
276
|
});
|
|
270
277
|
dQKTransposed = await recordTranspose(recorder, dQK, seqLen, seqLen);
|
|
271
278
|
dK = await recordMatmul(recorder, dQKTransposed, qHead.buffer, seqLen, headDim, seqLen, {
|
|
272
279
|
transposeB: false,
|
|
273
280
|
alpha: scale,
|
|
274
281
|
bDtype: 'f32',
|
|
282
|
+
outputDtype: 'f32',
|
|
275
283
|
});
|
|
276
284
|
|
|
277
285
|
encoder.copyBufferToBuffer(dQ.buffer, 0, gradQBuf, qOffset, headBytes);
|
|
@@ -22,7 +22,7 @@ export interface DequantOptions extends OutputBufferOptions, OutputOffsetOptions
|
|
|
22
22
|
/**
|
|
23
23
|
* Select the best dequantization kernel variant
|
|
24
24
|
*/
|
|
25
|
-
export declare function selectDequantKernel(options
|
|
25
|
+
export declare function selectDequantKernel(options: DequantOptions): string;
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* Create bind group layout for dequant operation
|
|
@@ -35,7 +35,7 @@ export declare function createDequantBindGroupLayout(): GPUBindGroupLayout;
|
|
|
35
35
|
export declare function dequantize(
|
|
36
36
|
quantized: GPUBuffer,
|
|
37
37
|
numBlocks: number,
|
|
38
|
-
options
|
|
38
|
+
options: DequantOptions
|
|
39
39
|
): Promise<Tensor>;
|
|
40
40
|
|
|
41
41
|
/**
|
|
@@ -45,7 +45,7 @@ export declare function dequantizeRowwise(
|
|
|
45
45
|
quantized: GPUBuffer,
|
|
46
46
|
rows: number,
|
|
47
47
|
K: number,
|
|
48
|
-
options
|
|
48
|
+
options: OutputBufferOptions & OutputDtypeOptions
|
|
49
49
|
): Promise<Tensor>;
|
|
50
50
|
|
|
51
51
|
/**
|
|
@@ -69,7 +69,7 @@ export declare function dequantizeMXFP4Expert(
|
|
|
69
69
|
numExperts: number,
|
|
70
70
|
outDim: number,
|
|
71
71
|
numGroups: number,
|
|
72
|
-
options
|
|
72
|
+
options: DequantOptions
|
|
73
73
|
): Promise<Tensor>;
|
|
74
74
|
|
|
75
75
|
/**
|
|
@@ -78,7 +78,7 @@ export declare function dequantizeMXFP4Expert(
|
|
|
78
78
|
export declare function dequantizeQ6K(
|
|
79
79
|
quantized: GPUBuffer,
|
|
80
80
|
numBlocks: number,
|
|
81
|
-
options
|
|
81
|
+
options: DequantOptions
|
|
82
82
|
): Promise<Tensor>;
|
|
83
83
|
|
|
84
84
|
/**
|
|
@@ -88,5 +88,5 @@ export declare function recordDequantize(
|
|
|
88
88
|
recorder: CommandRecorder,
|
|
89
89
|
quantized: GPUBuffer,
|
|
90
90
|
numBlocks: number,
|
|
91
|
-
options
|
|
91
|
+
options: DequantOptions
|
|
92
92
|
): Promise<Tensor>;
|
|
@@ -11,10 +11,17 @@ import { releaseUniformBuffer } from '../uniform-cache.js';
|
|
|
11
11
|
import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
|
|
12
12
|
import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
|
|
13
13
|
|
|
14
|
+
function requireOutputDtype(value, label) {
|
|
15
|
+
if (value !== 'f16' && value !== 'f32') {
|
|
16
|
+
throw new Error(`[${label}] outputDtype is required and must be "f16" or "f32", got ${String(value)}.`);
|
|
17
|
+
}
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
14
20
|
|
|
15
21
|
export function selectDequantKernel(options = {}) {
|
|
16
22
|
const capabilities = getKernelCapabilities();
|
|
17
|
-
const { useVec4 = true
|
|
23
|
+
const { useVec4 = true } = options;
|
|
24
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'selectDequantKernel');
|
|
18
25
|
|
|
19
26
|
const wantsF16Out = outputDtype === 'f16' && capabilities.hasF16;
|
|
20
27
|
return selectKernelRuleValue(
|
|
@@ -90,8 +97,8 @@ export async function dequantize(
|
|
|
90
97
|
const {
|
|
91
98
|
outputOffset = 0,
|
|
92
99
|
outputBuffer = null,
|
|
93
|
-
outputDtype = 'f32',
|
|
94
100
|
} = options;
|
|
101
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'dequantize');
|
|
95
102
|
|
|
96
103
|
if (outputDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
97
104
|
throw new Error('[dequantize] f16 output requires shader-f16 support.');
|
|
@@ -160,7 +167,8 @@ export async function dequantizeRowwise(
|
|
|
160
167
|
) {
|
|
161
168
|
const device = getDevice();
|
|
162
169
|
const capabilities = getKernelCapabilities();
|
|
163
|
-
const { outputBuffer = null
|
|
170
|
+
const { outputBuffer = null } = options;
|
|
171
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeRowwise');
|
|
164
172
|
if (outputDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
165
173
|
throw new Error('[dequantizeRowwise] f16 output requires shader-f16 support.');
|
|
166
174
|
}
|
|
@@ -296,11 +304,11 @@ export async function dequantizeMXFP4Expert(
|
|
|
296
304
|
const device = getDevice();
|
|
297
305
|
const {
|
|
298
306
|
outputBuffer = null,
|
|
299
|
-
outputDtype = 'f32',
|
|
300
307
|
modelType = null,
|
|
301
308
|
groupSize = 32,
|
|
302
309
|
dequantTileShape = 'scalar',
|
|
303
310
|
} = options;
|
|
311
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeMXFP4Expert');
|
|
304
312
|
const caps = getKernelCapabilities();
|
|
305
313
|
|
|
306
314
|
const variant = selectKernelRuleValue('dequant', 'mxfp4ExpertVariant', {
|
|
@@ -375,8 +383,11 @@ export async function dequantizeQ6K(
|
|
|
375
383
|
const {
|
|
376
384
|
outputOffset = 0,
|
|
377
385
|
outputBuffer = null,
|
|
378
|
-
outputDtype = 'f16', // Q6_K always outputs f16 for now
|
|
379
386
|
} = options;
|
|
387
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeQ6K');
|
|
388
|
+
if (outputDtype !== 'f16') {
|
|
389
|
+
throw new Error(`[dequantizeQ6K] outputDtype must be "f16", got ${outputDtype}.`);
|
|
390
|
+
}
|
|
380
391
|
|
|
381
392
|
// Q6_K only has f16 output kernel currently
|
|
382
393
|
const pipeline = await getPipelineFast('dequant', 'q6k_f16out');
|
|
@@ -451,8 +462,8 @@ export async function recordDequantize(
|
|
|
451
462
|
const {
|
|
452
463
|
outputOffset = 0,
|
|
453
464
|
outputBuffer = null,
|
|
454
|
-
outputDtype = 'f32',
|
|
455
465
|
} = options;
|
|
466
|
+
const outputDtype = requireOutputDtype(options.outputDtype, 'recordDequantize');
|
|
456
467
|
|
|
457
468
|
// Select kernel
|
|
458
469
|
const variant = selectDequantKernel({ ...options, outputDtype });
|
|
@@ -374,12 +374,7 @@ function requireGpuBuffer(buffer, label) {
|
|
|
374
374
|
}
|
|
375
375
|
|
|
376
376
|
function resolveOutputDtype(outputDtype) {
|
|
377
|
-
const normalized = selectRuleValue(
|
|
378
|
-
'shared',
|
|
379
|
-
'dtype',
|
|
380
|
-
'f16OrF32FromDtype',
|
|
381
|
-
{ dtype: outputDtype === undefined ? 'f32' : outputDtype }
|
|
382
|
-
);
|
|
377
|
+
const normalized = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: outputDtype });
|
|
383
378
|
if (normalized === 'f16' || normalized === 'f32') {
|
|
384
379
|
return normalized;
|
|
385
380
|
}
|
|
@@ -26,10 +26,10 @@ export declare function isFusedQ4KDisabled(options?: {
|
|
|
26
26
|
|
|
27
27
|
export declare function toMatmulDtype(dtype: string | null | undefined): 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16';
|
|
28
28
|
|
|
29
|
-
export declare function selectMatmulKernel(options
|
|
29
|
+
export declare function selectMatmulKernel(options: {
|
|
30
30
|
preferF16?: boolean;
|
|
31
31
|
useVec4?: boolean;
|
|
32
|
-
outputDtype
|
|
32
|
+
outputDtype: TensorDtype | 'f16' | 'f32';
|
|
33
33
|
aDtype?: 'f16' | 'f32' | null;
|
|
34
34
|
bDtype?: 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16' | null;
|
|
35
35
|
isPrefill?: boolean;
|
|
@@ -129,13 +129,16 @@ export function selectMatmulKernel(options = {}) {
|
|
|
129
129
|
const {
|
|
130
130
|
preferF16 = true,
|
|
131
131
|
useVec4 = false,
|
|
132
|
-
outputDtype = 'f32',
|
|
133
132
|
aDtype = null,
|
|
134
133
|
bDtype = null,
|
|
135
134
|
isPrefill = false,
|
|
136
135
|
prefillRows = 0,
|
|
137
136
|
transposeB = true,
|
|
138
137
|
} = options;
|
|
138
|
+
const outputDtype = options.outputDtype;
|
|
139
|
+
if (outputDtype !== 'f16' && outputDtype !== 'f32') {
|
|
140
|
+
throw new Error(`[Matmul] selectMatmulKernel requires outputDtype "f16" or "f32", got ${String(outputDtype)}.`);
|
|
141
|
+
}
|
|
139
142
|
const { tiledPrefillMinRows } = getKernelThresholds().matmul;
|
|
140
143
|
|
|
141
144
|
const inputsAreF16 = aDtype === 'f16' && bDtype === 'f16';
|
|
@@ -56,7 +56,7 @@ export declare function isFusedQ4KDisabled(): boolean;
|
|
|
56
56
|
/**
|
|
57
57
|
* Select the best matmul kernel variant
|
|
58
58
|
*/
|
|
59
|
-
export declare function selectMatmulKernel(options
|
|
59
|
+
export declare function selectMatmulKernel(options: MatmulOptions): string;
|
|
60
60
|
|
|
61
61
|
/**
|
|
62
62
|
* Create bind group layout for matmul operation
|
|
@@ -260,6 +260,23 @@ function buildF16CapabilityErrorDetail({
|
|
|
260
260
|
return parts.length > 0 ? ` (${parts.join(', ')})` : '';
|
|
261
261
|
}
|
|
262
262
|
|
|
263
|
+
function requireMatmulOutputDtype(dtype, opLabel) {
|
|
264
|
+
if (dtype === 'f16' || dtype === 'f32') {
|
|
265
|
+
return dtype;
|
|
266
|
+
}
|
|
267
|
+
throw new Error(`[${opLabel}] options.outputDtype is required and must be "f16" or "f32", got ${String(dtype)}.`);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function requireMatmulWeightDtype(dtype, opLabel, details = []) {
|
|
271
|
+
if (dtype != null && dtype !== '') {
|
|
272
|
+
return dtype;
|
|
273
|
+
}
|
|
274
|
+
const suffix = details.filter(Boolean).length > 0
|
|
275
|
+
? ` (${details.filter(Boolean).join(', ')})`
|
|
276
|
+
: '';
|
|
277
|
+
throw new Error(`[${opLabel}] B dtype is required for matmul dispatch${suffix}.`);
|
|
278
|
+
}
|
|
279
|
+
|
|
263
280
|
async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
264
281
|
const isRecord = Boolean(recorder);
|
|
265
282
|
const mode = isRecord ? 'record' : 'run';
|
|
@@ -303,8 +320,13 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
303
320
|
validateMatmulDimensions(opLabel, M, N, K);
|
|
304
321
|
|
|
305
322
|
const aDtype = toMatmulDtype(A.dtype);
|
|
306
|
-
const
|
|
307
|
-
|
|
323
|
+
const rawBDtype = requireMatmulWeightDtype(weightDtype ?? options.bDtype, opLabel, [
|
|
324
|
+
options.role ? `role=${options.role}` : null,
|
|
325
|
+
Number.isFinite(options.layerIdx) ? `layer=${options.layerIdx}` : null,
|
|
326
|
+
weightLabel ? `label=${weightLabel}` : null,
|
|
327
|
+
]);
|
|
328
|
+
const bDtype = toMatmulDtype(rawBDtype);
|
|
329
|
+
const requestedOutputDtype = requireMatmulOutputDtype(options.outputDtype, opLabel);
|
|
308
330
|
|
|
309
331
|
if (bDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
310
332
|
const detail = buildF16CapabilityErrorDetail({
|
|
@@ -324,10 +346,6 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
324
346
|
throw new Error(`[${opLabel}] f16 output requires shader-f16 support.`);
|
|
325
347
|
}
|
|
326
348
|
|
|
327
|
-
if (!isRecord && isTraceEnabled('kernels') && !weightDtype && !options.bDtype && M <= 2) {
|
|
328
|
-
log.warn('Matmul', `runMatmul: B buffer dtype unknown! size=${bBuffer.size}, M=${M}, N=${N}, K=${K}. Assuming f32.`);
|
|
329
|
-
}
|
|
330
|
-
|
|
331
349
|
validateMatmulOffsets(opLabel, aOffset, bOffset, cOffset);
|
|
332
350
|
|
|
333
351
|
const runtimeSession = getRuntimeConfig().inference?.session;
|
package/src/gpu/kernels/moe.d.ts
CHANGED
|
@@ -19,6 +19,10 @@ export interface MoEOptions extends OutputBufferOptions {
|
|
|
19
19
|
perExpertScale?: GPUBuffer | null;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
export interface MoEScatterOptions extends MoEOptions {
|
|
23
|
+
weightsDtype: 'f16' | 'f32';
|
|
24
|
+
}
|
|
25
|
+
|
|
22
26
|
/** MoE gather result */
|
|
23
27
|
export interface MoEGatherResult {
|
|
24
28
|
gathered: Tensor;
|
|
@@ -105,5 +109,5 @@ export declare function runScatterAddDynamic(
|
|
|
105
109
|
numTokens: number,
|
|
106
110
|
hiddenSize: number,
|
|
107
111
|
topK: number,
|
|
108
|
-
options
|
|
112
|
+
options: MoEScatterOptions
|
|
109
113
|
): Promise<Tensor>;
|
package/src/gpu/kernels/moe.js
CHANGED
|
@@ -470,10 +470,14 @@ export async function recordMoEBuildTokenOffsets(recorder, tokenCounts, tokenMap
|
|
|
470
470
|
|
|
471
471
|
async function executeScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
|
|
472
472
|
const execution = resolveExecution(recorder);
|
|
473
|
-
const { outputBuffer = null,
|
|
473
|
+
const { outputBuffer = null, perExpertScale = null } = options;
|
|
474
|
+
const weightsDtype = options.weightsDtype;
|
|
474
475
|
const ownsOutput = outputBuffer == null;
|
|
475
476
|
const hasExpertScale = perExpertScale != null;
|
|
476
477
|
|
|
478
|
+
if (weightsDtype !== 'f16' && weightsDtype !== 'f32') {
|
|
479
|
+
throw new Error(`ScatterAddDynamic requires options.weightsDtype to be "f16" or "f32", got ${String(weightsDtype)}.`);
|
|
480
|
+
}
|
|
477
481
|
if (weightsDtype === 'f16' && expertOutputs.dtype !== 'f16') {
|
|
478
482
|
throw new Error('ScatterAddDynamic f16 weights require f16 expert outputs');
|
|
479
483
|
}
|
|
@@ -552,4 +556,3 @@ async function executeScatterAddDynamic(recorder, expertOutputs, indices, weight
|
|
|
552
556
|
export async function runScatterAddDynamic(expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
|
|
553
557
|
return executeScatterAddDynamic(null, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options);
|
|
554
558
|
}
|
|
555
|
-
|
|
@@ -21,8 +21,8 @@ export interface SoftmaxOptions extends OutputBufferOptions {
|
|
|
21
21
|
|
|
22
22
|
export interface SoftmaxTopKOptions {
|
|
23
23
|
normalize?: boolean;
|
|
24
|
-
inputDtype
|
|
25
|
-
weightsDtype
|
|
24
|
+
inputDtype: 'f16' | 'f32';
|
|
25
|
+
weightsDtype: 'f16' | 'f32';
|
|
26
26
|
modelType?: string | null;
|
|
27
27
|
}
|
|
28
28
|
|
|
@@ -43,7 +43,7 @@ export declare function runSoftmaxTopK(
|
|
|
43
43
|
numTokens: number,
|
|
44
44
|
numExperts: number,
|
|
45
45
|
topK: number,
|
|
46
|
-
options
|
|
46
|
+
options: SoftmaxTopKOptions
|
|
47
47
|
): Promise<{ indices: GPUBuffer; weights: GPUBuffer }>;
|
|
48
48
|
|
|
49
49
|
/**
|
|
@@ -62,10 +62,17 @@ export async function runSoftmaxTopK(logits, numTokens, numExperts, topK, option
|
|
|
62
62
|
const device = getDevice();
|
|
63
63
|
const {
|
|
64
64
|
normalize = true,
|
|
65
|
-
inputDtype = 'f32',
|
|
66
|
-
weightsDtype = 'f32',
|
|
67
65
|
modelType = null,
|
|
68
66
|
} = options;
|
|
67
|
+
const inputDtype = options.inputDtype;
|
|
68
|
+
const weightsDtype = options.weightsDtype;
|
|
69
|
+
|
|
70
|
+
if (inputDtype !== 'f16' && inputDtype !== 'f32') {
|
|
71
|
+
throw new Error(`SoftmaxTopK requires options.inputDtype to be "f16" or "f32", got ${String(inputDtype)}.`);
|
|
72
|
+
}
|
|
73
|
+
if (weightsDtype !== 'f16' && weightsDtype !== 'f32') {
|
|
74
|
+
throw new Error(`SoftmaxTopK requires options.weightsDtype to be "f16" or "f32", got ${String(weightsDtype)}.`);
|
|
75
|
+
}
|
|
69
76
|
|
|
70
77
|
if (weightsDtype === 'f16' && inputDtype !== 'f16') {
|
|
71
78
|
throw new Error('SoftmaxTopK f16 weights require f16 logits');
|
|
@@ -435,7 +435,7 @@ function retainSharedBuffer(cache, key, factory) {
|
|
|
435
435
|
|
|
436
436
|
export function retainTurboQuantSharedBuffers(device, options = {}) {
|
|
437
437
|
const headDim = Number(options.headDim);
|
|
438
|
-
const bitWidth = Number(options.bitWidth
|
|
438
|
+
const bitWidth = Number(options.bitWidth);
|
|
439
439
|
const prodMode = options.prodMode === true;
|
|
440
440
|
if (!device) {
|
|
441
441
|
throw new Error('TurboQuant shared buffers require a GPU device.');
|
package/src/gpu/profiler.js
CHANGED
|
@@ -65,7 +65,10 @@ export class GPUProfiler {
|
|
|
65
65
|
this.#queryCapacity = runtimeProfiler.queryCapacity;
|
|
66
66
|
this.#maxSamples = runtimeProfiler.maxSamples;
|
|
67
67
|
this.#maxDurationMs = runtimeProfiler.maxDurationMs;
|
|
68
|
-
this.#maxHistoryLabels =
|
|
68
|
+
this.#maxHistoryLabels = requirePositiveInteger(
|
|
69
|
+
runtimeProfiler.maxHistoryLabels,
|
|
70
|
+
'runtime.shared.debug.profiler.maxHistoryLabels'
|
|
71
|
+
);
|
|
69
72
|
|
|
70
73
|
// Initialize query resources if timestamp queries available
|
|
71
74
|
if (this.#hasTimestampQuery && this.#device) {
|
|
@@ -368,5 +371,12 @@ export class GPUProfiler {
|
|
|
368
371
|
}
|
|
369
372
|
}
|
|
370
373
|
|
|
374
|
+
function requirePositiveInteger(value, label) {
|
|
375
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
376
|
+
throw new Error(`${label} must be a positive integer.`);
|
|
377
|
+
}
|
|
378
|
+
return value;
|
|
379
|
+
}
|
|
380
|
+
|
|
371
381
|
|
|
372
382
|
export default GPUProfiler;
|
|
@@ -99,7 +99,10 @@ export async function resolveKernelPathForModel(options = {}) {
|
|
|
99
99
|
|
|
100
100
|
if (!manifest) return null;
|
|
101
101
|
|
|
102
|
-
const modelConfig = parseModelConfigFromManifest(
|
|
102
|
+
const modelConfig = parseModelConfigFromManifest(
|
|
103
|
+
manifest,
|
|
104
|
+
runtimeConfig?.inference?.modelOverrides ?? null
|
|
105
|
+
);
|
|
103
106
|
const kernelPathState = resolveKernelPathState({
|
|
104
107
|
manifest,
|
|
105
108
|
runtimeConfig,
|