doppler-gpu 0.4.6 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/README.md +43 -31
  2. package/assets/doppler-runtime-map.svg +1 -3
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/config/conversion/README.md +4 -0
  11. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  12. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  13. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  14. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  19. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  21. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  25. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  26. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  29. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  30. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  31. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  32. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  33. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  34. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  35. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  36. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  37. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  39. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  40. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  41. package/src/config/execution-contract-check.js +14 -10
  42. package/src/config/merge-contract-check.js +1 -0
  43. package/src/config/merge.js +30 -0
  44. package/src/config/param-validator.d.ts +5 -0
  45. package/src/config/param-validator.js +42 -8
  46. package/src/config/required-inference-fields-contract-check.js +14 -0
  47. package/src/config/schema/debug.schema.d.ts +2 -0
  48. package/src/config/schema/debug.schema.js +1 -0
  49. package/src/config/schema/doppler.schema.d.ts +5 -1
  50. package/src/config/schema/doppler.schema.js +82 -292
  51. package/src/config/schema/inference.schema.d.ts +17 -3
  52. package/src/config/schema/kvcache.schema.d.ts +19 -3
  53. package/src/config/schema/kvcache.schema.js +2 -1
  54. package/src/config/schema/manifest.schema.d.ts +38 -1
  55. package/src/config/schema/manifest.schema.js +18 -0
  56. package/src/config/schema/policy-schema-registry.json +5 -0
  57. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  58. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  59. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  60. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  61. package/src/config/transforms/execution-graph-transforms.js +8 -0
  62. package/src/converter/conversion-plan.js +2 -0
  63. package/src/converter/parsers/transformer.js +27 -1
  64. package/src/converter/quantization-info.js +16 -4
  65. package/src/converter/rope-config.js +55 -3
  66. package/src/experimental/training/distillation/student-fixture.js +1 -1
  67. package/src/experimental/training/lora-pipeline.js +1 -1
  68. package/src/experimental/training/lora.js +2 -2
  69. package/src/experimental/training/suite.js +1 -1
  70. package/src/gpu/kernels/README.md +8 -1
  71. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  72. package/src/gpu/kernels/dequant.d.ts +6 -6
  73. package/src/gpu/kernels/dequant.js +17 -6
  74. package/src/gpu/kernels/linear-attention-core.js +1 -6
  75. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  76. package/src/gpu/kernels/matmul-selection.js +4 -1
  77. package/src/gpu/kernels/matmul.d.ts +1 -1
  78. package/src/gpu/kernels/matmul.js +24 -6
  79. package/src/gpu/kernels/moe.d.ts +5 -1
  80. package/src/gpu/kernels/moe.js +5 -2
  81. package/src/gpu/kernels/softmax.d.ts +3 -3
  82. package/src/gpu/kernels/softmax.js +9 -2
  83. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  84. package/src/gpu/kernels/types.d.ts +1 -1
  85. package/src/gpu/profiler.js +11 -1
  86. package/src/inference/browser-harness-model-helpers.js +4 -1
  87. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  88. package/src/inference/browser-harness-text-helpers.js +357 -116
  89. package/src/inference/browser-harness.d.ts +3 -3
  90. package/src/inference/browser-harness.js +266 -11
  91. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  92. package/src/inference/kv-cache/quantized.js +16 -2
  93. package/src/inference/kv-cache/tiered.js +50 -15
  94. package/src/inference/moe-router.js +38 -7
  95. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  96. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  97. package/src/inference/pipelines/text/config.d.ts +5 -1
  98. package/src/inference/pipelines/text/config.js +135 -112
  99. package/src/inference/pipelines/text/embed.d.ts +1 -0
  100. package/src/inference/pipelines/text/embed.js +39 -12
  101. package/src/inference/pipelines/text/execution-plan.js +8 -2
  102. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  103. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  104. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  105. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  106. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  107. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  108. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  109. package/src/inference/pipelines/text/generator-steps.js +3 -0
  110. package/src/inference/pipelines/text/generator.js +6 -1
  111. package/src/inference/pipelines/text/init-rope.js +54 -6
  112. package/src/inference/pipelines/text/init.js +63 -15
  113. package/src/inference/pipelines/text/layer.js +32 -5
  114. package/src/inference/pipelines/text/linear-attention.js +12 -8
  115. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  116. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  117. package/src/inference/pipelines/text/logits/index.js +44 -87
  118. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  119. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  120. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  121. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  122. package/src/inference/pipelines/text/logits/utils.js +11 -0
  123. package/src/inference/pipelines/text/lora-apply.js +6 -4
  124. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  125. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  126. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  127. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  128. package/src/inference/pipelines/text/ops.d.ts +3 -0
  129. package/src/inference/pipelines/text/ops.js +26 -1
  130. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  131. package/src/inference/pipelines/text/weights.js +7 -2
  132. package/src/inference/pipelines/text.js +24 -4
  133. package/src/inference/tokenizers/behavior-flags.js +23 -2
  134. package/src/loader/final-weights-loader.js +12 -1
  135. package/src/rules/execution-rules-contract-check.js +12 -3
  136. package/src/rules/inference/capability-transforms.rules.json +14 -2
  137. package/src/rules/inference/config.rules.json +26 -2
  138. package/src/rules/inference/dtype.rules.json +12 -5
  139. package/src/rules/inference/execution.rules.json +5 -19
  140. package/src/rules/inference/moe.rules.json +97 -0
  141. package/src/tooling/command-api-constants.js +2 -2
  142. package/src/tooling/command-api-family-normalizers.js +2 -1
  143. package/src/tooling/command-api.d.ts +1 -1
  144. package/src/tooling/command-runner-shared.d.ts +1 -1
  145. package/src/tooling/command-runner-shared.js +3 -1
  146. package/src/tooling/node-converter.js +20 -0
  147. package/src/tooling/node-source-runtime.js +5 -1
  148. package/src/version.js +1 -1
  149. package/assets/doppler-readme-hero.svg +0 -84
@@ -28,6 +28,15 @@ function asNumberArray(value) {
28
28
  return normalized.map((entry) => Math.trunc(entry));
29
29
  }
30
30
 
31
+ function asPositiveNumberArray(value) {
32
+ if (!Array.isArray(value)) return null;
33
+ const normalized = value.map((entry) => asFiniteNumber(entry));
34
+ if (normalized.some((entry) => entry == null || entry <= 0)) {
35
+ return null;
36
+ }
37
+ return normalized.map((entry) => Number(entry));
38
+ }
39
+
31
40
  function normalizeRoPEType(value) {
32
41
  if (typeof value !== 'string') return null;
33
42
  const normalized = value.trim().toLowerCase();
@@ -90,6 +99,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
90
99
  yarnBetaFast: null,
91
100
  yarnBetaSlow: null,
92
101
  yarnOriginalMaxPos: null,
102
+ longropeShortFactor: null,
103
+ longropeLongFactor: null,
104
+ longropeOriginalMaxPos: null,
93
105
  };
94
106
  }
95
107
 
@@ -98,6 +110,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
98
110
  let yarnBetaFast = null;
99
111
  let yarnBetaSlow = null;
100
112
  let yarnOriginalMaxPos = null;
113
+ let longropeShortFactor = null;
114
+ let longropeLongFactor = null;
115
+ let longropeOriginalMaxPos = null;
101
116
 
102
117
  if (ropeScalingType == null) {
103
118
  if (factor != null && factor > 0 && factor !== 1.0) {
@@ -123,6 +138,23 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
123
138
  yarnBetaSlow = betaSlow;
124
139
  yarnOriginalMaxPos = origMaxPos;
125
140
  }
141
+ if (ropeScalingType === 'longrope') {
142
+ const shortFactor = asPositiveNumberArray(ropeScalingConfig.short_factor);
143
+ const longFactor = asPositiveNumberArray(ropeScalingConfig.long_factor);
144
+ const origMaxPos = asFiniteNumber(ropeScalingConfig.original_max_position_embeddings);
145
+ if (!shortFactor || !longFactor || origMaxPos == null) {
146
+ throw new Error(
147
+ 'LongRoPE scaling detected but required params missing in HF config. ' +
148
+ 'LongRoPE requires short_factor, long_factor, and original_max_position_embeddings. ' +
149
+ `Got: short_factor=${Array.isArray(ropeScalingConfig.short_factor) ? ropeScalingConfig.short_factor.length : ropeScalingConfig.short_factor}, ` +
150
+ `long_factor=${Array.isArray(ropeScalingConfig.long_factor) ? ropeScalingConfig.long_factor.length : ropeScalingConfig.long_factor}, ` +
151
+ `original_max_position_embeddings=${origMaxPos}`
152
+ );
153
+ }
154
+ longropeShortFactor = shortFactor;
155
+ longropeLongFactor = longFactor;
156
+ longropeOriginalMaxPos = origMaxPos;
157
+ }
126
158
 
127
159
  return {
128
160
  ropeScalingType,
@@ -130,6 +162,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
130
162
  yarnBetaFast,
131
163
  yarnBetaSlow,
132
164
  yarnOriginalMaxPos,
165
+ longropeShortFactor,
166
+ longropeLongFactor,
167
+ longropeOriginalMaxPos,
133
168
  };
134
169
  }
135
170
 
@@ -140,7 +175,9 @@ function hasScalingDirective(ropeScalingConfig) {
140
175
  || ropeScalingConfig.factor != null
141
176
  || ropeScalingConfig.beta_fast != null
142
177
  || ropeScalingConfig.beta_slow != null
143
- || ropeScalingConfig.original_max_position_embeddings != null;
178
+ || ropeScalingConfig.original_max_position_embeddings != null
179
+ || ropeScalingConfig.short_factor != null
180
+ || ropeScalingConfig.long_factor != null;
144
181
  }
145
182
 
146
183
  function hasMeaningfulScalingConfig(resolvedScaling) {
@@ -149,7 +186,10 @@ function hasMeaningfulScalingConfig(resolvedScaling) {
149
186
  || resolvedScaling.ropeScalingFactor !== DEFAULT_MANIFEST_INFERENCE.rope.ropeScalingFactor
150
187
  || resolvedScaling.yarnBetaFast != null
151
188
  || resolvedScaling.yarnBetaSlow != null
152
- || resolvedScaling.yarnOriginalMaxPos != null;
189
+ || resolvedScaling.yarnOriginalMaxPos != null
190
+ || resolvedScaling.longropeShortFactor != null
191
+ || resolvedScaling.longropeLongFactor != null
192
+ || resolvedScaling.longropeOriginalMaxPos != null;
153
193
  }
154
194
 
155
195
  function isSameScalingConfig(left, right) {
@@ -157,7 +197,10 @@ function isSameScalingConfig(left, right) {
157
197
  && left.ropeScalingFactor === right.ropeScalingFactor
158
198
  && left.yarnBetaFast === right.yarnBetaFast
159
199
  && left.yarnBetaSlow === right.yarnBetaSlow
160
- && left.yarnOriginalMaxPos === right.yarnOriginalMaxPos;
200
+ && left.yarnOriginalMaxPos === right.yarnOriginalMaxPos
201
+ && JSON.stringify(left.longropeShortFactor ?? null) === JSON.stringify(right.longropeShortFactor ?? null)
202
+ && JSON.stringify(left.longropeLongFactor ?? null) === JSON.stringify(right.longropeLongFactor ?? null)
203
+ && left.longropeOriginalMaxPos === right.longropeOriginalMaxPos;
161
204
  }
162
205
 
163
206
  function failOnConflictingScaling(sourceLabel, canonicalScaling, candidateScaling) {
@@ -210,6 +253,9 @@ export function buildRoPEConfig(converterInference, config) {
210
253
  yarnBetaFast: configuredRoPE.yarnBetaFast ?? null,
211
254
  yarnBetaSlow: configuredRoPE.yarnBetaSlow ?? null,
212
255
  yarnOriginalMaxPos: configuredRoPE.yarnOriginalMaxPos ?? null,
256
+ longropeShortFactor: configuredRoPE.longropeShortFactor ?? null,
257
+ longropeLongFactor: configuredRoPE.longropeLongFactor ?? null,
258
+ longropeOriginalMaxPos: configuredRoPE.longropeOriginalMaxPos ?? null,
213
259
  };
214
260
 
215
261
  if (ropeScaling) {
@@ -253,6 +299,9 @@ export function buildRoPEConfig(converterInference, config) {
253
299
  yarnBetaFast: configuredRoPE.ropeLocalYarnBetaFast ?? globalScaling.yarnBetaFast,
254
300
  yarnBetaSlow: configuredRoPE.ropeLocalYarnBetaSlow ?? globalScaling.yarnBetaSlow,
255
301
  yarnOriginalMaxPos: configuredRoPE.ropeLocalYarnOriginalMaxPos ?? globalScaling.yarnOriginalMaxPos,
302
+ longropeShortFactor: globalScaling.longropeShortFactor,
303
+ longropeLongFactor: globalScaling.longropeLongFactor,
304
+ longropeOriginalMaxPos: globalScaling.longropeOriginalMaxPos,
256
305
  }
257
306
  : { ...globalScaling };
258
307
  if (ropeScaling) {
@@ -323,6 +372,9 @@ export function buildRoPEConfig(converterInference, config) {
323
372
  yarnBetaFast: globalScaling.yarnBetaFast,
324
373
  yarnBetaSlow: globalScaling.yarnBetaSlow,
325
374
  yarnOriginalMaxPos: globalScaling.yarnOriginalMaxPos,
375
+ longropeShortFactor: globalScaling.longropeShortFactor,
376
+ longropeLongFactor: globalScaling.longropeLongFactor,
377
+ longropeOriginalMaxPos: globalScaling.longropeOriginalMaxPos,
326
378
  ropeLocalScalingType: localScaling.ropeScalingType,
327
379
  ropeLocalScalingFactor: localScaling.ropeScalingFactor,
328
380
  ropeLocalYarnBetaFast: localScaling.yarnBetaFast,
@@ -421,7 +421,7 @@ function createDistillStudentProjectionModelFixture(overrides = {}, options = {}
421
421
  const rows = Number.isFinite(inputTensor?.shape?.[0]) ? inputTensor.shape[0] : 1;
422
422
  return tape.record(
423
423
  OpType.MATMUL,
424
- (a, b) => runMatmul(a, b, rows, outputDim, embeddingDim, { transposeB: false }),
424
+ (a, b) => runMatmul(a, b, rows, outputDim, embeddingDim, { transposeB: false, outputDtype: 'f32' }),
425
425
  [inputTensor, projectionWeight],
426
426
  { M: rows, N: outputDim, K: embeddingDim, transposeB: false }
427
427
  );
@@ -400,7 +400,7 @@ function createToyLoraModel(workload) {
400
400
  const batchSize = Number.isInteger(inputTensor?.shape?.[0]) ? inputTensor.shape[0] : 1;
401
401
  const baseLogits = await tape.record(
402
402
  OpType.MATMUL,
403
- (a, b) => runMatmul(a, b, batchSize, 2, 3, { transposeB: false }),
403
+ (a, b) => runMatmul(a, b, batchSize, 2, 3, { transposeB: false, outputDtype: 'f32' }),
404
404
  [inputTensor, baseWeight],
405
405
  { M: batchSize, N: 2, K: 3, transposeB: false }
406
406
  );
@@ -68,13 +68,13 @@ export class LoraAdapter {
68
68
  const [tokens] = input.shape;
69
69
  const down = await tape.record(
70
70
  OpType.MATMUL,
71
- (a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0]),
71
+ (a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0], { outputDtype: input.dtype }),
72
72
  [input, this.A],
73
73
  { M: tokens, N: this.rank, K: this.A.shape[0] }
74
74
  );
75
75
  const up = await tape.record(
76
76
  OpType.MATMUL,
77
- (a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank),
77
+ (a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank, { outputDtype: down.dtype }),
78
78
  [down, this.B],
79
79
  { M: tokens, N: this.B.shape[1], K: this.rank }
80
80
  );
@@ -821,7 +821,7 @@ export function createToyModelFixture(overrides = {}) {
821
821
  async forward(inputTensor, tape) {
822
822
  return tape.record(
823
823
  OpType.MATMUL,
824
- (a, b) => runMatmul(a, b, 2, 2, 3, { transposeB: false }),
824
+ (a, b) => runMatmul(a, b, 2, 2, 3, { transposeB: false, outputDtype: 'f32' }),
825
825
  [inputTensor, baseWeight],
826
826
  { M: 2, N: 2, K: 3, transposeB: false }
827
827
  );
@@ -50,7 +50,10 @@ Three ways to make kernels flexible:
50
50
  | Change per dispatch | select different | recompile | yes |
51
51
  | Code duplication | high | minimal | none |
52
52
 
53
- **DOPPLER uses entry points** over override constants - more code duplication but simpler pipeline management.
53
+ Use the topology test from the WGSL style guide. Entry points are for algorithm
54
+ or synchronization changes. Override constants are for model/load-time
55
+ parameters. Uniforms are for per-dispatch values. The registry records which
56
+ choice a variant made; WGSL files must not become a second policy layer.
54
57
 
55
58
  ## Entry Points
56
59
 
@@ -123,6 +126,10 @@ Kernel operation IDs, variant IDs, WGSL filenames, entry points, feature
123
126
  requirements, bindings, uniforms, and metadata are defined in
124
127
  `src/config/kernels/registry.json`.
125
128
 
129
+ The management front door is
130
+ [`docs/developer-guides/config-source-of-truth.md`](../../../docs/developer-guides/config-source-of-truth.md#wgsl-kernel-management).
131
+ Do not add separate kernel inventories in docs, scripts, or runtime helpers.
132
+
126
133
  Use `npm run kernels:registry:check` to audit the on-disk WGSL inventory
127
134
  against the registry. This README must not define separate filename suffix
128
135
  rules.
@@ -141,12 +141,14 @@ async function runAttentionBackwardCore(
141
141
  dV = await runMatmul(sTransposed, dHead.buffer, seqLen, headDim, seqLen, {
142
142
  transposeB: false,
143
143
  bDtype: 'f32',
144
+ outputDtype: 'f32',
144
145
  });
145
146
 
146
147
  vTransposed = await runTranspose(vHead, seqLen, headDim);
147
148
  dS = await runMatmul(dHead, vTransposed.buffer, seqLen, seqLen, headDim, {
148
149
  transposeB: false,
149
150
  bDtype: 'f32',
151
+ outputDtype: 'f32',
150
152
  });
151
153
  dQK = causal
152
154
  ? await runBackwardKernel(
@@ -166,12 +168,14 @@ async function runAttentionBackwardCore(
166
168
  transposeB: false,
167
169
  alpha: scale,
168
170
  bDtype: 'f32',
171
+ outputDtype: 'f32',
169
172
  });
170
173
  dQKTransposed = await runTranspose(dQK, seqLen, seqLen);
171
174
  dK = await runMatmul(dQKTransposed, qHead.buffer, seqLen, headDim, seqLen, {
172
175
  transposeB: false,
173
176
  alpha: scale,
174
177
  bDtype: 'f32',
178
+ outputDtype: 'f32',
175
179
  });
176
180
 
177
181
  const copyEncoder = getDevice().createCommandEncoder();
@@ -240,12 +244,14 @@ async function runAttentionBackwardCore(
240
244
  dV = await recordMatmul(recorder, sTransposed, dHead.buffer, seqLen, headDim, seqLen, {
241
245
  transposeB: false,
242
246
  bDtype: 'f32',
247
+ outputDtype: 'f32',
243
248
  });
244
249
 
245
250
  vTransposed = await recordTranspose(recorder, vHead, seqLen, headDim);
246
251
  dS = await recordMatmul(recorder, dHead, vTransposed.buffer, seqLen, seqLen, headDim, {
247
252
  transposeB: false,
248
253
  bDtype: 'f32',
254
+ outputDtype: 'f32',
249
255
  });
250
256
  dQK = causal
251
257
  ? await recordBackwardKernel(
@@ -266,12 +272,14 @@ async function runAttentionBackwardCore(
266
272
  transposeB: false,
267
273
  alpha: scale,
268
274
  bDtype: 'f32',
275
+ outputDtype: 'f32',
269
276
  });
270
277
  dQKTransposed = await recordTranspose(recorder, dQK, seqLen, seqLen);
271
278
  dK = await recordMatmul(recorder, dQKTransposed, qHead.buffer, seqLen, headDim, seqLen, {
272
279
  transposeB: false,
273
280
  alpha: scale,
274
281
  bDtype: 'f32',
282
+ outputDtype: 'f32',
275
283
  });
276
284
 
277
285
  encoder.copyBufferToBuffer(dQ.buffer, 0, gradQBuf, qOffset, headBytes);
@@ -22,7 +22,7 @@ export interface DequantOptions extends OutputBufferOptions, OutputOffsetOptions
22
22
  /**
23
23
  * Select the best dequantization kernel variant
24
24
  */
25
- export declare function selectDequantKernel(options?: DequantOptions): string;
25
+ export declare function selectDequantKernel(options: DequantOptions): string;
26
26
 
27
27
  /**
28
28
  * Create bind group layout for dequant operation
@@ -35,7 +35,7 @@ export declare function createDequantBindGroupLayout(): GPUBindGroupLayout;
35
35
  export declare function dequantize(
36
36
  quantized: GPUBuffer,
37
37
  numBlocks: number,
38
- options?: DequantOptions
38
+ options: DequantOptions
39
39
  ): Promise<Tensor>;
40
40
 
41
41
  /**
@@ -45,7 +45,7 @@ export declare function dequantizeRowwise(
45
45
  quantized: GPUBuffer,
46
46
  rows: number,
47
47
  K: number,
48
- options?: OutputBufferOptions & OutputDtypeOptions
48
+ options: OutputBufferOptions & OutputDtypeOptions
49
49
  ): Promise<Tensor>;
50
50
 
51
51
  /**
@@ -69,7 +69,7 @@ export declare function dequantizeMXFP4Expert(
69
69
  numExperts: number,
70
70
  outDim: number,
71
71
  numGroups: number,
72
- options?: DequantOptions
72
+ options: DequantOptions
73
73
  ): Promise<Tensor>;
74
74
 
75
75
  /**
@@ -78,7 +78,7 @@ export declare function dequantizeMXFP4Expert(
78
78
  export declare function dequantizeQ6K(
79
79
  quantized: GPUBuffer,
80
80
  numBlocks: number,
81
- options?: DequantOptions
81
+ options: DequantOptions
82
82
  ): Promise<Tensor>;
83
83
 
84
84
  /**
@@ -88,5 +88,5 @@ export declare function recordDequantize(
88
88
  recorder: CommandRecorder,
89
89
  quantized: GPUBuffer,
90
90
  numBlocks: number,
91
- options?: DequantOptions
91
+ options: DequantOptions
92
92
  ): Promise<Tensor>;
@@ -11,10 +11,17 @@ import { releaseUniformBuffer } from '../uniform-cache.js';
11
11
  import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
12
12
  import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
13
13
 
14
+ function requireOutputDtype(value, label) {
15
+ if (value !== 'f16' && value !== 'f32') {
16
+ throw new Error(`[${label}] outputDtype is required and must be "f16" or "f32", got ${String(value)}.`);
17
+ }
18
+ return value;
19
+ }
14
20
 
15
21
  export function selectDequantKernel(options = {}) {
16
22
  const capabilities = getKernelCapabilities();
17
- const { useVec4 = true, outputDtype = 'f32' } = options;
23
+ const { useVec4 = true } = options;
24
+ const outputDtype = requireOutputDtype(options.outputDtype, 'selectDequantKernel');
18
25
 
19
26
  const wantsF16Out = outputDtype === 'f16' && capabilities.hasF16;
20
27
  return selectKernelRuleValue(
@@ -90,8 +97,8 @@ export async function dequantize(
90
97
  const {
91
98
  outputOffset = 0,
92
99
  outputBuffer = null,
93
- outputDtype = 'f32',
94
100
  } = options;
101
+ const outputDtype = requireOutputDtype(options.outputDtype, 'dequantize');
95
102
 
96
103
  if (outputDtype === 'f16' && capabilities?.hasF16 !== true) {
97
104
  throw new Error('[dequantize] f16 output requires shader-f16 support.');
@@ -160,7 +167,8 @@ export async function dequantizeRowwise(
160
167
  ) {
161
168
  const device = getDevice();
162
169
  const capabilities = getKernelCapabilities();
163
- const { outputBuffer = null, outputDtype = 'f16' } = options;
170
+ const { outputBuffer = null } = options;
171
+ const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeRowwise');
164
172
  if (outputDtype === 'f16' && capabilities?.hasF16 !== true) {
165
173
  throw new Error('[dequantizeRowwise] f16 output requires shader-f16 support.');
166
174
  }
@@ -296,11 +304,11 @@ export async function dequantizeMXFP4Expert(
296
304
  const device = getDevice();
297
305
  const {
298
306
  outputBuffer = null,
299
- outputDtype = 'f32',
300
307
  modelType = null,
301
308
  groupSize = 32,
302
309
  dequantTileShape = 'scalar',
303
310
  } = options;
311
+ const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeMXFP4Expert');
304
312
  const caps = getKernelCapabilities();
305
313
 
306
314
  const variant = selectKernelRuleValue('dequant', 'mxfp4ExpertVariant', {
@@ -375,8 +383,11 @@ export async function dequantizeQ6K(
375
383
  const {
376
384
  outputOffset = 0,
377
385
  outputBuffer = null,
378
- outputDtype = 'f16', // Q6_K always outputs f16 for now
379
386
  } = options;
387
+ const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeQ6K');
388
+ if (outputDtype !== 'f16') {
389
+ throw new Error(`[dequantizeQ6K] outputDtype must be "f16", got ${outputDtype}.`);
390
+ }
380
391
 
381
392
  // Q6_K only has f16 output kernel currently
382
393
  const pipeline = await getPipelineFast('dequant', 'q6k_f16out');
@@ -451,8 +462,8 @@ export async function recordDequantize(
451
462
  const {
452
463
  outputOffset = 0,
453
464
  outputBuffer = null,
454
- outputDtype = 'f32',
455
465
  } = options;
466
+ const outputDtype = requireOutputDtype(options.outputDtype, 'recordDequantize');
456
467
 
457
468
  // Select kernel
458
469
  const variant = selectDequantKernel({ ...options, outputDtype });
@@ -374,12 +374,7 @@ function requireGpuBuffer(buffer, label) {
374
374
  }
375
375
 
376
376
  function resolveOutputDtype(outputDtype) {
377
- const normalized = selectRuleValue(
378
- 'shared',
379
- 'dtype',
380
- 'f16OrF32FromDtype',
381
- { dtype: outputDtype === undefined ? 'f32' : outputDtype }
382
- );
377
+ const normalized = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: outputDtype });
383
378
  if (normalized === 'f16' || normalized === 'f32') {
384
379
  return normalized;
385
380
  }
@@ -26,10 +26,10 @@ export declare function isFusedQ4KDisabled(options?: {
26
26
 
27
27
  export declare function toMatmulDtype(dtype: string | null | undefined): 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16';
28
28
 
29
- export declare function selectMatmulKernel(options?: {
29
+ export declare function selectMatmulKernel(options: {
30
30
  preferF16?: boolean;
31
31
  useVec4?: boolean;
32
- outputDtype?: TensorDtype | 'f16' | 'f32';
32
+ outputDtype: TensorDtype | 'f16' | 'f32';
33
33
  aDtype?: 'f16' | 'f32' | null;
34
34
  bDtype?: 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16' | null;
35
35
  isPrefill?: boolean;
@@ -129,13 +129,16 @@ export function selectMatmulKernel(options = {}) {
129
129
  const {
130
130
  preferF16 = true,
131
131
  useVec4 = false,
132
- outputDtype = 'f32',
133
132
  aDtype = null,
134
133
  bDtype = null,
135
134
  isPrefill = false,
136
135
  prefillRows = 0,
137
136
  transposeB = true,
138
137
  } = options;
138
+ const outputDtype = options.outputDtype;
139
+ if (outputDtype !== 'f16' && outputDtype !== 'f32') {
140
+ throw new Error(`[Matmul] selectMatmulKernel requires outputDtype "f16" or "f32", got ${String(outputDtype)}.`);
141
+ }
139
142
  const { tiledPrefillMinRows } = getKernelThresholds().matmul;
140
143
 
141
144
  const inputsAreF16 = aDtype === 'f16' && bDtype === 'f16';
@@ -56,7 +56,7 @@ export declare function isFusedQ4KDisabled(): boolean;
56
56
  /**
57
57
  * Select the best matmul kernel variant
58
58
  */
59
- export declare function selectMatmulKernel(options?: MatmulOptions): string;
59
+ export declare function selectMatmulKernel(options: MatmulOptions): string;
60
60
 
61
61
  /**
62
62
  * Create bind group layout for matmul operation
@@ -260,6 +260,23 @@ function buildF16CapabilityErrorDetail({
260
260
  return parts.length > 0 ? ` (${parts.join(', ')})` : '';
261
261
  }
262
262
 
263
+ function requireMatmulOutputDtype(dtype, opLabel) {
264
+ if (dtype === 'f16' || dtype === 'f32') {
265
+ return dtype;
266
+ }
267
+ throw new Error(`[${opLabel}] options.outputDtype is required and must be "f16" or "f32", got ${String(dtype)}.`);
268
+ }
269
+
270
+ function requireMatmulWeightDtype(dtype, opLabel, details = []) {
271
+ if (dtype != null && dtype !== '') {
272
+ return dtype;
273
+ }
274
+ const suffix = details.filter(Boolean).length > 0
275
+ ? ` (${details.filter(Boolean).join(', ')})`
276
+ : '';
277
+ throw new Error(`[${opLabel}] B dtype is required for matmul dispatch${suffix}.`);
278
+ }
279
+
263
280
  async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
264
281
  const isRecord = Boolean(recorder);
265
282
  const mode = isRecord ? 'record' : 'run';
@@ -303,8 +320,13 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
303
320
  validateMatmulDimensions(opLabel, M, N, K);
304
321
 
305
322
  const aDtype = toMatmulDtype(A.dtype);
306
- const bDtype = toMatmulDtype(weightDtype ?? options.bDtype);
307
- const requestedOutputDtype = options.outputDtype || A.dtype;
323
+ const rawBDtype = requireMatmulWeightDtype(weightDtype ?? options.bDtype, opLabel, [
324
+ options.role ? `role=${options.role}` : null,
325
+ Number.isFinite(options.layerIdx) ? `layer=${options.layerIdx}` : null,
326
+ weightLabel ? `label=${weightLabel}` : null,
327
+ ]);
328
+ const bDtype = toMatmulDtype(rawBDtype);
329
+ const requestedOutputDtype = requireMatmulOutputDtype(options.outputDtype, opLabel);
308
330
 
309
331
  if (bDtype === 'f16' && capabilities?.hasF16 !== true) {
310
332
  const detail = buildF16CapabilityErrorDetail({
@@ -324,10 +346,6 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
324
346
  throw new Error(`[${opLabel}] f16 output requires shader-f16 support.`);
325
347
  }
326
348
 
327
- if (!isRecord && isTraceEnabled('kernels') && !weightDtype && !options.bDtype && M <= 2) {
328
- log.warn('Matmul', `runMatmul: B buffer dtype unknown! size=${bBuffer.size}, M=${M}, N=${N}, K=${K}. Assuming f32.`);
329
- }
330
-
331
349
  validateMatmulOffsets(opLabel, aOffset, bOffset, cOffset);
332
350
 
333
351
  const runtimeSession = getRuntimeConfig().inference?.session;
@@ -19,6 +19,10 @@ export interface MoEOptions extends OutputBufferOptions {
19
19
  perExpertScale?: GPUBuffer | null;
20
20
  }
21
21
 
22
+ export interface MoEScatterOptions extends MoEOptions {
23
+ weightsDtype: 'f16' | 'f32';
24
+ }
25
+
22
26
  /** MoE gather result */
23
27
  export interface MoEGatherResult {
24
28
  gathered: Tensor;
@@ -105,5 +109,5 @@ export declare function runScatterAddDynamic(
105
109
  numTokens: number,
106
110
  hiddenSize: number,
107
111
  topK: number,
108
- options?: MoEOptions
112
+ options: MoEScatterOptions
109
113
  ): Promise<Tensor>;
@@ -470,10 +470,14 @@ export async function recordMoEBuildTokenOffsets(recorder, tokenCounts, tokenMap
470
470
 
471
471
  async function executeScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
472
472
  const execution = resolveExecution(recorder);
473
- const { outputBuffer = null, weightsDtype = 'f32', perExpertScale = null } = options;
473
+ const { outputBuffer = null, perExpertScale = null } = options;
474
+ const weightsDtype = options.weightsDtype;
474
475
  const ownsOutput = outputBuffer == null;
475
476
  const hasExpertScale = perExpertScale != null;
476
477
 
478
+ if (weightsDtype !== 'f16' && weightsDtype !== 'f32') {
479
+ throw new Error(`ScatterAddDynamic requires options.weightsDtype to be "f16" or "f32", got ${String(weightsDtype)}.`);
480
+ }
477
481
  if (weightsDtype === 'f16' && expertOutputs.dtype !== 'f16') {
478
482
  throw new Error('ScatterAddDynamic f16 weights require f16 expert outputs');
479
483
  }
@@ -552,4 +556,3 @@ async function executeScatterAddDynamic(recorder, expertOutputs, indices, weight
552
556
  export async function runScatterAddDynamic(expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
553
557
  return executeScatterAddDynamic(null, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options);
554
558
  }
555
-
@@ -21,8 +21,8 @@ export interface SoftmaxOptions extends OutputBufferOptions {
21
21
 
22
22
  export interface SoftmaxTopKOptions {
23
23
  normalize?: boolean;
24
- inputDtype?: 'f16' | 'f32';
25
- weightsDtype?: 'f16' | 'f32';
24
+ inputDtype: 'f16' | 'f32';
25
+ weightsDtype: 'f16' | 'f32';
26
26
  modelType?: string | null;
27
27
  }
28
28
 
@@ -43,7 +43,7 @@ export declare function runSoftmaxTopK(
43
43
  numTokens: number,
44
44
  numExperts: number,
45
45
  topK: number,
46
- options?: SoftmaxTopKOptions
46
+ options: SoftmaxTopKOptions
47
47
  ): Promise<{ indices: GPUBuffer; weights: GPUBuffer }>;
48
48
 
49
49
  /**
@@ -62,10 +62,17 @@ export async function runSoftmaxTopK(logits, numTokens, numExperts, topK, option
62
62
  const device = getDevice();
63
63
  const {
64
64
  normalize = true,
65
- inputDtype = 'f32',
66
- weightsDtype = 'f32',
67
65
  modelType = null,
68
66
  } = options;
67
+ const inputDtype = options.inputDtype;
68
+ const weightsDtype = options.weightsDtype;
69
+
70
+ if (inputDtype !== 'f16' && inputDtype !== 'f32') {
71
+ throw new Error(`SoftmaxTopK requires options.inputDtype to be "f16" or "f32", got ${String(inputDtype)}.`);
72
+ }
73
+ if (weightsDtype !== 'f16' && weightsDtype !== 'f32') {
74
+ throw new Error(`SoftmaxTopK requires options.weightsDtype to be "f16" or "f32", got ${String(weightsDtype)}.`);
75
+ }
69
76
 
70
77
  if (weightsDtype === 'f16' && inputDtype !== 'f16') {
71
78
  throw new Error('SoftmaxTopK f16 weights require f16 logits');
@@ -435,7 +435,7 @@ function retainSharedBuffer(cache, key, factory) {
435
435
 
436
436
  export function retainTurboQuantSharedBuffers(device, options = {}) {
437
437
  const headDim = Number(options.headDim);
438
- const bitWidth = Number(options.bitWidth ?? 4);
438
+ const bitWidth = Number(options.bitWidth);
439
439
  const prodMode = options.prodMode === true;
440
440
  if (!device) {
441
441
  throw new Error('TurboQuant shared buffers require a GPU device.');
@@ -13,7 +13,7 @@ export interface OutputOffsetOptions {
13
13
  }
14
14
 
15
15
  export interface OutputDtypeOptions {
16
- outputDtype?: 'f16' | 'f32';
16
+ outputDtype: 'f16' | 'f32';
17
17
  }
18
18
 
19
19
  export interface Vec4Options {
@@ -65,7 +65,10 @@ export class GPUProfiler {
65
65
  this.#queryCapacity = runtimeProfiler.queryCapacity;
66
66
  this.#maxSamples = runtimeProfiler.maxSamples;
67
67
  this.#maxDurationMs = runtimeProfiler.maxDurationMs;
68
- this.#maxHistoryLabels = runtimeProfiler.maxHistoryLabels ?? 1024;
68
+ this.#maxHistoryLabels = requirePositiveInteger(
69
+ runtimeProfiler.maxHistoryLabels,
70
+ 'runtime.shared.debug.profiler.maxHistoryLabels'
71
+ );
69
72
 
70
73
  // Initialize query resources if timestamp queries available
71
74
  if (this.#hasTimestampQuery && this.#device) {
@@ -368,5 +371,12 @@ export class GPUProfiler {
368
371
  }
369
372
  }
370
373
 
374
+ function requirePositiveInteger(value, label) {
375
+ if (!Number.isInteger(value) || value <= 0) {
376
+ throw new Error(`${label} must be a positive integer.`);
377
+ }
378
+ return value;
379
+ }
380
+
371
381
 
372
382
  export default GPUProfiler;
@@ -99,7 +99,10 @@ export async function resolveKernelPathForModel(options = {}) {
99
99
 
100
100
  if (!manifest) return null;
101
101
 
102
- const modelConfig = parseModelConfigFromManifest(manifest, runtimeConfig);
102
+ const modelConfig = parseModelConfigFromManifest(
103
+ manifest,
104
+ runtimeConfig?.inference?.modelOverrides ?? null
105
+ );
103
106
  const kernelPathState = resolveKernelPathState({
104
107
  manifest,
105
108
  runtimeConfig,