doppler-gpu 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +45 -29
  2. package/assets/doppler-runtime-map.svg +82 -0
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/client/runtime/model-session.d.ts +1 -0
  11. package/src/client/runtime/model-session.js +9 -0
  12. package/src/config/conversion/README.md +4 -0
  13. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  14. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  26. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  29. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  30. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  32. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  33. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  34. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  36. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  39. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  41. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  42. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  43. package/src/config/execution-contract-check.js +14 -10
  44. package/src/config/merge-contract-check.js +1 -0
  45. package/src/config/merge.js +30 -0
  46. package/src/config/param-validator.d.ts +5 -0
  47. package/src/config/param-validator.js +42 -8
  48. package/src/config/required-inference-fields-contract-check.js +14 -0
  49. package/src/config/schema/debug.schema.d.ts +2 -0
  50. package/src/config/schema/debug.schema.js +1 -0
  51. package/src/config/schema/doppler.schema.d.ts +5 -1
  52. package/src/config/schema/doppler.schema.js +82 -292
  53. package/src/config/schema/inference.schema.d.ts +17 -3
  54. package/src/config/schema/kvcache.schema.d.ts +19 -3
  55. package/src/config/schema/kvcache.schema.js +2 -1
  56. package/src/config/schema/manifest.schema.d.ts +38 -1
  57. package/src/config/schema/manifest.schema.js +18 -0
  58. package/src/config/schema/policy-schema-registry.json +5 -0
  59. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  60. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  61. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  62. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  63. package/src/config/transforms/execution-graph-transforms.js +8 -0
  64. package/src/converter/conversion-plan.js +2 -0
  65. package/src/converter/parsers/transformer.js +27 -1
  66. package/src/converter/quantization-info.js +16 -4
  67. package/src/converter/rope-config.js +55 -3
  68. package/src/experimental/training/distillation/student-fixture.js +1 -1
  69. package/src/experimental/training/lora-pipeline.js +1 -1
  70. package/src/experimental/training/lora.js +2 -2
  71. package/src/experimental/training/suite.js +1 -1
  72. package/src/gpu/device.js +54 -11
  73. package/src/gpu/kernels/README.md +8 -1
  74. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  75. package/src/gpu/kernels/dequant.d.ts +6 -6
  76. package/src/gpu/kernels/dequant.js +17 -6
  77. package/src/gpu/kernels/linear-attention-core.js +1 -6
  78. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  79. package/src/gpu/kernels/matmul-selection.js +9 -2
  80. package/src/gpu/kernels/matmul.d.ts +1 -1
  81. package/src/gpu/kernels/matmul.js +63 -9
  82. package/src/gpu/kernels/moe.d.ts +5 -1
  83. package/src/gpu/kernels/moe.js +5 -2
  84. package/src/gpu/kernels/softmax.d.ts +3 -3
  85. package/src/gpu/kernels/softmax.js +9 -2
  86. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  87. package/src/gpu/kernels/types.d.ts +1 -1
  88. package/src/gpu/profiler.js +11 -1
  89. package/src/inference/browser-harness-model-helpers.js +4 -1
  90. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  91. package/src/inference/browser-harness-text-helpers.js +357 -116
  92. package/src/inference/browser-harness.d.ts +3 -3
  93. package/src/inference/browser-harness.js +266 -11
  94. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  95. package/src/inference/kv-cache/quantized.js +16 -2
  96. package/src/inference/kv-cache/tiered.js +50 -15
  97. package/src/inference/moe-router.js +38 -7
  98. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  99. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  100. package/src/inference/pipelines/text/config.d.ts +5 -1
  101. package/src/inference/pipelines/text/config.js +135 -112
  102. package/src/inference/pipelines/text/embed.d.ts +1 -0
  103. package/src/inference/pipelines/text/embed.js +39 -12
  104. package/src/inference/pipelines/text/execution-plan.js +8 -2
  105. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  106. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  107. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  108. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  109. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  110. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  111. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  112. package/src/inference/pipelines/text/generator-steps.js +3 -0
  113. package/src/inference/pipelines/text/generator.d.ts +1 -0
  114. package/src/inference/pipelines/text/generator.js +14 -1
  115. package/src/inference/pipelines/text/init-rope.js +54 -6
  116. package/src/inference/pipelines/text/init.js +63 -15
  117. package/src/inference/pipelines/text/layer.js +32 -5
  118. package/src/inference/pipelines/text/linear-attention.js +12 -8
  119. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  120. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  121. package/src/inference/pipelines/text/logits/index.js +44 -87
  122. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  123. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  124. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  125. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  126. package/src/inference/pipelines/text/logits/utils.js +11 -0
  127. package/src/inference/pipelines/text/lora-apply.js +6 -4
  128. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  129. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  130. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  131. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  132. package/src/inference/pipelines/text/ops.d.ts +3 -0
  133. package/src/inference/pipelines/text/ops.js +26 -1
  134. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  135. package/src/inference/pipelines/text/sampling.js +52 -10
  136. package/src/inference/pipelines/text/weights.js +7 -2
  137. package/src/inference/pipelines/text.js +24 -4
  138. package/src/inference/tokenizers/behavior-flags.js +23 -2
  139. package/src/loader/final-weights-loader.js +12 -1
  140. package/src/loader/tensors/tensor-loader.js +37 -3
  141. package/src/rules/execution-rules-contract-check.js +12 -3
  142. package/src/rules/inference/capability-transforms.rules.json +14 -2
  143. package/src/rules/inference/config.rules.json +26 -2
  144. package/src/rules/inference/dtype.rules.json +12 -5
  145. package/src/rules/inference/execution.rules.json +5 -19
  146. package/src/rules/inference/moe.rules.json +97 -0
  147. package/src/rules/loader/weights.rules.json +1 -0
  148. package/src/tooling/command-api-constants.js +2 -2
  149. package/src/tooling/command-api-family-normalizers.js +2 -1
  150. package/src/tooling/command-api.d.ts +1 -1
  151. package/src/tooling/command-runner-shared.d.ts +1 -1
  152. package/src/tooling/command-runner-shared.js +3 -1
  153. package/src/tooling/node-converter.js +20 -0
  154. package/src/tooling/node-source-runtime.js +5 -1
  155. package/src/version.js +1 -1
@@ -28,6 +28,15 @@ function asNumberArray(value) {
28
28
  return normalized.map((entry) => Math.trunc(entry));
29
29
  }
30
30
 
31
+ function asPositiveNumberArray(value) {
32
+ if (!Array.isArray(value)) return null;
33
+ const normalized = value.map((entry) => asFiniteNumber(entry));
34
+ if (normalized.some((entry) => entry == null || entry <= 0)) {
35
+ return null;
36
+ }
37
+ return normalized.map((entry) => Number(entry));
38
+ }
39
+
31
40
  function normalizeRoPEType(value) {
32
41
  if (typeof value !== 'string') return null;
33
42
  const normalized = value.trim().toLowerCase();
@@ -90,6 +99,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
90
99
  yarnBetaFast: null,
91
100
  yarnBetaSlow: null,
92
101
  yarnOriginalMaxPos: null,
102
+ longropeShortFactor: null,
103
+ longropeLongFactor: null,
104
+ longropeOriginalMaxPos: null,
93
105
  };
94
106
  }
95
107
 
@@ -98,6 +110,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
98
110
  let yarnBetaFast = null;
99
111
  let yarnBetaSlow = null;
100
112
  let yarnOriginalMaxPos = null;
113
+ let longropeShortFactor = null;
114
+ let longropeLongFactor = null;
115
+ let longropeOriginalMaxPos = null;
101
116
 
102
117
  if (ropeScalingType == null) {
103
118
  if (factor != null && factor > 0 && factor !== 1.0) {
@@ -123,6 +138,23 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
123
138
  yarnBetaSlow = betaSlow;
124
139
  yarnOriginalMaxPos = origMaxPos;
125
140
  }
141
+ if (ropeScalingType === 'longrope') {
142
+ const shortFactor = asPositiveNumberArray(ropeScalingConfig.short_factor);
143
+ const longFactor = asPositiveNumberArray(ropeScalingConfig.long_factor);
144
+ const origMaxPos = asFiniteNumber(ropeScalingConfig.original_max_position_embeddings);
145
+ if (!shortFactor || !longFactor || origMaxPos == null) {
146
+ throw new Error(
147
+ 'LongRoPE scaling detected but required params missing in HF config. ' +
148
+ 'LongRoPE requires short_factor, long_factor, and original_max_position_embeddings. ' +
149
+ `Got: short_factor=${Array.isArray(ropeScalingConfig.short_factor) ? ropeScalingConfig.short_factor.length : ropeScalingConfig.short_factor}, ` +
150
+ `long_factor=${Array.isArray(ropeScalingConfig.long_factor) ? ropeScalingConfig.long_factor.length : ropeScalingConfig.long_factor}, ` +
151
+ `original_max_position_embeddings=${origMaxPos}`
152
+ );
153
+ }
154
+ longropeShortFactor = shortFactor;
155
+ longropeLongFactor = longFactor;
156
+ longropeOriginalMaxPos = origMaxPos;
157
+ }
126
158
 
127
159
  return {
128
160
  ropeScalingType,
@@ -130,6 +162,9 @@ function resolveScalingConfig(ropeScalingConfig, options = {}) {
130
162
  yarnBetaFast,
131
163
  yarnBetaSlow,
132
164
  yarnOriginalMaxPos,
165
+ longropeShortFactor,
166
+ longropeLongFactor,
167
+ longropeOriginalMaxPos,
133
168
  };
134
169
  }
135
170
 
@@ -140,7 +175,9 @@ function hasScalingDirective(ropeScalingConfig) {
140
175
  || ropeScalingConfig.factor != null
141
176
  || ropeScalingConfig.beta_fast != null
142
177
  || ropeScalingConfig.beta_slow != null
143
- || ropeScalingConfig.original_max_position_embeddings != null;
178
+ || ropeScalingConfig.original_max_position_embeddings != null
179
+ || ropeScalingConfig.short_factor != null
180
+ || ropeScalingConfig.long_factor != null;
144
181
  }
145
182
 
146
183
  function hasMeaningfulScalingConfig(resolvedScaling) {
@@ -149,7 +186,10 @@ function hasMeaningfulScalingConfig(resolvedScaling) {
149
186
  || resolvedScaling.ropeScalingFactor !== DEFAULT_MANIFEST_INFERENCE.rope.ropeScalingFactor
150
187
  || resolvedScaling.yarnBetaFast != null
151
188
  || resolvedScaling.yarnBetaSlow != null
152
- || resolvedScaling.yarnOriginalMaxPos != null;
189
+ || resolvedScaling.yarnOriginalMaxPos != null
190
+ || resolvedScaling.longropeShortFactor != null
191
+ || resolvedScaling.longropeLongFactor != null
192
+ || resolvedScaling.longropeOriginalMaxPos != null;
153
193
  }
154
194
 
155
195
  function isSameScalingConfig(left, right) {
@@ -157,7 +197,10 @@ function isSameScalingConfig(left, right) {
157
197
  && left.ropeScalingFactor === right.ropeScalingFactor
158
198
  && left.yarnBetaFast === right.yarnBetaFast
159
199
  && left.yarnBetaSlow === right.yarnBetaSlow
160
- && left.yarnOriginalMaxPos === right.yarnOriginalMaxPos;
200
+ && left.yarnOriginalMaxPos === right.yarnOriginalMaxPos
201
+ && JSON.stringify(left.longropeShortFactor ?? null) === JSON.stringify(right.longropeShortFactor ?? null)
202
+ && JSON.stringify(left.longropeLongFactor ?? null) === JSON.stringify(right.longropeLongFactor ?? null)
203
+ && left.longropeOriginalMaxPos === right.longropeOriginalMaxPos;
161
204
  }
162
205
 
163
206
  function failOnConflictingScaling(sourceLabel, canonicalScaling, candidateScaling) {
@@ -210,6 +253,9 @@ export function buildRoPEConfig(converterInference, config) {
210
253
  yarnBetaFast: configuredRoPE.yarnBetaFast ?? null,
211
254
  yarnBetaSlow: configuredRoPE.yarnBetaSlow ?? null,
212
255
  yarnOriginalMaxPos: configuredRoPE.yarnOriginalMaxPos ?? null,
256
+ longropeShortFactor: configuredRoPE.longropeShortFactor ?? null,
257
+ longropeLongFactor: configuredRoPE.longropeLongFactor ?? null,
258
+ longropeOriginalMaxPos: configuredRoPE.longropeOriginalMaxPos ?? null,
213
259
  };
214
260
 
215
261
  if (ropeScaling) {
@@ -253,6 +299,9 @@ export function buildRoPEConfig(converterInference, config) {
253
299
  yarnBetaFast: configuredRoPE.ropeLocalYarnBetaFast ?? globalScaling.yarnBetaFast,
254
300
  yarnBetaSlow: configuredRoPE.ropeLocalYarnBetaSlow ?? globalScaling.yarnBetaSlow,
255
301
  yarnOriginalMaxPos: configuredRoPE.ropeLocalYarnOriginalMaxPos ?? globalScaling.yarnOriginalMaxPos,
302
+ longropeShortFactor: globalScaling.longropeShortFactor,
303
+ longropeLongFactor: globalScaling.longropeLongFactor,
304
+ longropeOriginalMaxPos: globalScaling.longropeOriginalMaxPos,
256
305
  }
257
306
  : { ...globalScaling };
258
307
  if (ropeScaling) {
@@ -323,6 +372,9 @@ export function buildRoPEConfig(converterInference, config) {
323
372
  yarnBetaFast: globalScaling.yarnBetaFast,
324
373
  yarnBetaSlow: globalScaling.yarnBetaSlow,
325
374
  yarnOriginalMaxPos: globalScaling.yarnOriginalMaxPos,
375
+ longropeShortFactor: globalScaling.longropeShortFactor,
376
+ longropeLongFactor: globalScaling.longropeLongFactor,
377
+ longropeOriginalMaxPos: globalScaling.longropeOriginalMaxPos,
326
378
  ropeLocalScalingType: localScaling.ropeScalingType,
327
379
  ropeLocalScalingFactor: localScaling.ropeScalingFactor,
328
380
  ropeLocalYarnBetaFast: localScaling.yarnBetaFast,
@@ -421,7 +421,7 @@ function createDistillStudentProjectionModelFixture(overrides = {}, options = {}
421
421
  const rows = Number.isFinite(inputTensor?.shape?.[0]) ? inputTensor.shape[0] : 1;
422
422
  return tape.record(
423
423
  OpType.MATMUL,
424
- (a, b) => runMatmul(a, b, rows, outputDim, embeddingDim, { transposeB: false }),
424
+ (a, b) => runMatmul(a, b, rows, outputDim, embeddingDim, { transposeB: false, outputDtype: 'f32' }),
425
425
  [inputTensor, projectionWeight],
426
426
  { M: rows, N: outputDim, K: embeddingDim, transposeB: false }
427
427
  );
@@ -400,7 +400,7 @@ function createToyLoraModel(workload) {
400
400
  const batchSize = Number.isInteger(inputTensor?.shape?.[0]) ? inputTensor.shape[0] : 1;
401
401
  const baseLogits = await tape.record(
402
402
  OpType.MATMUL,
403
- (a, b) => runMatmul(a, b, batchSize, 2, 3, { transposeB: false }),
403
+ (a, b) => runMatmul(a, b, batchSize, 2, 3, { transposeB: false, outputDtype: 'f32' }),
404
404
  [inputTensor, baseWeight],
405
405
  { M: batchSize, N: 2, K: 3, transposeB: false }
406
406
  );
@@ -68,13 +68,13 @@ export class LoraAdapter {
68
68
  const [tokens] = input.shape;
69
69
  const down = await tape.record(
70
70
  OpType.MATMUL,
71
- (a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0]),
71
+ (a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0], { outputDtype: input.dtype }),
72
72
  [input, this.A],
73
73
  { M: tokens, N: this.rank, K: this.A.shape[0] }
74
74
  );
75
75
  const up = await tape.record(
76
76
  OpType.MATMUL,
77
- (a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank),
77
+ (a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank, { outputDtype: down.dtype }),
78
78
  [down, this.B],
79
79
  { M: tokens, N: this.B.shape[1], K: this.rank }
80
80
  );
@@ -821,7 +821,7 @@ export function createToyModelFixture(overrides = {}) {
821
821
  async forward(inputTensor, tape) {
822
822
  return tape.record(
823
823
  OpType.MATMUL,
824
- (a, b) => runMatmul(a, b, 2, 2, 3, { transposeB: false }),
824
+ (a, b) => runMatmul(a, b, 2, 2, 3, { transposeB: false, outputDtype: 'f32' }),
825
825
  [inputTensor, baseWeight],
826
826
  { M: 2, N: 2, K: 3, transposeB: false }
827
827
  );
package/src/gpu/device.js CHANGED
@@ -369,6 +369,59 @@ function buildFeatureRequests(available) {
369
369
  return requested;
370
370
  }
371
371
 
372
+ function addFeatureRequestCandidate(candidates, seen, features) {
373
+ const unique = features.filter((feature, index) => features.indexOf(feature) === index);
374
+ const key = unique.join('|');
375
+ if (seen.has(key)) return;
376
+ seen.add(key);
377
+ candidates.push(unique);
378
+ }
379
+
380
+ function buildFeatureRequestCandidates(requestedFeatures) {
381
+ const requested = Array.isArray(requestedFeatures) ? requestedFeatures : [];
382
+ const candidates = [];
383
+ const seen = new Set();
384
+ addFeatureRequestCandidate(candidates, seen, requested);
385
+ addFeatureRequestCandidate(
386
+ candidates,
387
+ seen,
388
+ requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY)
389
+ );
390
+ addFeatureRequestCandidate(
391
+ candidates,
392
+ seen,
393
+ requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY && feature !== FEATURES.SUBGROUPS)
394
+ );
395
+ addFeatureRequestCandidate(
396
+ candidates,
397
+ seen,
398
+ requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY && feature !== FEATURES.SHADER_F16)
399
+ );
400
+ addFeatureRequestCandidate(candidates, seen, []);
401
+ return candidates;
402
+ }
403
+
404
+ async function requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits) {
405
+ const candidates = buildFeatureRequestCandidates(requestedFeatures);
406
+ let lastError = null;
407
+ for (const features of candidates) {
408
+ try {
409
+ if (features.length === 0) {
410
+ return await adapter.requestDevice();
411
+ }
412
+ return await adapter.requestDevice({
413
+ requiredFeatures: features,
414
+ requiredLimits: limits,
415
+ });
416
+ } catch (error) {
417
+ lastError = error;
418
+ const label = features.length > 0 ? features.join(', ') : 'none';
419
+ log.warn('GPU', 'Failed to request device with features [' + label + ']: ' + error.message);
420
+ }
421
+ }
422
+ throw lastError || new Error('Failed to request WebGPU device');
423
+ }
424
+
372
425
 
373
426
  function buildLimits(adapter) {
374
427
  const adapterLimits = adapter.limits;
@@ -457,17 +510,7 @@ export async function initDevice() {
457
510
  // Get adapter info (adapter.info is synchronous in modern WebGPU)
458
511
  const adapterInfo = adapter.info || { vendor: 'unknown', architecture: 'unknown', device: 'unknown', description: '' };
459
512
 
460
- try {
461
- gpuDevice = await adapter.requestDevice({
462
- requiredFeatures: requestedFeatures,
463
- requiredLimits: limits,
464
- });
465
- } catch (e) {
466
- // Fallback: request device without optional features
467
- const lostFeatures = requestedFeatures.length > 0 ? requestedFeatures.join(', ') : 'none';
468
- log.warn('GPU', 'Failed to request device with features [' + lostFeatures + '], trying minimal config: ' + (e).message);
469
- gpuDevice = await adapter.requestDevice();
470
- }
513
+ gpuDevice = await requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits);
471
514
 
472
515
  if (!gpuDevice) {
473
516
  throw createDopplerError(ERROR_CODES.GPU_DEVICE_FAILED, 'Failed to create WebGPU device');
@@ -50,7 +50,10 @@ Three ways to make kernels flexible:
50
50
  | Change per dispatch | select different | recompile | yes |
51
51
  | Code duplication | high | minimal | none |
52
52
 
53
- **DOPPLER uses entry points** over override constants - more code duplication but simpler pipeline management.
53
+ Use the topology test from the WGSL style guide. Entry points are for algorithm
54
+ or synchronization changes. Override constants are for model/load-time
55
+ parameters. Uniforms are for per-dispatch values. The registry records which
56
+ choice a variant made; WGSL files must not become a second policy layer.
54
57
 
55
58
  ## Entry Points
56
59
 
@@ -123,6 +126,10 @@ Kernel operation IDs, variant IDs, WGSL filenames, entry points, feature
123
126
  requirements, bindings, uniforms, and metadata are defined in
124
127
  `src/config/kernels/registry.json`.
125
128
 
129
+ The management front door is
130
+ [`docs/developer-guides/config-source-of-truth.md`](../../../docs/developer-guides/config-source-of-truth.md#wgsl-kernel-management).
131
+ Do not add separate kernel inventories in docs, scripts, or runtime helpers.
132
+
126
133
  Use `npm run kernels:registry:check` to audit the on-disk WGSL inventory
127
134
  against the registry. This README must not define separate filename suffix
128
135
  rules.
@@ -141,12 +141,14 @@ async function runAttentionBackwardCore(
141
141
  dV = await runMatmul(sTransposed, dHead.buffer, seqLen, headDim, seqLen, {
142
142
  transposeB: false,
143
143
  bDtype: 'f32',
144
+ outputDtype: 'f32',
144
145
  });
145
146
 
146
147
  vTransposed = await runTranspose(vHead, seqLen, headDim);
147
148
  dS = await runMatmul(dHead, vTransposed.buffer, seqLen, seqLen, headDim, {
148
149
  transposeB: false,
149
150
  bDtype: 'f32',
151
+ outputDtype: 'f32',
150
152
  });
151
153
  dQK = causal
152
154
  ? await runBackwardKernel(
@@ -166,12 +168,14 @@ async function runAttentionBackwardCore(
166
168
  transposeB: false,
167
169
  alpha: scale,
168
170
  bDtype: 'f32',
171
+ outputDtype: 'f32',
169
172
  });
170
173
  dQKTransposed = await runTranspose(dQK, seqLen, seqLen);
171
174
  dK = await runMatmul(dQKTransposed, qHead.buffer, seqLen, headDim, seqLen, {
172
175
  transposeB: false,
173
176
  alpha: scale,
174
177
  bDtype: 'f32',
178
+ outputDtype: 'f32',
175
179
  });
176
180
 
177
181
  const copyEncoder = getDevice().createCommandEncoder();
@@ -240,12 +244,14 @@ async function runAttentionBackwardCore(
240
244
  dV = await recordMatmul(recorder, sTransposed, dHead.buffer, seqLen, headDim, seqLen, {
241
245
  transposeB: false,
242
246
  bDtype: 'f32',
247
+ outputDtype: 'f32',
243
248
  });
244
249
 
245
250
  vTransposed = await recordTranspose(recorder, vHead, seqLen, headDim);
246
251
  dS = await recordMatmul(recorder, dHead, vTransposed.buffer, seqLen, seqLen, headDim, {
247
252
  transposeB: false,
248
253
  bDtype: 'f32',
254
+ outputDtype: 'f32',
249
255
  });
250
256
  dQK = causal
251
257
  ? await recordBackwardKernel(
@@ -266,12 +272,14 @@ async function runAttentionBackwardCore(
266
272
  transposeB: false,
267
273
  alpha: scale,
268
274
  bDtype: 'f32',
275
+ outputDtype: 'f32',
269
276
  });
270
277
  dQKTransposed = await recordTranspose(recorder, dQK, seqLen, seqLen);
271
278
  dK = await recordMatmul(recorder, dQKTransposed, qHead.buffer, seqLen, headDim, seqLen, {
272
279
  transposeB: false,
273
280
  alpha: scale,
274
281
  bDtype: 'f32',
282
+ outputDtype: 'f32',
275
283
  });
276
284
 
277
285
  encoder.copyBufferToBuffer(dQ.buffer, 0, gradQBuf, qOffset, headBytes);
@@ -22,7 +22,7 @@ export interface DequantOptions extends OutputBufferOptions, OutputOffsetOptions
22
22
  /**
23
23
  * Select the best dequantization kernel variant
24
24
  */
25
- export declare function selectDequantKernel(options?: DequantOptions): string;
25
+ export declare function selectDequantKernel(options: DequantOptions): string;
26
26
 
27
27
  /**
28
28
  * Create bind group layout for dequant operation
@@ -35,7 +35,7 @@ export declare function createDequantBindGroupLayout(): GPUBindGroupLayout;
35
35
  export declare function dequantize(
36
36
  quantized: GPUBuffer,
37
37
  numBlocks: number,
38
- options?: DequantOptions
38
+ options: DequantOptions
39
39
  ): Promise<Tensor>;
40
40
 
41
41
  /**
@@ -45,7 +45,7 @@ export declare function dequantizeRowwise(
45
45
  quantized: GPUBuffer,
46
46
  rows: number,
47
47
  K: number,
48
- options?: OutputBufferOptions & OutputDtypeOptions
48
+ options: OutputBufferOptions & OutputDtypeOptions
49
49
  ): Promise<Tensor>;
50
50
 
51
51
  /**
@@ -69,7 +69,7 @@ export declare function dequantizeMXFP4Expert(
69
69
  numExperts: number,
70
70
  outDim: number,
71
71
  numGroups: number,
72
- options?: DequantOptions
72
+ options: DequantOptions
73
73
  ): Promise<Tensor>;
74
74
 
75
75
  /**
@@ -78,7 +78,7 @@ export declare function dequantizeMXFP4Expert(
78
78
  export declare function dequantizeQ6K(
79
79
  quantized: GPUBuffer,
80
80
  numBlocks: number,
81
- options?: DequantOptions
81
+ options: DequantOptions
82
82
  ): Promise<Tensor>;
83
83
 
84
84
  /**
@@ -88,5 +88,5 @@ export declare function recordDequantize(
88
88
  recorder: CommandRecorder,
89
89
  quantized: GPUBuffer,
90
90
  numBlocks: number,
91
- options?: DequantOptions
91
+ options: DequantOptions
92
92
  ): Promise<Tensor>;
@@ -11,10 +11,17 @@ import { releaseUniformBuffer } from '../uniform-cache.js';
11
11
  import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
12
12
  import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
13
13
 
14
+ function requireOutputDtype(value, label) {
15
+ if (value !== 'f16' && value !== 'f32') {
16
+ throw new Error(`[${label}] outputDtype is required and must be "f16" or "f32", got ${String(value)}.`);
17
+ }
18
+ return value;
19
+ }
14
20
 
15
21
  export function selectDequantKernel(options = {}) {
16
22
  const capabilities = getKernelCapabilities();
17
- const { useVec4 = true, outputDtype = 'f32' } = options;
23
+ const { useVec4 = true } = options;
24
+ const outputDtype = requireOutputDtype(options.outputDtype, 'selectDequantKernel');
18
25
 
19
26
  const wantsF16Out = outputDtype === 'f16' && capabilities.hasF16;
20
27
  return selectKernelRuleValue(
@@ -90,8 +97,8 @@ export async function dequantize(
90
97
  const {
91
98
  outputOffset = 0,
92
99
  outputBuffer = null,
93
- outputDtype = 'f32',
94
100
  } = options;
101
+ const outputDtype = requireOutputDtype(options.outputDtype, 'dequantize');
95
102
 
96
103
  if (outputDtype === 'f16' && capabilities?.hasF16 !== true) {
97
104
  throw new Error('[dequantize] f16 output requires shader-f16 support.');
@@ -160,7 +167,8 @@ export async function dequantizeRowwise(
160
167
  ) {
161
168
  const device = getDevice();
162
169
  const capabilities = getKernelCapabilities();
163
- const { outputBuffer = null, outputDtype = 'f16' } = options;
170
+ const { outputBuffer = null } = options;
171
+ const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeRowwise');
164
172
  if (outputDtype === 'f16' && capabilities?.hasF16 !== true) {
165
173
  throw new Error('[dequantizeRowwise] f16 output requires shader-f16 support.');
166
174
  }
@@ -296,11 +304,11 @@ export async function dequantizeMXFP4Expert(
296
304
  const device = getDevice();
297
305
  const {
298
306
  outputBuffer = null,
299
- outputDtype = 'f32',
300
307
  modelType = null,
301
308
  groupSize = 32,
302
309
  dequantTileShape = 'scalar',
303
310
  } = options;
311
+ const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeMXFP4Expert');
304
312
  const caps = getKernelCapabilities();
305
313
 
306
314
  const variant = selectKernelRuleValue('dequant', 'mxfp4ExpertVariant', {
@@ -375,8 +383,11 @@ export async function dequantizeQ6K(
375
383
  const {
376
384
  outputOffset = 0,
377
385
  outputBuffer = null,
378
- outputDtype = 'f16', // Q6_K always outputs f16 for now
379
386
  } = options;
387
+ const outputDtype = requireOutputDtype(options.outputDtype, 'dequantizeQ6K');
388
+ if (outputDtype !== 'f16') {
389
+ throw new Error(`[dequantizeQ6K] outputDtype must be "f16", got ${outputDtype}.`);
390
+ }
380
391
 
381
392
  // Q6_K only has f16 output kernel currently
382
393
  const pipeline = await getPipelineFast('dequant', 'q6k_f16out');
@@ -451,8 +462,8 @@ export async function recordDequantize(
451
462
  const {
452
463
  outputOffset = 0,
453
464
  outputBuffer = null,
454
- outputDtype = 'f32',
455
465
  } = options;
466
+ const outputDtype = requireOutputDtype(options.outputDtype, 'recordDequantize');
456
467
 
457
468
  // Select kernel
458
469
  const variant = selectDequantKernel({ ...options, outputDtype });
@@ -374,12 +374,7 @@ function requireGpuBuffer(buffer, label) {
374
374
  }
375
375
 
376
376
  function resolveOutputDtype(outputDtype) {
377
- const normalized = selectRuleValue(
378
- 'shared',
379
- 'dtype',
380
- 'f16OrF32FromDtype',
381
- { dtype: outputDtype === undefined ? 'f32' : outputDtype }
382
- );
377
+ const normalized = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: outputDtype });
383
378
  if (normalized === 'f16' || normalized === 'f32') {
384
379
  return normalized;
385
380
  }
@@ -26,10 +26,10 @@ export declare function isFusedQ4KDisabled(options?: {
26
26
 
27
27
  export declare function toMatmulDtype(dtype: string | null | undefined): 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16';
28
28
 
29
- export declare function selectMatmulKernel(options?: {
29
+ export declare function selectMatmulKernel(options: {
30
30
  preferF16?: boolean;
31
31
  useVec4?: boolean;
32
- outputDtype?: TensorDtype | 'f16' | 'f32';
32
+ outputDtype: TensorDtype | 'f16' | 'f32';
33
33
  aDtype?: 'f16' | 'f32' | null;
34
34
  bDtype?: 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16' | null;
35
35
  isPrefill?: boolean;
@@ -129,13 +129,16 @@ export function selectMatmulKernel(options = {}) {
129
129
  const {
130
130
  preferF16 = true,
131
131
  useVec4 = false,
132
- outputDtype = 'f32',
133
132
  aDtype = null,
134
133
  bDtype = null,
135
134
  isPrefill = false,
136
135
  prefillRows = 0,
137
136
  transposeB = true,
138
137
  } = options;
138
+ const outputDtype = options.outputDtype;
139
+ if (outputDtype !== 'f16' && outputDtype !== 'f32') {
140
+ throw new Error(`[Matmul] selectMatmulKernel requires outputDtype "f16" or "f32", got ${String(outputDtype)}.`);
141
+ }
139
142
  const { tiledPrefillMinRows } = getKernelThresholds().matmul;
140
143
 
141
144
  const inputsAreF16 = aDtype === 'f16' && bDtype === 'f16';
@@ -334,7 +337,11 @@ function resolveMatmulOverride(
334
337
  if (!weightDtypeOk) {
335
338
  const overridePolicy = selectKernelRuleValue(
336
339
  'matmul', 'weightDtypeMismatchPolicy',
337
- { requiredWeightDtype, actualWeightDtype: bDtype }
340
+ {
341
+ requiredWeightDtype,
342
+ actualWeightDtype: bDtype,
343
+ hasSubgroups: capabilities?.hasSubgroups === true,
344
+ }
338
345
  );
339
346
  if (overridePolicy === 'fallthrough') {
340
347
  return null;
@@ -56,7 +56,7 @@ export declare function isFusedQ4KDisabled(): boolean;
56
56
  /**
57
57
  * Select the best matmul kernel variant
58
58
  */
59
- export declare function selectMatmulKernel(options?: MatmulOptions): string;
59
+ export declare function selectMatmulKernel(options: MatmulOptions): string;
60
60
 
61
61
  /**
62
62
  * Create bind group layout for matmul operation
@@ -212,7 +212,7 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
212
212
  return entries;
213
213
  }
214
214
 
215
- function resolvePreferredWeightDtype(variant, hasQ4KMaterialization) {
215
+ function resolvePreferredWeightDtype(variant, hasQ4KMaterialization, capabilities) {
216
216
  if (typeof variant !== 'string' || variant.length === 0) {
217
217
  return null;
218
218
  }
@@ -232,9 +232,51 @@ function resolvePreferredWeightDtype(variant, hasQ4KMaterialization) {
232
232
  return selectKernelRuleValue('matmul', 'preferredWeightDtype', {
233
233
  variantWeightDtype,
234
234
  hasQ4KMaterialization,
235
+ hasSubgroups: capabilities?.hasSubgroups === true,
235
236
  });
236
237
  }
237
238
 
239
+ function buildF16CapabilityErrorDetail({
240
+ role,
241
+ layerIdx,
242
+ pathVariant,
243
+ preferredWeightDtype,
244
+ weightDtype,
245
+ weightLabel,
246
+ weightLayout,
247
+ weightShape,
248
+ hasQ4KMaterialization,
249
+ }) {
250
+ const parts = [];
251
+ if (role) parts.push(`role=${role}`);
252
+ if (Number.isFinite(layerIdx)) parts.push(`layer=${layerIdx}`);
253
+ if (pathVariant) parts.push(`variant=${pathVariant}`);
254
+ if (preferredWeightDtype) parts.push(`preferredWeightDtype=${preferredWeightDtype}`);
255
+ if (weightDtype) parts.push(`weightDtype=${weightDtype}`);
256
+ if (weightLabel) parts.push(`label=${weightLabel}`);
257
+ if (weightLayout) parts.push(`layout=${weightLayout}`);
258
+ if (weightShape) parts.push(`shape=${weightShape}`);
259
+ if (hasQ4KMaterialization) parts.push('q4kMaterialization=true');
260
+ return parts.length > 0 ? ` (${parts.join(', ')})` : '';
261
+ }
262
+
263
+ function requireMatmulOutputDtype(dtype, opLabel) {
264
+ if (dtype === 'f16' || dtype === 'f32') {
265
+ return dtype;
266
+ }
267
+ throw new Error(`[${opLabel}] options.outputDtype is required and must be "f16" or "f32", got ${String(dtype)}.`);
268
+ }
269
+
270
+ function requireMatmulWeightDtype(dtype, opLabel, details = []) {
271
+ if (dtype != null && dtype !== '') {
272
+ return dtype;
273
+ }
274
+ const suffix = details.filter(Boolean).length > 0
275
+ ? ` (${details.filter(Boolean).join(', ')})`
276
+ : '';
277
+ throw new Error(`[${opLabel}] B dtype is required for matmul dispatch${suffix}.`);
278
+ }
279
+
238
280
  async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
239
281
  const isRecord = Boolean(recorder);
240
282
  const mode = isRecord ? 'record' : 'run';
@@ -254,7 +296,7 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
254
296
  const phase = resolveMatmulPhase(M, options.phaseOverride ?? null);
255
297
  const pathVariant = getKernelPathMatmulVariant(options.role, phase, options.layerIdx, options.kernelPath);
256
298
  const hasQ4KMat = isWeightBuffer(B) && B.materializations?.q4k?.buffer != null;
257
- const preferredWeightDtype = resolvePreferredWeightDtype(pathVariant, hasQ4KMat);
299
+ const preferredWeightDtype = resolvePreferredWeightDtype(pathVariant, hasQ4KMat, capabilities);
258
300
  const resolvedWeight = resolveWeightBufferMaterialization(B, preferredWeightDtype);
259
301
  const bBuffer = getBuffer(resolvedWeight);
260
302
  const weightDtype = getWeightDtype(resolvedWeight);
@@ -278,20 +320,32 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
278
320
  validateMatmulDimensions(opLabel, M, N, K);
279
321
 
280
322
  const aDtype = toMatmulDtype(A.dtype);
281
- const bDtype = toMatmulDtype(weightDtype ?? options.bDtype);
282
- const requestedOutputDtype = options.outputDtype || A.dtype;
323
+ const rawBDtype = requireMatmulWeightDtype(weightDtype ?? options.bDtype, opLabel, [
324
+ options.role ? `role=${options.role}` : null,
325
+ Number.isFinite(options.layerIdx) ? `layer=${options.layerIdx}` : null,
326
+ weightLabel ? `label=${weightLabel}` : null,
327
+ ]);
328
+ const bDtype = toMatmulDtype(rawBDtype);
329
+ const requestedOutputDtype = requireMatmulOutputDtype(options.outputDtype, opLabel);
283
330
 
284
331
  if (bDtype === 'f16' && capabilities?.hasF16 !== true) {
285
- throw new Error(`[${opLabel}] f16 weights require shader-f16 support.`);
332
+ const detail = buildF16CapabilityErrorDetail({
333
+ role: options.role,
334
+ layerIdx: options.layerIdx,
335
+ pathVariant,
336
+ preferredWeightDtype,
337
+ weightDtype,
338
+ weightLabel,
339
+ weightLayout,
340
+ weightShape,
341
+ hasQ4KMaterialization: hasQ4KMat,
342
+ });
343
+ throw new Error(`[${opLabel}] f16 weights require shader-f16 support.${detail}`);
286
344
  }
287
345
  if (requestedOutputDtype === 'f16' && capabilities?.hasF16 !== true) {
288
346
  throw new Error(`[${opLabel}] f16 output requires shader-f16 support.`);
289
347
  }
290
348
 
291
- if (!isRecord && isTraceEnabled('kernels') && !weightDtype && !options.bDtype && M <= 2) {
292
- log.warn('Matmul', `runMatmul: B buffer dtype unknown! size=${bBuffer.size}, M=${M}, N=${N}, K=${K}. Assuming f32.`);
293
- }
294
-
295
349
  validateMatmulOffsets(opLabel, aOffset, bOffset, cOffset);
296
350
 
297
351
  const runtimeSession = getRuntimeConfig().inference?.session;
@@ -19,6 +19,10 @@ export interface MoEOptions extends OutputBufferOptions {
19
19
  perExpertScale?: GPUBuffer | null;
20
20
  }
21
21
 
22
+ export interface MoEScatterOptions extends MoEOptions {
23
+ weightsDtype: 'f16' | 'f32';
24
+ }
25
+
22
26
  /** MoE gather result */
23
27
  export interface MoEGatherResult {
24
28
  gathered: Tensor;
@@ -105,5 +109,5 @@ export declare function runScatterAddDynamic(
105
109
  numTokens: number,
106
110
  hiddenSize: number,
107
111
  topK: number,
108
- options?: MoEOptions
112
+ options: MoEScatterOptions
109
113
  ): Promise<Tensor>;