doppler-gpu 0.4.10 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +95 -0
  2. package/README.md +40 -1
  3. package/models/catalog.json +637 -32
  4. package/package.json +24 -3
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +135 -13
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +69 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
  18. package/src/config/kernels/kernel-ref-digests.js +2 -1
  19. package/src/config/kernels/registry.json +73 -11
  20. package/src/config/merge.js +24 -0
  21. package/src/config/required-inference-fields-contract-check.js +2 -0
  22. package/src/config/schema/adapter.schema.d.ts +6 -1
  23. package/src/config/schema/adapter.schema.js +5 -0
  24. package/src/config/schema/converter.schema.d.ts +7 -0
  25. package/src/config/schema/converter.schema.js +15 -0
  26. package/src/config/schema/inference.schema.d.ts +8 -0
  27. package/src/config/schema/manifest.schema.d.ts +32 -1
  28. package/src/config/schema/manifest.schema.js +26 -0
  29. package/src/converter/conversion-plan.js +15 -1
  30. package/src/converter/core.js +5 -4
  31. package/src/converter/source-tensor-rules.d.ts +50 -0
  32. package/src/converter/source-tensor-rules.js +180 -0
  33. package/src/converter/source-tokenizer.d.ts +45 -0
  34. package/src/converter/source-tokenizer.js +155 -0
  35. package/src/experimental/adapters/artifact-contract.d.ts +83 -0
  36. package/src/experimental/adapters/artifact-contract.js +254 -0
  37. package/src/experimental/adapters/index.d.ts +25 -0
  38. package/src/experimental/adapters/index.js +12 -0
  39. package/src/experimental/adapters/lora-loader.js +24 -4
  40. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  41. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  42. package/src/experimental/training/artifacts.d.ts +160 -0
  43. package/src/experimental/training/attention-backward.d.ts +64 -0
  44. package/src/experimental/training/attention-forward.d.ts +22 -0
  45. package/src/experimental/training/attention-forward.js +82 -0
  46. package/src/experimental/training/autograd.d.ts +76 -0
  47. package/src/experimental/training/checkpoint.d.ts +36 -0
  48. package/src/experimental/training/clip.d.ts +9 -0
  49. package/src/experimental/training/dataloader.d.ts +8 -0
  50. package/src/experimental/training/distillation/student-fixture.js +1 -0
  51. package/src/experimental/training/export.d.ts +41 -0
  52. package/src/experimental/training/lora-pipeline.d.ts +186 -0
  53. package/src/experimental/training/lora-pipeline.js +5 -0
  54. package/src/experimental/training/lora.d.ts +20 -0
  55. package/src/experimental/training/lora.js +4 -3
  56. package/src/experimental/training/loss-scaling.d.ts +21 -0
  57. package/src/experimental/training/loss.d.ts +10 -0
  58. package/src/experimental/training/native-qwen-lora.d.ts +106 -0
  59. package/src/experimental/training/native-qwen-lora.js +449 -0
  60. package/src/experimental/training/objectives/base.d.ts +58 -0
  61. package/src/experimental/training/optimizer.d.ts +22 -0
  62. package/src/experimental/training/optimizer.js +1 -0
  63. package/src/experimental/training/runner.d.ts +249 -0
  64. package/src/experimental/training/trainer.d.ts +89 -0
  65. package/src/experimental/training/workloads.d.ts +205 -0
  66. package/src/gpu/kernels/backward/adam.d.ts +1 -0
  67. package/src/gpu/kernels/backward/adam.js +6 -2
  68. package/src/gpu/kernels/backward/adam.wgsl +3 -2
  69. package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
  70. package/src/gpu/kernels/backward/utils.js +18 -4
  71. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  72. package/src/inference/pipelines/text/attention/projections.js +161 -24
  73. package/src/inference/pipelines/text/attention/record.js +67 -9
  74. package/src/inference/pipelines/text/attention/run.js +60 -7
  75. package/src/inference/pipelines/text/attention/types.js +18 -9
  76. package/src/inference/pipelines/text/config.d.ts +5 -0
  77. package/src/inference/pipelines/text/config.js +91 -0
  78. package/src/inference/pipelines/text/ffn/dense.js +205 -2
  79. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  80. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  81. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  82. package/src/inference/pipelines/text/generator-steps.js +3 -1
  83. package/src/inference/pipelines/text/generator.d.ts +5 -1
  84. package/src/inference/pipelines/text/generator.js +196 -3
  85. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  86. package/src/inference/pipelines/text/init.d.ts +2 -0
  87. package/src/inference/pipelines/text/init.js +2 -0
  88. package/src/inference/pipelines/text/layer.js +6 -0
  89. package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
  90. package/src/inference/pipelines/text/linear-attention.js +77 -6
  91. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  92. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  93. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  94. package/src/inference/pipelines/text/logits/index.js +85 -11
  95. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  96. package/src/inference/pipelines/text/lora-types.d.ts +6 -1
  97. package/src/inference/pipelines/text/lora-types.js +5 -0
  98. package/src/inference/pipelines/text/ops.js +33 -0
  99. package/src/inference/pipelines/text/types.d.ts +66 -0
  100. package/src/inference/pipelines/text/weights.d.ts +10 -0
  101. package/src/inference/pipelines/text/weights.js +33 -1
  102. package/src/inference/pipelines/text.d.ts +9 -2
  103. package/src/inference/pipelines/text.js +116 -0
  104. package/src/inference/tokenizers/behavior-flags.js +5 -1
  105. package/src/inference/tokenizers/bundled.js +99 -8
  106. package/src/loader/doppler-loader.d.ts +2 -0
  107. package/src/loader/doppler-loader.js +9 -0
  108. package/src/loader/final-weights-loader.d.ts +8 -0
  109. package/src/loader/final-weights-loader.js +42 -0
  110. package/src/loader/layer-loader.d.ts +6 -0
  111. package/src/loader/layer-loader.js +102 -3
  112. package/src/loader/loader-types.d.ts +15 -2
  113. package/src/models/diffusiongemma.js +1 -1
  114. package/src/models/embeddinggemma.js +1 -1
  115. package/src/models/gemma3.js +1 -1
  116. package/src/models/gemma4.js +1 -1
  117. package/src/models/qwen3.js +1 -1
  118. package/src/tooling/hf-registry-utils.js +1 -1
  119. package/src/tooling/node-browser-command-runner.d.ts +16 -0
  120. package/src/tooling/node-browser-command-runner.js +1 -1
  121. package/src/tooling/node-converter.js +33 -1
  122. package/src/tooling/node-webgpu.js +85 -2
  123. package/src/training-browser.d.ts +19 -0
  124. package/src/training-browser.js +13 -0
  125. package/src/training.d.ts +137 -0
  126. package/src/training.js +219 -0
  127. package/src/version.js +1 -1
  128. package/tests/kernels/browser/kernel-suite.js +9 -1
  129. package/tests/kernels/browser/test-page.js +4 -2
  130. package/tests/kernels/reference/adam.js +4 -2
  131. package/NOTICE +0 -5
@@ -196,16 +196,28 @@ export function resolveMatmulBackwardDxVariant(weight) {
196
196
  const dtype = String(weight?.dtype || 'f32').toLowerCase();
197
197
  if (dtype === 'f16') return 'f16_weight';
198
198
  if (dtype === 'f32') return 'default';
199
+ if (dtype === 'q4k') return 'q4k_weight';
199
200
  throw new Error(`matmul backward dX does not support weight dtype "${dtype}".`);
200
201
  }
201
202
 
203
+ function resolveMatmulBackwardDxDispatch(weight, K, transposeB) {
204
+ const variant = resolveMatmulBackwardDxVariant(weight);
205
+ if (variant === 'q4k_weight' && !transposeB) {
206
+ throw new Error('matmul backward dX requires transposeB=true for row-wise Q4K weights [N,K].');
207
+ }
208
+ return {
209
+ variant,
210
+ numBlocksPerRow: variant === 'q4k_weight' ? Math.ceil(K / 256) : 0,
211
+ };
212
+ }
213
+
202
214
  export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
203
215
  const { alpha = 1.0, transposeB = false, outputBuffer = null } = options;
204
216
  const device = getDevice();
205
217
  const outputSize = M * K * 4;
206
218
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
207
-
208
- const pipeline = await createPipeline('matmul_backward', resolveMatmulBackwardDxVariant(W));
219
+ const dispatchConfig = resolveMatmulBackwardDxDispatch(W, K, transposeB);
220
+ const pipeline = await createPipeline('matmul_backward', dispatchConfig.variant);
209
221
 
210
222
  const uniformBuffer = createUniformBufferWithView(
211
223
  'matmul_backward_uniforms',
@@ -216,6 +228,7 @@ export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
216
228
  view.setUint32(8, K, true);
217
229
  view.setFloat32(12, alpha, true);
218
230
  view.setUint32(16, transposeB ? 1 : 0, true);
231
+ view.setUint32(20, dispatchConfig.numBlocksPerRow, true);
219
232
  },
220
233
  null,
221
234
  device
@@ -250,8 +263,8 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
250
263
  const device = recorder.device;
251
264
  const outputSize = M * K * 4;
252
265
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
253
-
254
- const pipeline = await createPipeline('matmul_backward', resolveMatmulBackwardDxVariant(W));
266
+ const dispatchConfig = resolveMatmulBackwardDxDispatch(W, K, transposeB);
267
+ const pipeline = await createPipeline('matmul_backward', dispatchConfig.variant);
255
268
 
256
269
  const uniformBuffer = createUniformBufferWithView(
257
270
  'matmul_backward_uniforms',
@@ -262,6 +275,7 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
262
275
  view.setUint32(8, K, true);
263
276
  view.setFloat32(12, alpha, true);
264
277
  view.setUint32(16, transposeB ? 1 : 0, true);
278
+ view.setUint32(20, dispatchConfig.numBlocksPerRow, true);
265
279
  },
266
280
  recorder
267
281
  );
@@ -19,6 +19,8 @@ const ATTN_CONFIG_REQUIRED_FIELDS = Object.freeze([
19
19
  'activationDtype',
20
20
  'attnSoftcap',
21
21
  'queryPreAttnScalar',
22
+ 'queryKeyNormType',
23
+ 'queryKeyNormAxis',
22
24
  ]);
23
25
 
24
26
  /**
@@ -9,6 +9,8 @@ import {
9
9
  recordSplitQG,
10
10
  runRMSNorm,
11
11
  recordRMSNorm,
12
+ runLayerNorm,
13
+ recordLayerNorm,
12
14
  canUseRMSNormQK,
13
15
  runRMSNormQK,
14
16
  recordRMSNormQK,
@@ -22,6 +24,8 @@ import {
22
24
  castF32ToF16,
23
25
  recordCastF16ToF32,
24
26
  recordCastF32ToF16,
27
+ runBiasAdd,
28
+ recordBiasAdd,
25
29
  } from '../../../../gpu/kernel-selector.js';
26
30
  import { createTensor } from '../../../../gpu/tensor.js';
27
31
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
@@ -29,7 +33,8 @@ import { QK_K, Q4K_BLOCK_BYTES } from '../../../../config/schema/index.js';
29
33
  import { getKernelPathMatmulPrecision } from '../../../../config/kernel-path-loader.js';
30
34
  import { applyLoRA } from '../lora-apply.js';
31
35
  import { getLoRAModule } from '../lora.js';
32
- import { getQKNormOnesBuffer } from './types.js';
36
+ import { getQKNormOnesBuffer, getQKNormZerosBuffer } from './types.js';
37
+ import { getVectorTensor } from '../weights.js';
33
38
 
34
39
  function getMatmulRunner(recorder) {
35
40
  if (!recorder) {
@@ -52,6 +57,20 @@ function getSplitQGRunner(recorder) {
52
57
  return (qgTensor, options) => recordSplitQG(recorder, qgTensor, options);
53
58
  }
54
59
 
60
+ function getBiasAddRunner(recorder) {
61
+ if (!recorder) {
62
+ return (data, bias, numTokens, dim, options) => runBiasAdd(data, bias, numTokens, dim, options);
63
+ }
64
+ return (data, bias, numTokens, dim, options) => recordBiasAdd(
65
+ recorder,
66
+ data,
67
+ bias,
68
+ numTokens,
69
+ dim,
70
+ options
71
+ );
72
+ }
73
+
55
74
  function getRmsNormRunner(recorder) {
56
75
  if (!recorder) {
57
76
  return (input, weight, eps, options) => runRMSNorm(input, weight, eps, options);
@@ -59,6 +78,20 @@ function getRmsNormRunner(recorder) {
59
78
  return (input, weight, eps, options) => recordRMSNorm(recorder, input, weight, eps, options);
60
79
  }
61
80
 
81
+ function getLayerNormRunner(recorder) {
82
+ if (!recorder) {
83
+ return (input, weight, bias, eps, options) => runLayerNorm(input, weight, bias, eps, options);
84
+ }
85
+ return (input, weight, bias, eps, options) => recordLayerNorm(
86
+ recorder,
87
+ input,
88
+ weight,
89
+ bias,
90
+ eps,
91
+ options
92
+ );
93
+ }
94
+
62
95
  function getRmsNormQKRunner(recorder) {
63
96
  if (!recorder) {
64
97
  return (q, k, qWeight, kWeight, eps, options) => runRMSNormQK(q, k, qWeight, kWeight, eps, options);
@@ -148,13 +181,24 @@ function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemp
148
181
  releaseTemporary(buffer);
149
182
  }
150
183
 
151
- function normBufferMatchesHeadDim(buffer, headDim) {
184
+ export function normBufferMatchesSize(buffer, expectedSize, layerWeight = null) {
185
+ // Logical shape proves how many elements a loaded weight represents; it does
186
+ // not prove that this invocation actually resolved a buffer. Reused-KV
187
+ // layers deliberately leave kNormBuf null, so accepting shape alone would
188
+ // dispatch K normalization with a missing binding.
152
189
  if (!buffer || !Number.isFinite(buffer.size)) {
153
190
  return false;
154
191
  }
192
+ if (Array.isArray(layerWeight?.shape)) {
193
+ const logicalElements = layerWeight.shape.reduce((product, dimension) => product * dimension, 1);
194
+ return logicalElements === expectedSize;
195
+ }
196
+ if (ArrayBuffer.isView(layerWeight)) {
197
+ return layerWeight.length === expectedSize;
198
+ }
155
199
  const elemsF32 = buffer.size / 4;
156
200
  const elemsF16 = buffer.size / 2;
157
- return elemsF32 === headDim || elemsF16 === headDim;
201
+ return elemsF32 === expectedSize || elemsF16 === expectedSize;
158
202
  }
159
203
 
160
204
  function ownsNormBuffer(layerWeight) {
@@ -367,6 +411,32 @@ async function projectSingleQkvTensor({
367
411
  }
368
412
  }
369
413
 
414
+ const biasWeight = layerWeights?.[`${weightKey}Bias`];
415
+ if (biasWeight) {
416
+ const { tensor: biasTensor, owned } = getVectorTensor(
417
+ biasWeight,
418
+ `${role}_bias`,
419
+ outputSize,
420
+ {},
421
+ {}
422
+ );
423
+ try {
424
+ projected = await getBiasAddRunner(recorder)(
425
+ projected,
426
+ biasTensor,
427
+ numTokens,
428
+ outputSize,
429
+ {
430
+ label: `L${layerIdx}.${role}_bias`,
431
+ layerIdx,
432
+ executionPolicies,
433
+ }
434
+ );
435
+ } finally {
436
+ if (owned) releaseTemporary(biasTensor.buffer);
437
+ }
438
+ }
439
+
370
440
  return projected;
371
441
  }
372
442
 
@@ -674,6 +744,30 @@ export async function projectAttentionQKV({
674
744
  rmsNormEps: fusedNormEps,
675
745
  rmsNormOffset: fusedNormOffset,
676
746
  });
747
+ if (layerWeights.qkvProjBias) {
748
+ const { tensor: qkvBiasTensor, owned: qkvBiasOwned } = getVectorTensor(
749
+ layerWeights.qkvProjBias,
750
+ `L${layerIdx}.qkv_proj_bias`,
751
+ qkvSizeTotal,
752
+ {},
753
+ {}
754
+ );
755
+ try {
756
+ qkvTensor = await getBiasAddRunner(recorder)(
757
+ qkvTensor,
758
+ qkvBiasTensor,
759
+ numTokens,
760
+ qkvSizeTotal,
761
+ {
762
+ label: `L${layerIdx}.qkv_proj_bias`,
763
+ layerIdx,
764
+ executionPolicies,
765
+ }
766
+ );
767
+ } finally {
768
+ if (qkvBiasOwned) releaseTemporary(qkvBiasTensor.buffer);
769
+ }
770
+ }
677
771
  const qkNormRoPEOptions = qkNormRoPEFusion
678
772
  ? { ...qkNormRoPEFusion, headDim }
679
773
  : null;
@@ -691,8 +785,8 @@ export async function projectAttentionQKV({
691
785
  if (canFuseSplitQKNormAndRoPE) {
692
786
  qNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
693
787
  kNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
694
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
695
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
788
+ const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
789
+ const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
696
790
  if (qNormApplies && kNormApplies) {
697
791
  const fused = await runSplitQKVRMSNormRoPEQKForMode(
698
792
  qkvTensor,
@@ -779,8 +873,8 @@ export async function projectAttentionQKV({
779
873
  if (canFuseSplitAndQKNorm) {
780
874
  qNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
781
875
  kNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
782
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
783
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
876
+ const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
877
+ const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
784
878
  if (qNormApplies && kNormApplies) {
785
879
  const fused = await runSplitQKVRMSNormQKForMode(
786
880
  qkvTensor,
@@ -1099,6 +1193,8 @@ export async function applyAttentionQKNorm({
1099
1193
  numKVHeads,
1100
1194
  headDim,
1101
1195
  rmsNormWeightOffset = false,
1196
+ normalizationType = 'rmsnorm',
1197
+ normalizationAxis = 'head',
1102
1198
  releaseTemporary,
1103
1199
  onQNormApplied = null,
1104
1200
  onKNormApplied = null,
@@ -1108,6 +1204,17 @@ export async function applyAttentionQKNorm({
1108
1204
  }) {
1109
1205
  const runRmsNormForMode = getRmsNormRunner(recorder);
1110
1206
  const runRmsNormQKForMode = getRmsNormQKRunner(recorder);
1207
+ const runLayerNormForMode = getLayerNormRunner(recorder);
1208
+ if (normalizationType !== 'rmsnorm' && normalizationType !== 'layernorm') {
1209
+ throw new Error(`Unsupported Q/K normalization type "${normalizationType}".`);
1210
+ }
1211
+ if (normalizationAxis !== 'head' && normalizationAxis !== 'projection') {
1212
+ throw new Error(`Unsupported Q/K normalization axis "${normalizationAxis}".`);
1213
+ }
1214
+ const qNormSize = normalizationAxis === 'projection' ? numHeads * headDim : headDim;
1215
+ const kNormSize = normalizationAxis === 'projection' ? numKVHeads * headDim : headDim;
1216
+ const qNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numHeads;
1217
+ const kNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numKVHeads;
1111
1218
  let nextQ = qTensor;
1112
1219
  let nextK = kTensor;
1113
1220
  let qNormBuf = null;
@@ -1121,18 +1228,32 @@ export async function applyAttentionQKNorm({
1121
1228
  if (wantsQNorm) {
1122
1229
  qNormBuf = layerWeights.qNorm && getNormWeightBuffer
1123
1230
  ? getNormWeightBuffer(layerWeights.qNorm, 'q_norm')
1124
- : getQKNormOnesBuffer(headDim);
1231
+ : getQKNormOnesBuffer(qNormSize);
1125
1232
  }
1126
1233
  if (wantsKNorm) {
1127
1234
  kNormBuf = layerWeights.kNorm && getNormWeightBuffer
1128
1235
  ? getNormWeightBuffer(layerWeights.kNorm, 'k_norm')
1129
- : getQKNormOnesBuffer(headDim);
1236
+ : getQKNormOnesBuffer(kNormSize);
1130
1237
  }
1131
1238
 
1132
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
1133
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
1239
+ const qNormApplies = normBufferMatchesSize(qNormBuf, qNormSize, layerWeights.qNorm);
1240
+ const kNormApplies = normBufferMatchesSize(kNormBuf, kNormSize, layerWeights.kNorm);
1241
+ if (wantsQNorm && !qNormApplies) {
1242
+ throw new Error(
1243
+ `Q normalization weight has ${String(qNormBuf?.size ?? 'unknown')} bytes; ` +
1244
+ `expected ${qNormSize} f16 or f32 elements.`
1245
+ );
1246
+ }
1247
+ if (wantsKNorm && !kNormApplies) {
1248
+ throw new Error(
1249
+ `K normalization weight has ${String(kNormBuf?.size ?? 'unknown')} bytes; ` +
1250
+ `expected ${kNormSize} f16 or f32 elements.`
1251
+ );
1252
+ }
1134
1253
  if (
1135
- qNormApplies
1254
+ normalizationType === 'rmsnorm'
1255
+ && normalizationAxis === 'head'
1256
+ && qNormApplies
1136
1257
  && kNormApplies
1137
1258
  && canUseRMSNormQK(nextQ, nextK, { skipKNorm })
1138
1259
  ) {
@@ -1159,12 +1280,20 @@ export async function applyAttentionQKNorm({
1159
1280
  }
1160
1281
 
1161
1282
  if (qNormApplies) {
1162
- const qNormedTensor = await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
1163
- batchSize: numTokens * numHeads,
1164
- hiddenSize: headDim,
1165
- rmsNormWeightOffset,
1166
- label: 'q_norm',
1167
- });
1283
+ const qNormedTensor = normalizationType === 'layernorm'
1284
+ ? await runLayerNormForMode(
1285
+ nextQ,
1286
+ qNormBuf,
1287
+ getQKNormZerosBuffer(qNormSize),
1288
+ rmsNormEps,
1289
+ { batchSize: qNormBatchSize, hiddenSize: qNormSize, label: 'q_norm' }
1290
+ )
1291
+ : await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
1292
+ batchSize: qNormBatchSize,
1293
+ hiddenSize: qNormSize,
1294
+ rmsNormWeightOffset,
1295
+ label: 'q_norm',
1296
+ });
1168
1297
  releaseTemporary(nextQ.buffer);
1169
1298
  nextQ = qNormedTensor;
1170
1299
  if (onQNormApplied) {
@@ -1173,12 +1302,20 @@ export async function applyAttentionQKNorm({
1173
1302
  }
1174
1303
 
1175
1304
  if (kNormApplies) {
1176
- const kNormedTensor = await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
1177
- batchSize: numTokens * numKVHeads,
1178
- hiddenSize: headDim,
1179
- rmsNormWeightOffset,
1180
- label: 'k_norm',
1181
- });
1305
+ const kNormedTensor = normalizationType === 'layernorm'
1306
+ ? await runLayerNormForMode(
1307
+ nextK,
1308
+ kNormBuf,
1309
+ getQKNormZerosBuffer(kNormSize),
1310
+ rmsNormEps,
1311
+ { batchSize: kNormBatchSize, hiddenSize: kNormSize, label: 'k_norm' }
1312
+ )
1313
+ : await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
1314
+ batchSize: kNormBatchSize,
1315
+ hiddenSize: kNormSize,
1316
+ rmsNormWeightOffset,
1317
+ label: 'k_norm',
1318
+ });
1182
1319
  if (!retainKInput) {
1183
1320
  releaseTemporary(nextK.buffer);
1184
1321
  }
@@ -6,6 +6,8 @@ import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
6
6
  import {
7
7
  recordMatmul,
8
8
  recordRMSNorm,
9
+ recordLayerNorm,
10
+ recordBiasAdd,
9
11
  recordRoPE,
10
12
  canUseRoPEQK,
11
13
  recordRoPEQK,
@@ -63,6 +65,7 @@ import {
63
65
  } from './precision-contract.js';
64
66
  import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
65
67
  import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
68
+ import { getVectorTensor } from '../weights.js';
66
69
 
67
70
  const ATTENTION_DTYPE_LOGGED = new Set();
68
71
 
@@ -284,6 +287,7 @@ export async function recordLayerAttentionGPU(
284
287
  sharedKVSourceLayerIdx != null
285
288
  );
286
289
  const canFuseInputNormProjRec = rmsNormFusionFlagRec
290
+ && config.normalizationType === 'rmsnorm'
287
291
  && canSelectFusedRmsNormProjectionRec
288
292
  && !skipInputNorm
289
293
  && layerWeights.inputNorm
@@ -319,12 +323,36 @@ export async function recordLayerAttentionGPU(
319
323
  // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
320
324
  } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
321
325
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
322
- normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
323
- batchSize: numTokens,
324
- hiddenSize,
325
- rmsNormWeightOffset: config.rmsNormWeightOffset,
326
- label: 'input_norm',
327
- });
326
+ let normBiasTensor = null;
327
+ let normBiasOwned = false;
328
+ if (config.normalizationType === 'layernorm') {
329
+ if (!layerWeights.inputNormBias) {
330
+ throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
331
+ }
332
+ ({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
333
+ layerWeights.inputNormBias,
334
+ `L${layerIdx}.input_norm_bias`,
335
+ hiddenSize,
336
+ {},
337
+ debugFlags
338
+ ));
339
+ normed = await recordLayerNorm(
340
+ recorder,
341
+ attentionInput,
342
+ normWeightBuf,
343
+ normBiasTensor.buffer,
344
+ rmsNormEps,
345
+ { batchSize: numTokens, hiddenSize, label: 'input_norm' }
346
+ );
347
+ } else {
348
+ normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
349
+ batchSize: numTokens,
350
+ hiddenSize,
351
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
352
+ label: 'input_norm',
353
+ });
354
+ }
355
+ if (normBiasOwned && normBiasTensor) releaseOrTrack(recorder, normBiasTensor.buffer);
328
356
  if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
329
357
  await runProbes('post_input_norm', normed.buffer, {
330
358
  layerIdx, numTokens, hiddenSize,
@@ -438,7 +466,10 @@ export async function recordLayerAttentionGPU(
438
466
  fusedNormEps: fusedNormEpsRec,
439
467
  fusedNormOffset: fusedNormOffsetRec,
440
468
  qkNormFusion: {
441
- enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
469
+ enabled: qkNormFusionFlag
470
+ && qkNormState.wantsQKNorm
471
+ && config.queryKeyNormType === 'rmsnorm'
472
+ && config.queryKeyNormAxis === 'head',
442
473
  getNormWeightBuffer,
443
474
  rmsNormEps,
444
475
  rmsNormWeightOffset: config.rmsNormWeightOffset,
@@ -449,6 +480,8 @@ export async function recordLayerAttentionGPU(
449
480
  qkNormRoPEFusion: {
450
481
  enabled: qkNormRoPEFusionFlag
451
482
  && qkNormState.wantsQKNorm
483
+ && config.queryKeyNormType === 'rmsnorm'
484
+ && config.queryKeyNormAxis === 'head'
452
485
  && !disableRoPE
453
486
  && !!state.ropeFreqsCos
454
487
  && !!state.ropeFreqsSin,
@@ -520,8 +553,8 @@ export async function recordLayerAttentionGPU(
520
553
  );
521
554
  }
522
555
 
523
- // Optional per-head Q/K normalization.
524
- // Some models use RMSNorm with (1+weight) offset formula, controlled by rmsNormWeightOffset.
556
+ // Optional Q/K normalization. The manifest owns both its numerical family
557
+ // and whether it applies per head or across the complete projection.
525
558
  if (qkNormState.wantsQKNorm && !qkNormApplied) {
526
559
  ({ qTensor, kTensor } = await applyAttentionQKNorm({
527
560
  recorder,
@@ -535,6 +568,8 @@ export async function recordLayerAttentionGPU(
535
568
  numKVHeads,
536
569
  headDim,
537
570
  rmsNormWeightOffset: config.rmsNormWeightOffset,
571
+ normalizationType: config.queryKeyNormType,
572
+ normalizationAxis: config.queryKeyNormAxis,
538
573
  releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
539
574
  skipKNorm: qkNormState.skipKNorm,
540
575
  retainKInput: valueAliasesKey,
@@ -1113,6 +1148,29 @@ export async function recordLayerAttentionGPU(
1113
1148
  }
1114
1149
  }
1115
1150
 
1151
+ if (layerWeights.oProjBias) {
1152
+ const { tensor: oProjBiasTensor, owned } = getVectorTensor(
1153
+ layerWeights.oProjBias,
1154
+ `L${layerIdx}.o_proj_bias`,
1155
+ hiddenSize,
1156
+ {},
1157
+ debugFlags
1158
+ );
1159
+ output = await recordBiasAdd(
1160
+ recorder,
1161
+ output,
1162
+ oProjBiasTensor,
1163
+ numTokens,
1164
+ hiddenSize,
1165
+ {
1166
+ label: `L${layerIdx}.o_proj_bias`,
1167
+ layerIdx,
1168
+ executionPolicies: state.executionPolicies ?? null,
1169
+ }
1170
+ );
1171
+ if (owned) releaseOrTrack(recorder, oProjBiasTensor.buffer);
1172
+ }
1173
+
1116
1174
  finalOutput = output;
1117
1175
  if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
1118
1176
  enqueueRecordedTensorHealth(
@@ -7,6 +7,8 @@ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js'
7
7
  import {
8
8
  runMatmul,
9
9
  runRMSNorm,
10
+ runLayerNorm,
11
+ runBiasAdd,
10
12
  runRoPE,
11
13
  canUseRoPEQK,
12
14
  runRoPEQK,
@@ -67,6 +69,7 @@ import {
67
69
  } from './precision-contract.js';
68
70
  import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
69
71
  import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
72
+ import { getVectorTensor } from '../weights.js';
70
73
 
71
74
  const ATTENTION_DTYPE_LOGGED = new Set();
72
75
 
@@ -244,6 +247,7 @@ export async function runLayerAttentionGPU(
244
247
  const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
245
248
  const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
246
249
  const canFuseInputNormIntoProjections = rmsNormFusionFlag
250
+ && config.normalizationType === 'rmsnorm'
247
251
  && hasInputNormWeight
248
252
  && canSelectFusedRmsNormProjection
249
253
  && !skipInputNorm
@@ -283,18 +287,41 @@ export async function runLayerAttentionGPU(
283
287
  // norm internally via the fused kernel variant.
284
288
  } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
285
289
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
290
+ let normBiasTensor = null;
291
+ let normBiasOwned = false;
286
292
  try {
287
293
  // Debug: norm weights for configured layers
288
294
  if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'norm_weights', debugFlags) && debugCheckBuffer) {
289
295
  await debugCheckBuffer(normWeightBuf, `L${layerIdx} input norm weights (GPU)`, 1, hiddenSize);
290
296
  }
291
297
 
292
- normed = await runRMSNorm(attentionInput, normWeightBuf, rmsNormEps, {
293
- batchSize: numTokens,
294
- hiddenSize,
295
- rmsNormWeightOffset: config.rmsNormWeightOffset,
296
- });
298
+ if (config.normalizationType === 'layernorm') {
299
+ if (!layerWeights.inputNormBias) {
300
+ throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
301
+ }
302
+ ({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
303
+ layerWeights.inputNormBias,
304
+ `L${layerIdx}.input_norm_bias`,
305
+ hiddenSize,
306
+ {},
307
+ debugFlags
308
+ ));
309
+ normed = await runLayerNorm(
310
+ attentionInput,
311
+ normWeightBuf,
312
+ normBiasTensor.buffer,
313
+ rmsNormEps,
314
+ { batchSize: numTokens, hiddenSize }
315
+ );
316
+ } else {
317
+ normed = await runRMSNorm(attentionInput, normWeightBuf, rmsNormEps, {
318
+ batchSize: numTokens,
319
+ hiddenSize,
320
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
321
+ });
322
+ }
297
323
  } finally {
324
+ if (normBiasOwned && normBiasTensor) releaseBuffer(normBiasTensor.buffer);
298
325
  if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseBuffer(normWeightBuf);
299
326
  }
300
327
 
@@ -408,7 +435,10 @@ export async function runLayerAttentionGPU(
408
435
  fusedNormEps,
409
436
  fusedNormOffset,
410
437
  qkNormFusion: {
411
- enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
438
+ enabled: qkNormFusionFlag
439
+ && qkNormState.wantsQKNorm
440
+ && config.queryKeyNormType === 'rmsnorm'
441
+ && config.queryKeyNormAxis === 'head',
412
442
  getNormWeightBuffer,
413
443
  rmsNormEps,
414
444
  rmsNormWeightOffset: config.rmsNormWeightOffset,
@@ -419,6 +449,8 @@ export async function runLayerAttentionGPU(
419
449
  qkNormRoPEFusion: {
420
450
  enabled: qkNormRoPEFusionFlag
421
451
  && qkNormState.wantsQKNorm
452
+ && config.queryKeyNormType === 'rmsnorm'
453
+ && config.queryKeyNormAxis === 'head'
422
454
  && !disableRoPE
423
455
  && !!state.ropeFreqsCos
424
456
  && !!state.ropeFreqsSin,
@@ -508,7 +540,7 @@ export async function runLayerAttentionGPU(
508
540
  await debugCheckBuffer(vTensor.buffer, `L${layerIdx} V after proj (GPU)`, numTokens, numKVHeads * headDim);
509
541
  }
510
542
 
511
- // Optional per-head Q/K normalization
543
+ // Optional Q/K normalization; the manifest owns the axis and norm family.
512
544
  if (isKernelDebugEnabled(layerIdx)) {
513
545
  logKernelStep(
514
546
  'qk_norm',
@@ -533,6 +565,8 @@ export async function runLayerAttentionGPU(
533
565
  numKVHeads,
534
566
  headDim,
535
567
  rmsNormWeightOffset: config.rmsNormWeightOffset,
568
+ normalizationType: config.queryKeyNormType,
569
+ normalizationAxis: config.queryKeyNormAxis,
536
570
  releaseTemporary: (buffer) => releaseBuffer(buffer),
537
571
  skipKNorm: qkNormState.skipKNorm,
538
572
  onQNormApplied: isKernelDebugEnabled(layerIdx)
@@ -1229,6 +1263,25 @@ export async function runLayerAttentionGPU(
1229
1263
  }
1230
1264
  }
1231
1265
 
1266
+ if (layerWeights.oProjBias) {
1267
+ const { tensor: oProjBiasTensor, owned } = getVectorTensor(
1268
+ layerWeights.oProjBias,
1269
+ `L${layerIdx}.o_proj_bias`,
1270
+ hiddenSize,
1271
+ {},
1272
+ debugFlags
1273
+ );
1274
+ try {
1275
+ output = await runBiasAdd(output, oProjBiasTensor, numTokens, hiddenSize, {
1276
+ label: `L${layerIdx}.o_proj_bias`,
1277
+ layerIdx,
1278
+ executionPolicies: state.executionPolicies ?? null,
1279
+ });
1280
+ } finally {
1281
+ if (owned) releaseBuffer(oProjBiasTensor.buffer);
1282
+ }
1283
+ }
1284
+
1232
1285
  if (oProjInputTemp) {
1233
1286
  releaseBuffer(oProjInputTemp.buffer);
1234
1287
  }
@@ -45,28 +45,37 @@ export function releaseOrTrack(recorder, buffer) {
45
45
 
46
46
 
47
47
  const qkNormOnesCache = new WeakMap();
48
+ const qkNormZerosCache = new WeakMap();
48
49
 
49
-
50
- export function getQKNormOnesBuffer(headDim) {
50
+ function getCachedQKNormConstantBuffer(cache, size, value, label) {
51
51
  const device = getDevice();
52
52
  if (!device) {
53
53
  throw new Error('No GPU device available for Q/K norm buffer');
54
54
  }
55
- let perDeviceCache = qkNormOnesCache.get(device);
55
+ let perDeviceCache = cache.get(device);
56
56
  if (!perDeviceCache) {
57
57
  perDeviceCache = new Map();
58
- qkNormOnesCache.set(device, perDeviceCache);
58
+ cache.set(device, perDeviceCache);
59
59
  }
60
- const cached = perDeviceCache.get(headDim);
60
+ const cached = perDeviceCache.get(size);
61
61
  if (cached) return cached;
62
- const data = new Float32Array(headDim);
63
- data.fill(1);
62
+ const data = new Float32Array(size);
63
+ data.fill(value);
64
64
  const buffer = device.createBuffer({
65
- label: `qk_norm_ones_${headDim}`,
65
+ label: `${label}_${size}`,
66
66
  size: data.byteLength,
67
67
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
68
68
  });
69
69
  device.queue.writeBuffer(buffer, 0, data);
70
- perDeviceCache.set(headDim, buffer);
70
+ perDeviceCache.set(size, buffer);
71
71
  return buffer;
72
72
  }
73
+
74
+
75
+ export function getQKNormOnesBuffer(headDim) {
76
+ return getCachedQKNormConstantBuffer(qkNormOnesCache, headDim, 1, 'qk_norm_ones');
77
+ }
78
+
79
+ export function getQKNormZerosBuffer(size) {
80
+ return getCachedQKNormConstantBuffer(qkNormZerosCache, size, 0, 'qk_norm_zeros');
81
+ }