doppler-gpu 0.4.9 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/README.md +89 -12
  3. package/models/catalog.json +612 -7
  4. package/package.json +30 -1
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +126 -4
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +64 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/kernels/registry.json +52 -9
  18. package/src/config/merge.js +24 -0
  19. package/src/config/required-inference-fields-contract-check.js +2 -0
  20. package/src/config/schema/converter.schema.d.ts +7 -0
  21. package/src/config/schema/converter.schema.js +15 -0
  22. package/src/config/schema/inference.schema.d.ts +8 -0
  23. package/src/config/schema/manifest.schema.d.ts +32 -1
  24. package/src/config/schema/manifest.schema.js +26 -0
  25. package/src/converter/conversion-plan.js +15 -1
  26. package/src/converter/core.js +5 -4
  27. package/src/converter/source-tensor-rules.d.ts +50 -0
  28. package/src/converter/source-tensor-rules.js +180 -0
  29. package/src/converter/source-tokenizer.d.ts +45 -0
  30. package/src/converter/source-tokenizer.js +155 -0
  31. package/src/experimental/adapters/lora-loader.js +15 -0
  32. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  33. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  34. package/src/gpu/device.js +100 -32
  35. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  36. package/src/inference/pipelines/text/attention/projections.js +161 -24
  37. package/src/inference/pipelines/text/attention/record.js +67 -9
  38. package/src/inference/pipelines/text/attention/run.js +60 -7
  39. package/src/inference/pipelines/text/attention/types.js +18 -9
  40. package/src/inference/pipelines/text/config.d.ts +5 -0
  41. package/src/inference/pipelines/text/config.js +91 -0
  42. package/src/inference/pipelines/text/ffn/dense.js +189 -2
  43. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  44. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  45. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  46. package/src/inference/pipelines/text/generator-steps.js +3 -1
  47. package/src/inference/pipelines/text/generator.js +90 -3
  48. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  49. package/src/inference/pipelines/text/init.d.ts +2 -0
  50. package/src/inference/pipelines/text/init.js +2 -0
  51. package/src/inference/pipelines/text/layer.js +6 -0
  52. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  53. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  54. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  55. package/src/inference/pipelines/text/logits/index.js +85 -11
  56. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  57. package/src/inference/pipelines/text/ops.js +32 -0
  58. package/src/inference/pipelines/text/types.d.ts +38 -0
  59. package/src/inference/pipelines/text/weights.d.ts +10 -0
  60. package/src/inference/pipelines/text/weights.js +33 -1
  61. package/src/inference/pipelines/text.d.ts +4 -2
  62. package/src/inference/pipelines/text.js +112 -0
  63. package/src/inference/tokenizers/behavior-flags.js +5 -1
  64. package/src/inference/tokenizers/bundled.js +99 -8
  65. package/src/loader/doppler-loader.d.ts +2 -0
  66. package/src/loader/doppler-loader.js +9 -0
  67. package/src/loader/final-weights-loader.d.ts +8 -0
  68. package/src/loader/final-weights-loader.js +42 -0
  69. package/src/loader/layer-loader.d.ts +6 -0
  70. package/src/loader/layer-loader.js +102 -3
  71. package/src/loader/loader-types.d.ts +15 -2
  72. package/src/tooling/node-converter.js +33 -1
  73. package/src/tooling/node-webgpu.js +85 -2
  74. package/src/tooling/runtime-optimization.d.ts +109 -0
  75. package/src/tooling/runtime-optimization.js +872 -0
  76. package/src/tooling-exports.shared.d.ts +19 -0
  77. package/src/tooling-exports.shared.js +11 -0
  78. package/src/version.js +1 -1
@@ -9,6 +9,8 @@ import {
9
9
  recordSplitQG,
10
10
  runRMSNorm,
11
11
  recordRMSNorm,
12
+ runLayerNorm,
13
+ recordLayerNorm,
12
14
  canUseRMSNormQK,
13
15
  runRMSNormQK,
14
16
  recordRMSNormQK,
@@ -22,6 +24,8 @@ import {
22
24
  castF32ToF16,
23
25
  recordCastF16ToF32,
24
26
  recordCastF32ToF16,
27
+ runBiasAdd,
28
+ recordBiasAdd,
25
29
  } from '../../../../gpu/kernel-selector.js';
26
30
  import { createTensor } from '../../../../gpu/tensor.js';
27
31
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
@@ -29,7 +33,8 @@ import { QK_K, Q4K_BLOCK_BYTES } from '../../../../config/schema/index.js';
29
33
  import { getKernelPathMatmulPrecision } from '../../../../config/kernel-path-loader.js';
30
34
  import { applyLoRA } from '../lora-apply.js';
31
35
  import { getLoRAModule } from '../lora.js';
32
- import { getQKNormOnesBuffer } from './types.js';
36
+ import { getQKNormOnesBuffer, getQKNormZerosBuffer } from './types.js';
37
+ import { getVectorTensor } from '../weights.js';
33
38
 
34
39
  function getMatmulRunner(recorder) {
35
40
  if (!recorder) {
@@ -52,6 +57,20 @@ function getSplitQGRunner(recorder) {
52
57
  return (qgTensor, options) => recordSplitQG(recorder, qgTensor, options);
53
58
  }
54
59
 
60
+ function getBiasAddRunner(recorder) {
61
+ if (!recorder) {
62
+ return (data, bias, numTokens, dim, options) => runBiasAdd(data, bias, numTokens, dim, options);
63
+ }
64
+ return (data, bias, numTokens, dim, options) => recordBiasAdd(
65
+ recorder,
66
+ data,
67
+ bias,
68
+ numTokens,
69
+ dim,
70
+ options
71
+ );
72
+ }
73
+
55
74
  function getRmsNormRunner(recorder) {
56
75
  if (!recorder) {
57
76
  return (input, weight, eps, options) => runRMSNorm(input, weight, eps, options);
@@ -59,6 +78,20 @@ function getRmsNormRunner(recorder) {
59
78
  return (input, weight, eps, options) => recordRMSNorm(recorder, input, weight, eps, options);
60
79
  }
61
80
 
81
+ function getLayerNormRunner(recorder) {
82
+ if (!recorder) {
83
+ return (input, weight, bias, eps, options) => runLayerNorm(input, weight, bias, eps, options);
84
+ }
85
+ return (input, weight, bias, eps, options) => recordLayerNorm(
86
+ recorder,
87
+ input,
88
+ weight,
89
+ bias,
90
+ eps,
91
+ options
92
+ );
93
+ }
94
+
62
95
  function getRmsNormQKRunner(recorder) {
63
96
  if (!recorder) {
64
97
  return (q, k, qWeight, kWeight, eps, options) => runRMSNormQK(q, k, qWeight, kWeight, eps, options);
@@ -148,13 +181,24 @@ function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemp
148
181
  releaseTemporary(buffer);
149
182
  }
150
183
 
151
- function normBufferMatchesHeadDim(buffer, headDim) {
184
+ export function normBufferMatchesSize(buffer, expectedSize, layerWeight = null) {
185
+ // Logical shape proves how many elements a loaded weight represents; it does
186
+ // not prove that this invocation actually resolved a buffer. Reused-KV
187
+ // layers deliberately leave kNormBuf null, so accepting shape alone would
188
+ // dispatch K normalization with a missing binding.
152
189
  if (!buffer || !Number.isFinite(buffer.size)) {
153
190
  return false;
154
191
  }
192
+ if (Array.isArray(layerWeight?.shape)) {
193
+ const logicalElements = layerWeight.shape.reduce((product, dimension) => product * dimension, 1);
194
+ return logicalElements === expectedSize;
195
+ }
196
+ if (ArrayBuffer.isView(layerWeight)) {
197
+ return layerWeight.length === expectedSize;
198
+ }
155
199
  const elemsF32 = buffer.size / 4;
156
200
  const elemsF16 = buffer.size / 2;
157
- return elemsF32 === headDim || elemsF16 === headDim;
201
+ return elemsF32 === expectedSize || elemsF16 === expectedSize;
158
202
  }
159
203
 
160
204
  function ownsNormBuffer(layerWeight) {
@@ -367,6 +411,32 @@ async function projectSingleQkvTensor({
367
411
  }
368
412
  }
369
413
 
414
+ const biasWeight = layerWeights?.[`${weightKey}Bias`];
415
+ if (biasWeight) {
416
+ const { tensor: biasTensor, owned } = getVectorTensor(
417
+ biasWeight,
418
+ `${role}_bias`,
419
+ outputSize,
420
+ {},
421
+ {}
422
+ );
423
+ try {
424
+ projected = await getBiasAddRunner(recorder)(
425
+ projected,
426
+ biasTensor,
427
+ numTokens,
428
+ outputSize,
429
+ {
430
+ label: `L${layerIdx}.${role}_bias`,
431
+ layerIdx,
432
+ executionPolicies,
433
+ }
434
+ );
435
+ } finally {
436
+ if (owned) releaseTemporary(biasTensor.buffer);
437
+ }
438
+ }
439
+
370
440
  return projected;
371
441
  }
372
442
 
@@ -674,6 +744,30 @@ export async function projectAttentionQKV({
674
744
  rmsNormEps: fusedNormEps,
675
745
  rmsNormOffset: fusedNormOffset,
676
746
  });
747
+ if (layerWeights.qkvProjBias) {
748
+ const { tensor: qkvBiasTensor, owned: qkvBiasOwned } = getVectorTensor(
749
+ layerWeights.qkvProjBias,
750
+ `L${layerIdx}.qkv_proj_bias`,
751
+ qkvSizeTotal,
752
+ {},
753
+ {}
754
+ );
755
+ try {
756
+ qkvTensor = await getBiasAddRunner(recorder)(
757
+ qkvTensor,
758
+ qkvBiasTensor,
759
+ numTokens,
760
+ qkvSizeTotal,
761
+ {
762
+ label: `L${layerIdx}.qkv_proj_bias`,
763
+ layerIdx,
764
+ executionPolicies,
765
+ }
766
+ );
767
+ } finally {
768
+ if (qkvBiasOwned) releaseTemporary(qkvBiasTensor.buffer);
769
+ }
770
+ }
677
771
  const qkNormRoPEOptions = qkNormRoPEFusion
678
772
  ? { ...qkNormRoPEFusion, headDim }
679
773
  : null;
@@ -691,8 +785,8 @@ export async function projectAttentionQKV({
691
785
  if (canFuseSplitQKNormAndRoPE) {
692
786
  qNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
693
787
  kNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
694
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
695
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
788
+ const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
789
+ const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
696
790
  if (qNormApplies && kNormApplies) {
697
791
  const fused = await runSplitQKVRMSNormRoPEQKForMode(
698
792
  qkvTensor,
@@ -779,8 +873,8 @@ export async function projectAttentionQKV({
779
873
  if (canFuseSplitAndQKNorm) {
780
874
  qNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
781
875
  kNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
782
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
783
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
876
+ const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
877
+ const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
784
878
  if (qNormApplies && kNormApplies) {
785
879
  const fused = await runSplitQKVRMSNormQKForMode(
786
880
  qkvTensor,
@@ -1099,6 +1193,8 @@ export async function applyAttentionQKNorm({
1099
1193
  numKVHeads,
1100
1194
  headDim,
1101
1195
  rmsNormWeightOffset = false,
1196
+ normalizationType = 'rmsnorm',
1197
+ normalizationAxis = 'head',
1102
1198
  releaseTemporary,
1103
1199
  onQNormApplied = null,
1104
1200
  onKNormApplied = null,
@@ -1108,6 +1204,17 @@ export async function applyAttentionQKNorm({
1108
1204
  }) {
1109
1205
  const runRmsNormForMode = getRmsNormRunner(recorder);
1110
1206
  const runRmsNormQKForMode = getRmsNormQKRunner(recorder);
1207
+ const runLayerNormForMode = getLayerNormRunner(recorder);
1208
+ if (normalizationType !== 'rmsnorm' && normalizationType !== 'layernorm') {
1209
+ throw new Error(`Unsupported Q/K normalization type "${normalizationType}".`);
1210
+ }
1211
+ if (normalizationAxis !== 'head' && normalizationAxis !== 'projection') {
1212
+ throw new Error(`Unsupported Q/K normalization axis "${normalizationAxis}".`);
1213
+ }
1214
+ const qNormSize = normalizationAxis === 'projection' ? numHeads * headDim : headDim;
1215
+ const kNormSize = normalizationAxis === 'projection' ? numKVHeads * headDim : headDim;
1216
+ const qNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numHeads;
1217
+ const kNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numKVHeads;
1111
1218
  let nextQ = qTensor;
1112
1219
  let nextK = kTensor;
1113
1220
  let qNormBuf = null;
@@ -1121,18 +1228,32 @@ export async function applyAttentionQKNorm({
1121
1228
  if (wantsQNorm) {
1122
1229
  qNormBuf = layerWeights.qNorm && getNormWeightBuffer
1123
1230
  ? getNormWeightBuffer(layerWeights.qNorm, 'q_norm')
1124
- : getQKNormOnesBuffer(headDim);
1231
+ : getQKNormOnesBuffer(qNormSize);
1125
1232
  }
1126
1233
  if (wantsKNorm) {
1127
1234
  kNormBuf = layerWeights.kNorm && getNormWeightBuffer
1128
1235
  ? getNormWeightBuffer(layerWeights.kNorm, 'k_norm')
1129
- : getQKNormOnesBuffer(headDim);
1236
+ : getQKNormOnesBuffer(kNormSize);
1130
1237
  }
1131
1238
 
1132
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
1133
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
1239
+ const qNormApplies = normBufferMatchesSize(qNormBuf, qNormSize, layerWeights.qNorm);
1240
+ const kNormApplies = normBufferMatchesSize(kNormBuf, kNormSize, layerWeights.kNorm);
1241
+ if (wantsQNorm && !qNormApplies) {
1242
+ throw new Error(
1243
+ `Q normalization weight has ${String(qNormBuf?.size ?? 'unknown')} bytes; ` +
1244
+ `expected ${qNormSize} f16 or f32 elements.`
1245
+ );
1246
+ }
1247
+ if (wantsKNorm && !kNormApplies) {
1248
+ throw new Error(
1249
+ `K normalization weight has ${String(kNormBuf?.size ?? 'unknown')} bytes; ` +
1250
+ `expected ${kNormSize} f16 or f32 elements.`
1251
+ );
1252
+ }
1134
1253
  if (
1135
- qNormApplies
1254
+ normalizationType === 'rmsnorm'
1255
+ && normalizationAxis === 'head'
1256
+ && qNormApplies
1136
1257
  && kNormApplies
1137
1258
  && canUseRMSNormQK(nextQ, nextK, { skipKNorm })
1138
1259
  ) {
@@ -1159,12 +1280,20 @@ export async function applyAttentionQKNorm({
1159
1280
  }
1160
1281
 
1161
1282
  if (qNormApplies) {
1162
- const qNormedTensor = await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
1163
- batchSize: numTokens * numHeads,
1164
- hiddenSize: headDim,
1165
- rmsNormWeightOffset,
1166
- label: 'q_norm',
1167
- });
1283
+ const qNormedTensor = normalizationType === 'layernorm'
1284
+ ? await runLayerNormForMode(
1285
+ nextQ,
1286
+ qNormBuf,
1287
+ getQKNormZerosBuffer(qNormSize),
1288
+ rmsNormEps,
1289
+ { batchSize: qNormBatchSize, hiddenSize: qNormSize, label: 'q_norm' }
1290
+ )
1291
+ : await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
1292
+ batchSize: qNormBatchSize,
1293
+ hiddenSize: qNormSize,
1294
+ rmsNormWeightOffset,
1295
+ label: 'q_norm',
1296
+ });
1168
1297
  releaseTemporary(nextQ.buffer);
1169
1298
  nextQ = qNormedTensor;
1170
1299
  if (onQNormApplied) {
@@ -1173,12 +1302,20 @@ export async function applyAttentionQKNorm({
1173
1302
  }
1174
1303
 
1175
1304
  if (kNormApplies) {
1176
- const kNormedTensor = await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
1177
- batchSize: numTokens * numKVHeads,
1178
- hiddenSize: headDim,
1179
- rmsNormWeightOffset,
1180
- label: 'k_norm',
1181
- });
1305
+ const kNormedTensor = normalizationType === 'layernorm'
1306
+ ? await runLayerNormForMode(
1307
+ nextK,
1308
+ kNormBuf,
1309
+ getQKNormZerosBuffer(kNormSize),
1310
+ rmsNormEps,
1311
+ { batchSize: kNormBatchSize, hiddenSize: kNormSize, label: 'k_norm' }
1312
+ )
1313
+ : await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
1314
+ batchSize: kNormBatchSize,
1315
+ hiddenSize: kNormSize,
1316
+ rmsNormWeightOffset,
1317
+ label: 'k_norm',
1318
+ });
1182
1319
  if (!retainKInput) {
1183
1320
  releaseTemporary(nextK.buffer);
1184
1321
  }
@@ -6,6 +6,8 @@ import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
6
6
  import {
7
7
  recordMatmul,
8
8
  recordRMSNorm,
9
+ recordLayerNorm,
10
+ recordBiasAdd,
9
11
  recordRoPE,
10
12
  canUseRoPEQK,
11
13
  recordRoPEQK,
@@ -63,6 +65,7 @@ import {
63
65
  } from './precision-contract.js';
64
66
  import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
65
67
  import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
68
+ import { getVectorTensor } from '../weights.js';
66
69
 
67
70
  const ATTENTION_DTYPE_LOGGED = new Set();
68
71
 
@@ -284,6 +287,7 @@ export async function recordLayerAttentionGPU(
284
287
  sharedKVSourceLayerIdx != null
285
288
  );
286
289
  const canFuseInputNormProjRec = rmsNormFusionFlagRec
290
+ && config.normalizationType === 'rmsnorm'
287
291
  && canSelectFusedRmsNormProjectionRec
288
292
  && !skipInputNorm
289
293
  && layerWeights.inputNorm
@@ -319,12 +323,36 @@ export async function recordLayerAttentionGPU(
319
323
  // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
320
324
  } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
321
325
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
322
- normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
323
- batchSize: numTokens,
324
- hiddenSize,
325
- rmsNormWeightOffset: config.rmsNormWeightOffset,
326
- label: 'input_norm',
327
- });
326
+ let normBiasTensor = null;
327
+ let normBiasOwned = false;
328
+ if (config.normalizationType === 'layernorm') {
329
+ if (!layerWeights.inputNormBias) {
330
+ throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
331
+ }
332
+ ({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
333
+ layerWeights.inputNormBias,
334
+ `L${layerIdx}.input_norm_bias`,
335
+ hiddenSize,
336
+ {},
337
+ debugFlags
338
+ ));
339
+ normed = await recordLayerNorm(
340
+ recorder,
341
+ attentionInput,
342
+ normWeightBuf,
343
+ normBiasTensor.buffer,
344
+ rmsNormEps,
345
+ { batchSize: numTokens, hiddenSize, label: 'input_norm' }
346
+ );
347
+ } else {
348
+ normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
349
+ batchSize: numTokens,
350
+ hiddenSize,
351
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
352
+ label: 'input_norm',
353
+ });
354
+ }
355
+ if (normBiasOwned && normBiasTensor) releaseOrTrack(recorder, normBiasTensor.buffer);
328
356
  if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
329
357
  await runProbes('post_input_norm', normed.buffer, {
330
358
  layerIdx, numTokens, hiddenSize,
@@ -438,7 +466,10 @@ export async function recordLayerAttentionGPU(
438
466
  fusedNormEps: fusedNormEpsRec,
439
467
  fusedNormOffset: fusedNormOffsetRec,
440
468
  qkNormFusion: {
441
- enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
469
+ enabled: qkNormFusionFlag
470
+ && qkNormState.wantsQKNorm
471
+ && config.queryKeyNormType === 'rmsnorm'
472
+ && config.queryKeyNormAxis === 'head',
442
473
  getNormWeightBuffer,
443
474
  rmsNormEps,
444
475
  rmsNormWeightOffset: config.rmsNormWeightOffset,
@@ -449,6 +480,8 @@ export async function recordLayerAttentionGPU(
449
480
  qkNormRoPEFusion: {
450
481
  enabled: qkNormRoPEFusionFlag
451
482
  && qkNormState.wantsQKNorm
483
+ && config.queryKeyNormType === 'rmsnorm'
484
+ && config.queryKeyNormAxis === 'head'
452
485
  && !disableRoPE
453
486
  && !!state.ropeFreqsCos
454
487
  && !!state.ropeFreqsSin,
@@ -520,8 +553,8 @@ export async function recordLayerAttentionGPU(
520
553
  );
521
554
  }
522
555
 
523
- // Optional per-head Q/K normalization.
524
- // Some models use RMSNorm with (1+weight) offset formula, controlled by rmsNormWeightOffset.
556
+ // Optional Q/K normalization. The manifest owns both its numerical family
557
+ // and whether it applies per head or across the complete projection.
525
558
  if (qkNormState.wantsQKNorm && !qkNormApplied) {
526
559
  ({ qTensor, kTensor } = await applyAttentionQKNorm({
527
560
  recorder,
@@ -535,6 +568,8 @@ export async function recordLayerAttentionGPU(
535
568
  numKVHeads,
536
569
  headDim,
537
570
  rmsNormWeightOffset: config.rmsNormWeightOffset,
571
+ normalizationType: config.queryKeyNormType,
572
+ normalizationAxis: config.queryKeyNormAxis,
538
573
  releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
539
574
  skipKNorm: qkNormState.skipKNorm,
540
575
  retainKInput: valueAliasesKey,
@@ -1113,6 +1148,29 @@ export async function recordLayerAttentionGPU(
1113
1148
  }
1114
1149
  }
1115
1150
 
1151
+ if (layerWeights.oProjBias) {
1152
+ const { tensor: oProjBiasTensor, owned } = getVectorTensor(
1153
+ layerWeights.oProjBias,
1154
+ `L${layerIdx}.o_proj_bias`,
1155
+ hiddenSize,
1156
+ {},
1157
+ debugFlags
1158
+ );
1159
+ output = await recordBiasAdd(
1160
+ recorder,
1161
+ output,
1162
+ oProjBiasTensor,
1163
+ numTokens,
1164
+ hiddenSize,
1165
+ {
1166
+ label: `L${layerIdx}.o_proj_bias`,
1167
+ layerIdx,
1168
+ executionPolicies: state.executionPolicies ?? null,
1169
+ }
1170
+ );
1171
+ if (owned) releaseOrTrack(recorder, oProjBiasTensor.buffer);
1172
+ }
1173
+
1116
1174
  finalOutput = output;
1117
1175
  if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
1118
1176
  enqueueRecordedTensorHealth(
@@ -7,6 +7,8 @@ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js'
7
7
  import {
8
8
  runMatmul,
9
9
  runRMSNorm,
10
+ runLayerNorm,
11
+ runBiasAdd,
10
12
  runRoPE,
11
13
  canUseRoPEQK,
12
14
  runRoPEQK,
@@ -67,6 +69,7 @@ import {
67
69
  } from './precision-contract.js';
68
70
  import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
69
71
  import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
72
+ import { getVectorTensor } from '../weights.js';
70
73
 
71
74
  const ATTENTION_DTYPE_LOGGED = new Set();
72
75
 
@@ -244,6 +247,7 @@ export async function runLayerAttentionGPU(
244
247
  const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
245
248
  const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
246
249
  const canFuseInputNormIntoProjections = rmsNormFusionFlag
250
+ && config.normalizationType === 'rmsnorm'
247
251
  && hasInputNormWeight
248
252
  && canSelectFusedRmsNormProjection
249
253
  && !skipInputNorm
@@ -283,18 +287,41 @@ export async function runLayerAttentionGPU(
283
287
  // norm internally via the fused kernel variant.
284
288
  } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
285
289
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
290
+ let normBiasTensor = null;
291
+ let normBiasOwned = false;
286
292
  try {
287
293
  // Debug: norm weights for configured layers
288
294
  if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'norm_weights', debugFlags) && debugCheckBuffer) {
289
295
  await debugCheckBuffer(normWeightBuf, `L${layerIdx} input norm weights (GPU)`, 1, hiddenSize);
290
296
  }
291
297
 
292
- normed = await runRMSNorm(attentionInput, normWeightBuf, rmsNormEps, {
293
- batchSize: numTokens,
294
- hiddenSize,
295
- rmsNormWeightOffset: config.rmsNormWeightOffset,
296
- });
298
+ if (config.normalizationType === 'layernorm') {
299
+ if (!layerWeights.inputNormBias) {
300
+ throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
301
+ }
302
+ ({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
303
+ layerWeights.inputNormBias,
304
+ `L${layerIdx}.input_norm_bias`,
305
+ hiddenSize,
306
+ {},
307
+ debugFlags
308
+ ));
309
+ normed = await runLayerNorm(
310
+ attentionInput,
311
+ normWeightBuf,
312
+ normBiasTensor.buffer,
313
+ rmsNormEps,
314
+ { batchSize: numTokens, hiddenSize }
315
+ );
316
+ } else {
317
+ normed = await runRMSNorm(attentionInput, normWeightBuf, rmsNormEps, {
318
+ batchSize: numTokens,
319
+ hiddenSize,
320
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
321
+ });
322
+ }
297
323
  } finally {
324
+ if (normBiasOwned && normBiasTensor) releaseBuffer(normBiasTensor.buffer);
298
325
  if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseBuffer(normWeightBuf);
299
326
  }
300
327
 
@@ -408,7 +435,10 @@ export async function runLayerAttentionGPU(
408
435
  fusedNormEps,
409
436
  fusedNormOffset,
410
437
  qkNormFusion: {
411
- enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
438
+ enabled: qkNormFusionFlag
439
+ && qkNormState.wantsQKNorm
440
+ && config.queryKeyNormType === 'rmsnorm'
441
+ && config.queryKeyNormAxis === 'head',
412
442
  getNormWeightBuffer,
413
443
  rmsNormEps,
414
444
  rmsNormWeightOffset: config.rmsNormWeightOffset,
@@ -419,6 +449,8 @@ export async function runLayerAttentionGPU(
419
449
  qkNormRoPEFusion: {
420
450
  enabled: qkNormRoPEFusionFlag
421
451
  && qkNormState.wantsQKNorm
452
+ && config.queryKeyNormType === 'rmsnorm'
453
+ && config.queryKeyNormAxis === 'head'
422
454
  && !disableRoPE
423
455
  && !!state.ropeFreqsCos
424
456
  && !!state.ropeFreqsSin,
@@ -508,7 +540,7 @@ export async function runLayerAttentionGPU(
508
540
  await debugCheckBuffer(vTensor.buffer, `L${layerIdx} V after proj (GPU)`, numTokens, numKVHeads * headDim);
509
541
  }
510
542
 
511
- // Optional per-head Q/K normalization
543
+ // Optional Q/K normalization; the manifest owns the axis and norm family.
512
544
  if (isKernelDebugEnabled(layerIdx)) {
513
545
  logKernelStep(
514
546
  'qk_norm',
@@ -533,6 +565,8 @@ export async function runLayerAttentionGPU(
533
565
  numKVHeads,
534
566
  headDim,
535
567
  rmsNormWeightOffset: config.rmsNormWeightOffset,
568
+ normalizationType: config.queryKeyNormType,
569
+ normalizationAxis: config.queryKeyNormAxis,
536
570
  releaseTemporary: (buffer) => releaseBuffer(buffer),
537
571
  skipKNorm: qkNormState.skipKNorm,
538
572
  onQNormApplied: isKernelDebugEnabled(layerIdx)
@@ -1229,6 +1263,25 @@ export async function runLayerAttentionGPU(
1229
1263
  }
1230
1264
  }
1231
1265
 
1266
+ if (layerWeights.oProjBias) {
1267
+ const { tensor: oProjBiasTensor, owned } = getVectorTensor(
1268
+ layerWeights.oProjBias,
1269
+ `L${layerIdx}.o_proj_bias`,
1270
+ hiddenSize,
1271
+ {},
1272
+ debugFlags
1273
+ );
1274
+ try {
1275
+ output = await runBiasAdd(output, oProjBiasTensor, numTokens, hiddenSize, {
1276
+ label: `L${layerIdx}.o_proj_bias`,
1277
+ layerIdx,
1278
+ executionPolicies: state.executionPolicies ?? null,
1279
+ });
1280
+ } finally {
1281
+ if (owned) releaseBuffer(oProjBiasTensor.buffer);
1282
+ }
1283
+ }
1284
+
1232
1285
  if (oProjInputTemp) {
1233
1286
  releaseBuffer(oProjInputTemp.buffer);
1234
1287
  }
@@ -45,28 +45,37 @@ export function releaseOrTrack(recorder, buffer) {
45
45
 
46
46
 
47
47
  const qkNormOnesCache = new WeakMap();
48
+ const qkNormZerosCache = new WeakMap();
48
49
 
49
-
50
- export function getQKNormOnesBuffer(headDim) {
50
+ function getCachedQKNormConstantBuffer(cache, size, value, label) {
51
51
  const device = getDevice();
52
52
  if (!device) {
53
53
  throw new Error('No GPU device available for Q/K norm buffer');
54
54
  }
55
- let perDeviceCache = qkNormOnesCache.get(device);
55
+ let perDeviceCache = cache.get(device);
56
56
  if (!perDeviceCache) {
57
57
  perDeviceCache = new Map();
58
- qkNormOnesCache.set(device, perDeviceCache);
58
+ cache.set(device, perDeviceCache);
59
59
  }
60
- const cached = perDeviceCache.get(headDim);
60
+ const cached = perDeviceCache.get(size);
61
61
  if (cached) return cached;
62
- const data = new Float32Array(headDim);
63
- data.fill(1);
62
+ const data = new Float32Array(size);
63
+ data.fill(value);
64
64
  const buffer = device.createBuffer({
65
- label: `qk_norm_ones_${headDim}`,
65
+ label: `${label}_${size}`,
66
66
  size: data.byteLength,
67
67
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
68
68
  });
69
69
  device.queue.writeBuffer(buffer, 0, data);
70
- perDeviceCache.set(headDim, buffer);
70
+ perDeviceCache.set(size, buffer);
71
71
  return buffer;
72
72
  }
73
+
74
+
75
+ export function getQKNormOnesBuffer(headDim) {
76
+ return getCachedQKNormConstantBuffer(qkNormOnesCache, headDim, 1, 'qk_norm_ones');
77
+ }
78
+
79
+ export function getQKNormZerosBuffer(size) {
80
+ return getCachedQKNormConstantBuffer(qkNormZerosCache, size, 0, 'qk_norm_zeros');
81
+ }
@@ -186,6 +186,8 @@ export interface ParsedModelConfig {
186
186
  ropeLocalScaling: RopeScalingConfig | null;
187
187
  quantization: string;
188
188
  quantMethod: string | null;
189
+ normalizationType: 'rmsnorm' | 'layernorm';
190
+ finalNormBiasTensor: string | null;
189
191
  rmsNormEps: number;
190
192
  rmsNormWeightOffset: boolean;
191
193
  postAttentionNorm: boolean;
@@ -200,6 +202,7 @@ export interface ParsedModelConfig {
200
202
  embeddingVocabSize: number | null;
201
203
  embeddingPostprocessor: ManifestEmbeddingPostprocessorSchema | null;
202
204
  hiddenActivation: ActivationType;
205
+ gatedActivation: boolean;
203
206
  ffnBranchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
204
207
  useDoubleWideMlp: boolean;
205
208
  swigluLimit: number | null;
@@ -216,6 +219,8 @@ export interface ParsedModelConfig {
216
219
  finalLogitSoftcapping: number | null;
217
220
  attnLogitSoftcapping: number | null;
218
221
  queryKeyNorm: boolean;
222
+ queryKeyNormType: 'rmsnorm' | 'layernorm';
223
+ queryKeyNormAxis: 'head' | 'projection';
219
224
  queryKeyNormLayers: number[] | null;
220
225
  queryKeyNormWeightLayers: number[] | null;
221
226
  valueNorm: boolean;