doppler-gpu 0.4.10 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +27 -0
  3. package/models/catalog.json +612 -7
  4. package/package.json +13 -1
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +126 -4
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +64 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/kernels/registry.json +52 -9
  18. package/src/config/merge.js +24 -0
  19. package/src/config/required-inference-fields-contract-check.js +2 -0
  20. package/src/config/schema/converter.schema.d.ts +7 -0
  21. package/src/config/schema/converter.schema.js +15 -0
  22. package/src/config/schema/inference.schema.d.ts +8 -0
  23. package/src/config/schema/manifest.schema.d.ts +32 -1
  24. package/src/config/schema/manifest.schema.js +26 -0
  25. package/src/converter/conversion-plan.js +15 -1
  26. package/src/converter/core.js +5 -4
  27. package/src/converter/source-tensor-rules.d.ts +50 -0
  28. package/src/converter/source-tensor-rules.js +180 -0
  29. package/src/converter/source-tokenizer.d.ts +45 -0
  30. package/src/converter/source-tokenizer.js +155 -0
  31. package/src/experimental/adapters/lora-loader.js +15 -0
  32. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  33. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  34. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  35. package/src/inference/pipelines/text/attention/projections.js +161 -24
  36. package/src/inference/pipelines/text/attention/record.js +67 -9
  37. package/src/inference/pipelines/text/attention/run.js +60 -7
  38. package/src/inference/pipelines/text/attention/types.js +18 -9
  39. package/src/inference/pipelines/text/config.d.ts +5 -0
  40. package/src/inference/pipelines/text/config.js +91 -0
  41. package/src/inference/pipelines/text/ffn/dense.js +189 -2
  42. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  43. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  44. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  45. package/src/inference/pipelines/text/generator-steps.js +3 -1
  46. package/src/inference/pipelines/text/generator.js +90 -3
  47. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  48. package/src/inference/pipelines/text/init.d.ts +2 -0
  49. package/src/inference/pipelines/text/init.js +2 -0
  50. package/src/inference/pipelines/text/layer.js +6 -0
  51. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  52. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  53. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  54. package/src/inference/pipelines/text/logits/index.js +85 -11
  55. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  56. package/src/inference/pipelines/text/ops.js +32 -0
  57. package/src/inference/pipelines/text/types.d.ts +38 -0
  58. package/src/inference/pipelines/text/weights.d.ts +10 -0
  59. package/src/inference/pipelines/text/weights.js +33 -1
  60. package/src/inference/pipelines/text.d.ts +4 -2
  61. package/src/inference/pipelines/text.js +112 -0
  62. package/src/inference/tokenizers/behavior-flags.js +5 -1
  63. package/src/inference/tokenizers/bundled.js +99 -8
  64. package/src/loader/doppler-loader.d.ts +2 -0
  65. package/src/loader/doppler-loader.js +9 -0
  66. package/src/loader/final-weights-loader.d.ts +8 -0
  67. package/src/loader/final-weights-loader.js +42 -0
  68. package/src/loader/layer-loader.d.ts +6 -0
  69. package/src/loader/layer-loader.js +102 -3
  70. package/src/loader/loader-types.d.ts +15 -2
  71. package/src/tooling/node-converter.js +33 -1
  72. package/src/tooling/node-webgpu.js +85 -2
  73. package/src/version.js +1 -1
@@ -0,0 +1,155 @@
1
+ function assertPlainObject(value, label) {
2
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
3
+ throw new Error(`${label} must be an object.`);
4
+ }
5
+ return value;
6
+ }
7
+
8
+ function requireToken(value, label) {
9
+ if (typeof value !== 'string' || value.length === 0) {
10
+ throw new Error(`${label} must be a non-empty string.`);
11
+ }
12
+ return value;
13
+ }
14
+
15
+ function validateRelativeFile(value) {
16
+ const file = requireToken(value, 'sourceTokenizer.vocabFile');
17
+ if (file.startsWith('/') || file.includes('\\') || file.split('/').includes('..')) {
18
+ throw new Error('sourceTokenizer.vocabFile must stay inside the source model directory.');
19
+ }
20
+ return file;
21
+ }
22
+
23
+ function parseVocab(text) {
24
+ if (typeof text !== 'string') {
25
+ throw new Error('Source vocabulary must be text.');
26
+ }
27
+ const tokens = text.replace(/\r/gu, '').split('\n');
28
+ if (tokens.at(-1) === '') tokens.pop();
29
+ if (tokens.length === 0 || tokens.some((token) => token.length === 0)) {
30
+ throw new Error('Source vocabulary must contain one non-empty token per line.');
31
+ }
32
+ const vocab = {};
33
+ for (let id = 0; id < tokens.length; id += 1) {
34
+ const token = tokens[id];
35
+ if (Object.prototype.hasOwnProperty.call(vocab, token)) {
36
+ throw new Error(`Source vocabulary contains duplicate token "${token}".`);
37
+ }
38
+ vocab[token] = id;
39
+ }
40
+ return { tokens, vocab };
41
+ }
42
+
43
+ export function validateSourceTokenizerPolicy(policy) {
44
+ if (policy == null) return null;
45
+ const value = assertPlainObject(policy, 'sourceTokenizer');
46
+ if (value.kind !== 'character_vocab' && value.kind !== 'greedy_vocab') {
47
+ throw new Error(`Unsupported sourceTokenizer.kind "${String(value.kind)}".`);
48
+ }
49
+ const specialTokens = assertPlainObject(value.specialTokens, 'sourceTokenizer.specialTokens');
50
+ const normalized = {
51
+ kind: value.kind,
52
+ vocabFile: validateRelativeFile(value.vocabFile),
53
+ specialTokens: {
54
+ pad: requireToken(specialTokens.pad, 'sourceTokenizer.specialTokens.pad'),
55
+ bos: requireToken(specialTokens.bos, 'sourceTokenizer.specialTokens.bos'),
56
+ eos: value.kind === 'character_vocab'
57
+ ? requireToken(specialTokens.eos, 'sourceTokenizer.specialTokens.eos')
58
+ : specialTokens.eos == null
59
+ ? null
60
+ : requireToken(specialTokens.eos, 'sourceTokenizer.specialTokens.eos'),
61
+ unk: requireToken(specialTokens.unk, 'sourceTokenizer.specialTokens.unk'),
62
+ mask: specialTokens.mask == null
63
+ ? null
64
+ : requireToken(specialTokens.mask, 'sourceTokenizer.specialTokens.mask'),
65
+ },
66
+ addBosToken: value.addBosToken === true,
67
+ addEosToken: value.addEosToken === true,
68
+ };
69
+ if (typeof value.addBosToken !== 'boolean' || typeof value.addEosToken !== 'boolean') {
70
+ throw new Error('sourceTokenizer.addBosToken and addEosToken must be booleans.');
71
+ }
72
+ return normalized;
73
+ }
74
+
75
+ export function buildSourceTokenizerJson(policy, vocabText) {
76
+ const normalized = validateSourceTokenizerPolicy(policy);
77
+ if (!normalized) throw new Error('sourceTokenizer policy is required.');
78
+ const { tokens, vocab } = parseVocab(vocabText);
79
+ for (const [role, token] of Object.entries(normalized.specialTokens)) {
80
+ if (token != null && !Object.prototype.hasOwnProperty.call(vocab, token)) {
81
+ throw new Error(`Source vocabulary is missing ${role} token "${token}".`);
82
+ }
83
+ }
84
+ const uniqueSpecialTokens = [...new Set(Object.values(normalized.specialTokens).filter(Boolean))];
85
+ const characterVocab = normalized.kind === 'character_vocab';
86
+ return {
87
+ version: '1.0',
88
+ truncation: null,
89
+ padding: null,
90
+ added_tokens: uniqueSpecialTokens.map((token) => ({
91
+ id: vocab[token],
92
+ content: token,
93
+ single_word: false,
94
+ lstrip: false,
95
+ rstrip: false,
96
+ normalized: false,
97
+ special: true,
98
+ })),
99
+ normalizer: null,
100
+ pre_tokenizer: characterVocab
101
+ ? {
102
+ type: 'Split',
103
+ pattern: { String: '' },
104
+ behavior: 'Removed',
105
+ invert: false,
106
+ }
107
+ : null,
108
+ post_processor: null,
109
+ decoder: characterVocab
110
+ ? {
111
+ type: 'WordPiece',
112
+ prefix: '##',
113
+ cleanup: false,
114
+ }
115
+ : null,
116
+ model: characterVocab
117
+ ? {
118
+ type: 'WordPiece',
119
+ dropout: null,
120
+ unk_token: normalized.specialTokens.unk,
121
+ continuing_subword_prefix: '##',
122
+ max_input_chars_per_word: 100,
123
+ vocab,
124
+ }
125
+ : {
126
+ type: 'BPE',
127
+ dropout: null,
128
+ unk_token: normalized.specialTokens.unk,
129
+ continuing_subword_prefix: null,
130
+ end_of_word_suffix: null,
131
+ fuse_unk: false,
132
+ byte_fallback: false,
133
+ vocab,
134
+ merges: [],
135
+ },
136
+ special_tokens_map: {
137
+ pad_token: normalized.specialTokens.pad,
138
+ bos_token: normalized.specialTokens.bos,
139
+ ...(normalized.specialTokens.eos ? { eos_token: normalized.specialTokens.eos } : {}),
140
+ unk_token: normalized.specialTokens.unk,
141
+ ...(normalized.specialTokens.mask ? { mask_token: normalized.specialTokens.mask } : {}),
142
+ },
143
+ add_bos_token: normalized.addBosToken,
144
+ add_eos_token: normalized.addEosToken,
145
+ source_vocab_size: tokens.length,
146
+ };
147
+ }
148
+
149
+ export function buildCharacterTokenizerJson(policy, vocabText) {
150
+ const normalized = validateSourceTokenizerPolicy(policy);
151
+ if (normalized?.kind !== 'character_vocab') {
152
+ throw new Error('buildCharacterTokenizerJson requires sourceTokenizer.kind="character_vocab".');
153
+ }
154
+ return buildSourceTokenizerJson(normalized, vocabText);
155
+ }
@@ -149,8 +149,15 @@ async function loadExternalWeights(manifest, options = {}) {
149
149
  }
150
150
 
151
151
  function assertCompleteAdapterLayers(adapter) {
152
+ const targetModules = new Set(adapter.targetModules || []);
153
+ const loadedModules = new Set();
152
154
  for (const [layerIndex, layer] of adapter.layers.entries()) {
153
155
  for (const [moduleName, weights] of Object.entries(layer)) {
156
+ if (!targetModules.has(moduleName)) {
157
+ throw new Error(
158
+ `LoRA adapter layer ${layerIndex} contains module ${moduleName} outside targetModules.`
159
+ );
160
+ }
154
161
  const hasA = weights?.a instanceof Float32Array && weights.a.length > 0;
155
162
  const hasB = weights?.b instanceof Float32Array && weights.b.length > 0;
156
163
  if (!hasA || !hasB) {
@@ -158,6 +165,14 @@ function assertCompleteAdapterLayers(adapter) {
158
165
  `LoRA adapter layer ${layerIndex} module ${moduleName} is incomplete; both lora_a and lora_b tensors are required.`
159
166
  );
160
167
  }
168
+ loadedModules.add(moduleName);
169
+ }
170
+ }
171
+ for (const moduleName of targetModules) {
172
+ if (!loadedModules.has(moduleName)) {
173
+ throw new Error(
174
+ `LoRA adapter targetModules declares ${moduleName}, but no complete tensors were loaded for it.`
175
+ );
161
176
  }
162
177
  }
163
178
  }
@@ -7,6 +7,7 @@ export declare const TRAINER_ARTIFACT_KIND_PEFT_ADAPTER: 'peft_adapter';
7
7
 
8
8
  export declare const TRANSLATION_FULL_CHECKPOINT_PARITY_CHECKS: readonly string[];
9
9
  export declare const COLUMBO_QWEN_ADAPTER_PARITY_CHECKS: readonly string[];
10
+ export declare const TINKER_PEFT_BROWSER_ADAPTER_PARITY_CHECKS: readonly string[];
10
11
 
11
12
  export interface TrainerArtifactFileIdentity {
12
13
  id: string;
@@ -67,7 +68,7 @@ export interface TrainerArtifactBridgeDescriptor {
67
68
  receipt: string | null;
68
69
  };
69
70
  parity: {
70
- profile: 'translation_full_checkpoint' | 'columbo_qwen_adapter';
71
+ profile: 'translation_full_checkpoint' | 'columbo_qwen_adapter' | 'tinker_peft_browser_adapter';
71
72
  requiredChecks: string[];
72
73
  };
73
74
  claimBoundary: string;
@@ -29,6 +29,19 @@ export const COLUMBO_QWEN_ADAPTER_PARITY_CHECKS = Object.freeze([
29
29
  'browser_capability_evaluation',
30
30
  ]);
31
31
 
32
+ export const TINKER_PEFT_BROWSER_ADAPTER_PARITY_CHECKS = Object.freeze([
33
+ 'source_artifact_byte_identity',
34
+ 'tokenizer_and_prompt_identity',
35
+ 'architecture_and_conversion_lineage',
36
+ 'evaluation_input_identity',
37
+ 'tinker_checkpoint_vs_peft_reference',
38
+ 'peft_reference_vs_doppler_f16',
39
+ 'doppler_f16_vs_quantized_browser',
40
+ 'gamma_sealed_task_gain',
41
+ 'gamma_retention_floor',
42
+ 'browser_capability_evaluation',
43
+ ]);
44
+
32
45
  const SHA256_PATTERN = /^[0-9a-f]{64}$/;
33
46
  const FILE_ROLES_BY_KIND = Object.freeze({
34
47
  [TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT]: Object.freeze([
@@ -44,9 +57,22 @@ const FILE_ROLES_BY_KIND = Object.freeze({
44
57
  'training_report',
45
58
  ]),
46
59
  });
47
- const PARITY_CHECKS_BY_PROFILE = Object.freeze({
48
- translation_full_checkpoint: TRANSLATION_FULL_CHECKPOINT_PARITY_CHECKS,
49
- columbo_qwen_adapter: COLUMBO_QWEN_ADAPTER_PARITY_CHECKS,
60
+ const PARITY_PROFILES = Object.freeze({
61
+ translation_full_checkpoint: Object.freeze({
62
+ artifactKind: TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT,
63
+ selectionAuthority: 'clocksmith/gamma',
64
+ requiredChecks: TRANSLATION_FULL_CHECKPOINT_PARITY_CHECKS,
65
+ }),
66
+ columbo_qwen_adapter: Object.freeze({
67
+ artifactKind: TRAINER_ARTIFACT_KIND_PEFT_ADAPTER,
68
+ selectionAuthority: 'clocksmith/columbo',
69
+ requiredChecks: COLUMBO_QWEN_ADAPTER_PARITY_CHECKS,
70
+ }),
71
+ tinker_peft_browser_adapter: Object.freeze({
72
+ artifactKind: TRAINER_ARTIFACT_KIND_PEFT_ADAPTER,
73
+ selectionAuthority: 'clocksmith/gamma',
74
+ requiredChecks: TINKER_PEFT_BROWSER_ADAPTER_PARITY_CHECKS,
75
+ }),
50
76
  });
51
77
 
52
78
  function stableJson(value) {
@@ -181,18 +207,16 @@ function normalizeParity(parity, artifactKind) {
181
207
  throw new Error('trainer artifact bridge: parity must be an object.');
182
208
  }
183
209
  const profile = requireText(parity.profile, 'parity.profile');
184
- const canonicalChecks = PARITY_CHECKS_BY_PROFILE[profile];
185
- if (!canonicalChecks) {
210
+ const profileContract = PARITY_PROFILES[profile];
211
+ if (!profileContract) {
186
212
  throw new Error(`trainer artifact bridge: unsupported parity profile "${profile}".`);
187
213
  }
188
- const expectedProfile = artifactKind === TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT
189
- ? 'translation_full_checkpoint'
190
- : 'columbo_qwen_adapter';
191
- if (profile !== expectedProfile) {
214
+ if (profileContract.artifactKind !== artifactKind) {
192
215
  throw new Error(
193
- `trainer artifact bridge: artifact kind "${artifactKind}" requires parity profile "${expectedProfile}".`
216
+ `trainer artifact bridge: parity profile "${profile}" does not support artifact kind "${artifactKind}".`
194
217
  );
195
218
  }
219
+ const canonicalChecks = profileContract.requiredChecks;
196
220
  const requiredChecks = Array.isArray(parity.requiredChecks) ? parity.requiredChecks : [];
197
221
  if (stableJson(requiredChecks) !== stableJson(canonicalChecks)) {
198
222
  throw new Error(`trainer artifact bridge: parity.requiredChecks must match profile "${profile}".`);
@@ -435,9 +459,7 @@ export function assertTrainerArtifactCandidateEntry(descriptor) {
435
459
  if (normalized.selection.status !== 'selected' || !normalized.selection.receipt) {
436
460
  throw new Error('trainer artifact bridge: candidate entry denied; selection receipt is required.');
437
461
  }
438
- const expectedAuthority = normalized.artifact.kind === TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT
439
- ? 'clocksmith/gamma'
440
- : 'clocksmith/columbo';
462
+ const expectedAuthority = PARITY_PROFILES[normalized.parity.profile].selectionAuthority;
441
463
  if (normalized.selection.authority !== expectedAuthority) {
442
464
  throw new Error(
443
465
  `trainer artifact bridge: candidate entry denied; selection authority must be "${expectedAuthority}".`
@@ -19,6 +19,8 @@ const ATTN_CONFIG_REQUIRED_FIELDS = Object.freeze([
19
19
  'activationDtype',
20
20
  'attnSoftcap',
21
21
  'queryPreAttnScalar',
22
+ 'queryKeyNormType',
23
+ 'queryKeyNormAxis',
22
24
  ]);
23
25
 
24
26
  /**
@@ -9,6 +9,8 @@ import {
9
9
  recordSplitQG,
10
10
  runRMSNorm,
11
11
  recordRMSNorm,
12
+ runLayerNorm,
13
+ recordLayerNorm,
12
14
  canUseRMSNormQK,
13
15
  runRMSNormQK,
14
16
  recordRMSNormQK,
@@ -22,6 +24,8 @@ import {
22
24
  castF32ToF16,
23
25
  recordCastF16ToF32,
24
26
  recordCastF32ToF16,
27
+ runBiasAdd,
28
+ recordBiasAdd,
25
29
  } from '../../../../gpu/kernel-selector.js';
26
30
  import { createTensor } from '../../../../gpu/tensor.js';
27
31
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
@@ -29,7 +33,8 @@ import { QK_K, Q4K_BLOCK_BYTES } from '../../../../config/schema/index.js';
29
33
  import { getKernelPathMatmulPrecision } from '../../../../config/kernel-path-loader.js';
30
34
  import { applyLoRA } from '../lora-apply.js';
31
35
  import { getLoRAModule } from '../lora.js';
32
- import { getQKNormOnesBuffer } from './types.js';
36
+ import { getQKNormOnesBuffer, getQKNormZerosBuffer } from './types.js';
37
+ import { getVectorTensor } from '../weights.js';
33
38
 
34
39
  function getMatmulRunner(recorder) {
35
40
  if (!recorder) {
@@ -52,6 +57,20 @@ function getSplitQGRunner(recorder) {
52
57
  return (qgTensor, options) => recordSplitQG(recorder, qgTensor, options);
53
58
  }
54
59
 
60
+ function getBiasAddRunner(recorder) {
61
+ if (!recorder) {
62
+ return (data, bias, numTokens, dim, options) => runBiasAdd(data, bias, numTokens, dim, options);
63
+ }
64
+ return (data, bias, numTokens, dim, options) => recordBiasAdd(
65
+ recorder,
66
+ data,
67
+ bias,
68
+ numTokens,
69
+ dim,
70
+ options
71
+ );
72
+ }
73
+
55
74
  function getRmsNormRunner(recorder) {
56
75
  if (!recorder) {
57
76
  return (input, weight, eps, options) => runRMSNorm(input, weight, eps, options);
@@ -59,6 +78,20 @@ function getRmsNormRunner(recorder) {
59
78
  return (input, weight, eps, options) => recordRMSNorm(recorder, input, weight, eps, options);
60
79
  }
61
80
 
81
+ function getLayerNormRunner(recorder) {
82
+ if (!recorder) {
83
+ return (input, weight, bias, eps, options) => runLayerNorm(input, weight, bias, eps, options);
84
+ }
85
+ return (input, weight, bias, eps, options) => recordLayerNorm(
86
+ recorder,
87
+ input,
88
+ weight,
89
+ bias,
90
+ eps,
91
+ options
92
+ );
93
+ }
94
+
62
95
  function getRmsNormQKRunner(recorder) {
63
96
  if (!recorder) {
64
97
  return (q, k, qWeight, kWeight, eps, options) => runRMSNormQK(q, k, qWeight, kWeight, eps, options);
@@ -148,13 +181,24 @@ function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemp
148
181
  releaseTemporary(buffer);
149
182
  }
150
183
 
151
- function normBufferMatchesHeadDim(buffer, headDim) {
184
+ export function normBufferMatchesSize(buffer, expectedSize, layerWeight = null) {
185
+ // Logical shape proves how many elements a loaded weight represents; it does
186
+ // not prove that this invocation actually resolved a buffer. Reused-KV
187
+ // layers deliberately leave kNormBuf null, so accepting shape alone would
188
+ // dispatch K normalization with a missing binding.
152
189
  if (!buffer || !Number.isFinite(buffer.size)) {
153
190
  return false;
154
191
  }
192
+ if (Array.isArray(layerWeight?.shape)) {
193
+ const logicalElements = layerWeight.shape.reduce((product, dimension) => product * dimension, 1);
194
+ return logicalElements === expectedSize;
195
+ }
196
+ if (ArrayBuffer.isView(layerWeight)) {
197
+ return layerWeight.length === expectedSize;
198
+ }
155
199
  const elemsF32 = buffer.size / 4;
156
200
  const elemsF16 = buffer.size / 2;
157
- return elemsF32 === headDim || elemsF16 === headDim;
201
+ return elemsF32 === expectedSize || elemsF16 === expectedSize;
158
202
  }
159
203
 
160
204
  function ownsNormBuffer(layerWeight) {
@@ -367,6 +411,32 @@ async function projectSingleQkvTensor({
367
411
  }
368
412
  }
369
413
 
414
+ const biasWeight = layerWeights?.[`${weightKey}Bias`];
415
+ if (biasWeight) {
416
+ const { tensor: biasTensor, owned } = getVectorTensor(
417
+ biasWeight,
418
+ `${role}_bias`,
419
+ outputSize,
420
+ {},
421
+ {}
422
+ );
423
+ try {
424
+ projected = await getBiasAddRunner(recorder)(
425
+ projected,
426
+ biasTensor,
427
+ numTokens,
428
+ outputSize,
429
+ {
430
+ label: `L${layerIdx}.${role}_bias`,
431
+ layerIdx,
432
+ executionPolicies,
433
+ }
434
+ );
435
+ } finally {
436
+ if (owned) releaseTemporary(biasTensor.buffer);
437
+ }
438
+ }
439
+
370
440
  return projected;
371
441
  }
372
442
 
@@ -674,6 +744,30 @@ export async function projectAttentionQKV({
674
744
  rmsNormEps: fusedNormEps,
675
745
  rmsNormOffset: fusedNormOffset,
676
746
  });
747
+ if (layerWeights.qkvProjBias) {
748
+ const { tensor: qkvBiasTensor, owned: qkvBiasOwned } = getVectorTensor(
749
+ layerWeights.qkvProjBias,
750
+ `L${layerIdx}.qkv_proj_bias`,
751
+ qkvSizeTotal,
752
+ {},
753
+ {}
754
+ );
755
+ try {
756
+ qkvTensor = await getBiasAddRunner(recorder)(
757
+ qkvTensor,
758
+ qkvBiasTensor,
759
+ numTokens,
760
+ qkvSizeTotal,
761
+ {
762
+ label: `L${layerIdx}.qkv_proj_bias`,
763
+ layerIdx,
764
+ executionPolicies,
765
+ }
766
+ );
767
+ } finally {
768
+ if (qkvBiasOwned) releaseTemporary(qkvBiasTensor.buffer);
769
+ }
770
+ }
677
771
  const qkNormRoPEOptions = qkNormRoPEFusion
678
772
  ? { ...qkNormRoPEFusion, headDim }
679
773
  : null;
@@ -691,8 +785,8 @@ export async function projectAttentionQKV({
691
785
  if (canFuseSplitQKNormAndRoPE) {
692
786
  qNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
693
787
  kNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
694
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
695
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
788
+ const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
789
+ const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
696
790
  if (qNormApplies && kNormApplies) {
697
791
  const fused = await runSplitQKVRMSNormRoPEQKForMode(
698
792
  qkvTensor,
@@ -779,8 +873,8 @@ export async function projectAttentionQKV({
779
873
  if (canFuseSplitAndQKNorm) {
780
874
  qNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
781
875
  kNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
782
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
783
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
876
+ const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
877
+ const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
784
878
  if (qNormApplies && kNormApplies) {
785
879
  const fused = await runSplitQKVRMSNormQKForMode(
786
880
  qkvTensor,
@@ -1099,6 +1193,8 @@ export async function applyAttentionQKNorm({
1099
1193
  numKVHeads,
1100
1194
  headDim,
1101
1195
  rmsNormWeightOffset = false,
1196
+ normalizationType = 'rmsnorm',
1197
+ normalizationAxis = 'head',
1102
1198
  releaseTemporary,
1103
1199
  onQNormApplied = null,
1104
1200
  onKNormApplied = null,
@@ -1108,6 +1204,17 @@ export async function applyAttentionQKNorm({
1108
1204
  }) {
1109
1205
  const runRmsNormForMode = getRmsNormRunner(recorder);
1110
1206
  const runRmsNormQKForMode = getRmsNormQKRunner(recorder);
1207
+ const runLayerNormForMode = getLayerNormRunner(recorder);
1208
+ if (normalizationType !== 'rmsnorm' && normalizationType !== 'layernorm') {
1209
+ throw new Error(`Unsupported Q/K normalization type "${normalizationType}".`);
1210
+ }
1211
+ if (normalizationAxis !== 'head' && normalizationAxis !== 'projection') {
1212
+ throw new Error(`Unsupported Q/K normalization axis "${normalizationAxis}".`);
1213
+ }
1214
+ const qNormSize = normalizationAxis === 'projection' ? numHeads * headDim : headDim;
1215
+ const kNormSize = normalizationAxis === 'projection' ? numKVHeads * headDim : headDim;
1216
+ const qNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numHeads;
1217
+ const kNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numKVHeads;
1111
1218
  let nextQ = qTensor;
1112
1219
  let nextK = kTensor;
1113
1220
  let qNormBuf = null;
@@ -1121,18 +1228,32 @@ export async function applyAttentionQKNorm({
1121
1228
  if (wantsQNorm) {
1122
1229
  qNormBuf = layerWeights.qNorm && getNormWeightBuffer
1123
1230
  ? getNormWeightBuffer(layerWeights.qNorm, 'q_norm')
1124
- : getQKNormOnesBuffer(headDim);
1231
+ : getQKNormOnesBuffer(qNormSize);
1125
1232
  }
1126
1233
  if (wantsKNorm) {
1127
1234
  kNormBuf = layerWeights.kNorm && getNormWeightBuffer
1128
1235
  ? getNormWeightBuffer(layerWeights.kNorm, 'k_norm')
1129
- : getQKNormOnesBuffer(headDim);
1236
+ : getQKNormOnesBuffer(kNormSize);
1130
1237
  }
1131
1238
 
1132
- const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
1133
- const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
1239
+ const qNormApplies = normBufferMatchesSize(qNormBuf, qNormSize, layerWeights.qNorm);
1240
+ const kNormApplies = normBufferMatchesSize(kNormBuf, kNormSize, layerWeights.kNorm);
1241
+ if (wantsQNorm && !qNormApplies) {
1242
+ throw new Error(
1243
+ `Q normalization weight has ${String(qNormBuf?.size ?? 'unknown')} bytes; ` +
1244
+ `expected ${qNormSize} f16 or f32 elements.`
1245
+ );
1246
+ }
1247
+ if (wantsKNorm && !kNormApplies) {
1248
+ throw new Error(
1249
+ `K normalization weight has ${String(kNormBuf?.size ?? 'unknown')} bytes; ` +
1250
+ `expected ${kNormSize} f16 or f32 elements.`
1251
+ );
1252
+ }
1134
1253
  if (
1135
- qNormApplies
1254
+ normalizationType === 'rmsnorm'
1255
+ && normalizationAxis === 'head'
1256
+ && qNormApplies
1136
1257
  && kNormApplies
1137
1258
  && canUseRMSNormQK(nextQ, nextK, { skipKNorm })
1138
1259
  ) {
@@ -1159,12 +1280,20 @@ export async function applyAttentionQKNorm({
1159
1280
  }
1160
1281
 
1161
1282
  if (qNormApplies) {
1162
- const qNormedTensor = await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
1163
- batchSize: numTokens * numHeads,
1164
- hiddenSize: headDim,
1165
- rmsNormWeightOffset,
1166
- label: 'q_norm',
1167
- });
1283
+ const qNormedTensor = normalizationType === 'layernorm'
1284
+ ? await runLayerNormForMode(
1285
+ nextQ,
1286
+ qNormBuf,
1287
+ getQKNormZerosBuffer(qNormSize),
1288
+ rmsNormEps,
1289
+ { batchSize: qNormBatchSize, hiddenSize: qNormSize, label: 'q_norm' }
1290
+ )
1291
+ : await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
1292
+ batchSize: qNormBatchSize,
1293
+ hiddenSize: qNormSize,
1294
+ rmsNormWeightOffset,
1295
+ label: 'q_norm',
1296
+ });
1168
1297
  releaseTemporary(nextQ.buffer);
1169
1298
  nextQ = qNormedTensor;
1170
1299
  if (onQNormApplied) {
@@ -1173,12 +1302,20 @@ export async function applyAttentionQKNorm({
1173
1302
  }
1174
1303
 
1175
1304
  if (kNormApplies) {
1176
- const kNormedTensor = await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
1177
- batchSize: numTokens * numKVHeads,
1178
- hiddenSize: headDim,
1179
- rmsNormWeightOffset,
1180
- label: 'k_norm',
1181
- });
1305
+ const kNormedTensor = normalizationType === 'layernorm'
1306
+ ? await runLayerNormForMode(
1307
+ nextK,
1308
+ kNormBuf,
1309
+ getQKNormZerosBuffer(kNormSize),
1310
+ rmsNormEps,
1311
+ { batchSize: kNormBatchSize, hiddenSize: kNormSize, label: 'k_norm' }
1312
+ )
1313
+ : await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
1314
+ batchSize: kNormBatchSize,
1315
+ hiddenSize: kNormSize,
1316
+ rmsNormWeightOffset,
1317
+ label: 'k_norm',
1318
+ });
1182
1319
  if (!retainKInput) {
1183
1320
  releaseTemporary(nextK.buffer);
1184
1321
  }