doppler-gpu 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +27 -0
- package/models/catalog.json +612 -7
- package/package.json +13 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +126 -4
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +64 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/kernels/registry.json +52 -9
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/lora-loader.js +15 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +189 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.js +90 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/ops.js +32 -0
- package/src/inference/pipelines/text/types.d.ts +38 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +4 -2
- package/src/inference/pipelines/text.js +112 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/version.js +1 -1
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
function assertPlainObject(value, label) {
|
|
2
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
3
|
+
throw new Error(`${label} must be an object.`);
|
|
4
|
+
}
|
|
5
|
+
return value;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
function requireToken(value, label) {
|
|
9
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
10
|
+
throw new Error(`${label} must be a non-empty string.`);
|
|
11
|
+
}
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function validateRelativeFile(value) {
|
|
16
|
+
const file = requireToken(value, 'sourceTokenizer.vocabFile');
|
|
17
|
+
if (file.startsWith('/') || file.includes('\\') || file.split('/').includes('..')) {
|
|
18
|
+
throw new Error('sourceTokenizer.vocabFile must stay inside the source model directory.');
|
|
19
|
+
}
|
|
20
|
+
return file;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function parseVocab(text) {
|
|
24
|
+
if (typeof text !== 'string') {
|
|
25
|
+
throw new Error('Source vocabulary must be text.');
|
|
26
|
+
}
|
|
27
|
+
const tokens = text.replace(/\r/gu, '').split('\n');
|
|
28
|
+
if (tokens.at(-1) === '') tokens.pop();
|
|
29
|
+
if (tokens.length === 0 || tokens.some((token) => token.length === 0)) {
|
|
30
|
+
throw new Error('Source vocabulary must contain one non-empty token per line.');
|
|
31
|
+
}
|
|
32
|
+
const vocab = {};
|
|
33
|
+
for (let id = 0; id < tokens.length; id += 1) {
|
|
34
|
+
const token = tokens[id];
|
|
35
|
+
if (Object.prototype.hasOwnProperty.call(vocab, token)) {
|
|
36
|
+
throw new Error(`Source vocabulary contains duplicate token "${token}".`);
|
|
37
|
+
}
|
|
38
|
+
vocab[token] = id;
|
|
39
|
+
}
|
|
40
|
+
return { tokens, vocab };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function validateSourceTokenizerPolicy(policy) {
|
|
44
|
+
if (policy == null) return null;
|
|
45
|
+
const value = assertPlainObject(policy, 'sourceTokenizer');
|
|
46
|
+
if (value.kind !== 'character_vocab' && value.kind !== 'greedy_vocab') {
|
|
47
|
+
throw new Error(`Unsupported sourceTokenizer.kind "${String(value.kind)}".`);
|
|
48
|
+
}
|
|
49
|
+
const specialTokens = assertPlainObject(value.specialTokens, 'sourceTokenizer.specialTokens');
|
|
50
|
+
const normalized = {
|
|
51
|
+
kind: value.kind,
|
|
52
|
+
vocabFile: validateRelativeFile(value.vocabFile),
|
|
53
|
+
specialTokens: {
|
|
54
|
+
pad: requireToken(specialTokens.pad, 'sourceTokenizer.specialTokens.pad'),
|
|
55
|
+
bos: requireToken(specialTokens.bos, 'sourceTokenizer.specialTokens.bos'),
|
|
56
|
+
eos: value.kind === 'character_vocab'
|
|
57
|
+
? requireToken(specialTokens.eos, 'sourceTokenizer.specialTokens.eos')
|
|
58
|
+
: specialTokens.eos == null
|
|
59
|
+
? null
|
|
60
|
+
: requireToken(specialTokens.eos, 'sourceTokenizer.specialTokens.eos'),
|
|
61
|
+
unk: requireToken(specialTokens.unk, 'sourceTokenizer.specialTokens.unk'),
|
|
62
|
+
mask: specialTokens.mask == null
|
|
63
|
+
? null
|
|
64
|
+
: requireToken(specialTokens.mask, 'sourceTokenizer.specialTokens.mask'),
|
|
65
|
+
},
|
|
66
|
+
addBosToken: value.addBosToken === true,
|
|
67
|
+
addEosToken: value.addEosToken === true,
|
|
68
|
+
};
|
|
69
|
+
if (typeof value.addBosToken !== 'boolean' || typeof value.addEosToken !== 'boolean') {
|
|
70
|
+
throw new Error('sourceTokenizer.addBosToken and addEosToken must be booleans.');
|
|
71
|
+
}
|
|
72
|
+
return normalized;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function buildSourceTokenizerJson(policy, vocabText) {
|
|
76
|
+
const normalized = validateSourceTokenizerPolicy(policy);
|
|
77
|
+
if (!normalized) throw new Error('sourceTokenizer policy is required.');
|
|
78
|
+
const { tokens, vocab } = parseVocab(vocabText);
|
|
79
|
+
for (const [role, token] of Object.entries(normalized.specialTokens)) {
|
|
80
|
+
if (token != null && !Object.prototype.hasOwnProperty.call(vocab, token)) {
|
|
81
|
+
throw new Error(`Source vocabulary is missing ${role} token "${token}".`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const uniqueSpecialTokens = [...new Set(Object.values(normalized.specialTokens).filter(Boolean))];
|
|
85
|
+
const characterVocab = normalized.kind === 'character_vocab';
|
|
86
|
+
return {
|
|
87
|
+
version: '1.0',
|
|
88
|
+
truncation: null,
|
|
89
|
+
padding: null,
|
|
90
|
+
added_tokens: uniqueSpecialTokens.map((token) => ({
|
|
91
|
+
id: vocab[token],
|
|
92
|
+
content: token,
|
|
93
|
+
single_word: false,
|
|
94
|
+
lstrip: false,
|
|
95
|
+
rstrip: false,
|
|
96
|
+
normalized: false,
|
|
97
|
+
special: true,
|
|
98
|
+
})),
|
|
99
|
+
normalizer: null,
|
|
100
|
+
pre_tokenizer: characterVocab
|
|
101
|
+
? {
|
|
102
|
+
type: 'Split',
|
|
103
|
+
pattern: { String: '' },
|
|
104
|
+
behavior: 'Removed',
|
|
105
|
+
invert: false,
|
|
106
|
+
}
|
|
107
|
+
: null,
|
|
108
|
+
post_processor: null,
|
|
109
|
+
decoder: characterVocab
|
|
110
|
+
? {
|
|
111
|
+
type: 'WordPiece',
|
|
112
|
+
prefix: '##',
|
|
113
|
+
cleanup: false,
|
|
114
|
+
}
|
|
115
|
+
: null,
|
|
116
|
+
model: characterVocab
|
|
117
|
+
? {
|
|
118
|
+
type: 'WordPiece',
|
|
119
|
+
dropout: null,
|
|
120
|
+
unk_token: normalized.specialTokens.unk,
|
|
121
|
+
continuing_subword_prefix: '##',
|
|
122
|
+
max_input_chars_per_word: 100,
|
|
123
|
+
vocab,
|
|
124
|
+
}
|
|
125
|
+
: {
|
|
126
|
+
type: 'BPE',
|
|
127
|
+
dropout: null,
|
|
128
|
+
unk_token: normalized.specialTokens.unk,
|
|
129
|
+
continuing_subword_prefix: null,
|
|
130
|
+
end_of_word_suffix: null,
|
|
131
|
+
fuse_unk: false,
|
|
132
|
+
byte_fallback: false,
|
|
133
|
+
vocab,
|
|
134
|
+
merges: [],
|
|
135
|
+
},
|
|
136
|
+
special_tokens_map: {
|
|
137
|
+
pad_token: normalized.specialTokens.pad,
|
|
138
|
+
bos_token: normalized.specialTokens.bos,
|
|
139
|
+
...(normalized.specialTokens.eos ? { eos_token: normalized.specialTokens.eos } : {}),
|
|
140
|
+
unk_token: normalized.specialTokens.unk,
|
|
141
|
+
...(normalized.specialTokens.mask ? { mask_token: normalized.specialTokens.mask } : {}),
|
|
142
|
+
},
|
|
143
|
+
add_bos_token: normalized.addBosToken,
|
|
144
|
+
add_eos_token: normalized.addEosToken,
|
|
145
|
+
source_vocab_size: tokens.length,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function buildCharacterTokenizerJson(policy, vocabText) {
|
|
150
|
+
const normalized = validateSourceTokenizerPolicy(policy);
|
|
151
|
+
if (normalized?.kind !== 'character_vocab') {
|
|
152
|
+
throw new Error('buildCharacterTokenizerJson requires sourceTokenizer.kind="character_vocab".');
|
|
153
|
+
}
|
|
154
|
+
return buildSourceTokenizerJson(normalized, vocabText);
|
|
155
|
+
}
|
|
@@ -149,8 +149,15 @@ async function loadExternalWeights(manifest, options = {}) {
|
|
|
149
149
|
}
|
|
150
150
|
|
|
151
151
|
function assertCompleteAdapterLayers(adapter) {
|
|
152
|
+
const targetModules = new Set(adapter.targetModules || []);
|
|
153
|
+
const loadedModules = new Set();
|
|
152
154
|
for (const [layerIndex, layer] of adapter.layers.entries()) {
|
|
153
155
|
for (const [moduleName, weights] of Object.entries(layer)) {
|
|
156
|
+
if (!targetModules.has(moduleName)) {
|
|
157
|
+
throw new Error(
|
|
158
|
+
`LoRA adapter layer ${layerIndex} contains module ${moduleName} outside targetModules.`
|
|
159
|
+
);
|
|
160
|
+
}
|
|
154
161
|
const hasA = weights?.a instanceof Float32Array && weights.a.length > 0;
|
|
155
162
|
const hasB = weights?.b instanceof Float32Array && weights.b.length > 0;
|
|
156
163
|
if (!hasA || !hasB) {
|
|
@@ -158,6 +165,14 @@ function assertCompleteAdapterLayers(adapter) {
|
|
|
158
165
|
`LoRA adapter layer ${layerIndex} module ${moduleName} is incomplete; both lora_a and lora_b tensors are required.`
|
|
159
166
|
);
|
|
160
167
|
}
|
|
168
|
+
loadedModules.add(moduleName);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
for (const moduleName of targetModules) {
|
|
172
|
+
if (!loadedModules.has(moduleName)) {
|
|
173
|
+
throw new Error(
|
|
174
|
+
`LoRA adapter targetModules declares ${moduleName}, but no complete tensors were loaded for it.`
|
|
175
|
+
);
|
|
161
176
|
}
|
|
162
177
|
}
|
|
163
178
|
}
|
|
@@ -7,6 +7,7 @@ export declare const TRAINER_ARTIFACT_KIND_PEFT_ADAPTER: 'peft_adapter';
|
|
|
7
7
|
|
|
8
8
|
export declare const TRANSLATION_FULL_CHECKPOINT_PARITY_CHECKS: readonly string[];
|
|
9
9
|
export declare const COLUMBO_QWEN_ADAPTER_PARITY_CHECKS: readonly string[];
|
|
10
|
+
export declare const TINKER_PEFT_BROWSER_ADAPTER_PARITY_CHECKS: readonly string[];
|
|
10
11
|
|
|
11
12
|
export interface TrainerArtifactFileIdentity {
|
|
12
13
|
id: string;
|
|
@@ -67,7 +68,7 @@ export interface TrainerArtifactBridgeDescriptor {
|
|
|
67
68
|
receipt: string | null;
|
|
68
69
|
};
|
|
69
70
|
parity: {
|
|
70
|
-
profile: 'translation_full_checkpoint' | 'columbo_qwen_adapter';
|
|
71
|
+
profile: 'translation_full_checkpoint' | 'columbo_qwen_adapter' | 'tinker_peft_browser_adapter';
|
|
71
72
|
requiredChecks: string[];
|
|
72
73
|
};
|
|
73
74
|
claimBoundary: string;
|
|
@@ -29,6 +29,19 @@ export const COLUMBO_QWEN_ADAPTER_PARITY_CHECKS = Object.freeze([
|
|
|
29
29
|
'browser_capability_evaluation',
|
|
30
30
|
]);
|
|
31
31
|
|
|
32
|
+
export const TINKER_PEFT_BROWSER_ADAPTER_PARITY_CHECKS = Object.freeze([
|
|
33
|
+
'source_artifact_byte_identity',
|
|
34
|
+
'tokenizer_and_prompt_identity',
|
|
35
|
+
'architecture_and_conversion_lineage',
|
|
36
|
+
'evaluation_input_identity',
|
|
37
|
+
'tinker_checkpoint_vs_peft_reference',
|
|
38
|
+
'peft_reference_vs_doppler_f16',
|
|
39
|
+
'doppler_f16_vs_quantized_browser',
|
|
40
|
+
'gamma_sealed_task_gain',
|
|
41
|
+
'gamma_retention_floor',
|
|
42
|
+
'browser_capability_evaluation',
|
|
43
|
+
]);
|
|
44
|
+
|
|
32
45
|
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
|
|
33
46
|
const FILE_ROLES_BY_KIND = Object.freeze({
|
|
34
47
|
[TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT]: Object.freeze([
|
|
@@ -44,9 +57,22 @@ const FILE_ROLES_BY_KIND = Object.freeze({
|
|
|
44
57
|
'training_report',
|
|
45
58
|
]),
|
|
46
59
|
});
|
|
47
|
-
const
|
|
48
|
-
translation_full_checkpoint:
|
|
49
|
-
|
|
60
|
+
const PARITY_PROFILES = Object.freeze({
|
|
61
|
+
translation_full_checkpoint: Object.freeze({
|
|
62
|
+
artifactKind: TRAINER_ARTIFACT_KIND_FULL_CHECKPOINT,
|
|
63
|
+
selectionAuthority: 'clocksmith/gamma',
|
|
64
|
+
requiredChecks: TRANSLATION_FULL_CHECKPOINT_PARITY_CHECKS,
|
|
65
|
+
}),
|
|
66
|
+
columbo_qwen_adapter: Object.freeze({
|
|
67
|
+
artifactKind: TRAINER_ARTIFACT_KIND_PEFT_ADAPTER,
|
|
68
|
+
selectionAuthority: 'clocksmith/columbo',
|
|
69
|
+
requiredChecks: COLUMBO_QWEN_ADAPTER_PARITY_CHECKS,
|
|
70
|
+
}),
|
|
71
|
+
tinker_peft_browser_adapter: Object.freeze({
|
|
72
|
+
artifactKind: TRAINER_ARTIFACT_KIND_PEFT_ADAPTER,
|
|
73
|
+
selectionAuthority: 'clocksmith/gamma',
|
|
74
|
+
requiredChecks: TINKER_PEFT_BROWSER_ADAPTER_PARITY_CHECKS,
|
|
75
|
+
}),
|
|
50
76
|
});
|
|
51
77
|
|
|
52
78
|
function stableJson(value) {
|
|
@@ -181,18 +207,16 @@ function normalizeParity(parity, artifactKind) {
|
|
|
181
207
|
throw new Error('trainer artifact bridge: parity must be an object.');
|
|
182
208
|
}
|
|
183
209
|
const profile = requireText(parity.profile, 'parity.profile');
|
|
184
|
-
const
|
|
185
|
-
if (!
|
|
210
|
+
const profileContract = PARITY_PROFILES[profile];
|
|
211
|
+
if (!profileContract) {
|
|
186
212
|
throw new Error(`trainer artifact bridge: unsupported parity profile "${profile}".`);
|
|
187
213
|
}
|
|
188
|
-
|
|
189
|
-
? 'translation_full_checkpoint'
|
|
190
|
-
: 'columbo_qwen_adapter';
|
|
191
|
-
if (profile !== expectedProfile) {
|
|
214
|
+
if (profileContract.artifactKind !== artifactKind) {
|
|
192
215
|
throw new Error(
|
|
193
|
-
`trainer artifact bridge:
|
|
216
|
+
`trainer artifact bridge: parity profile "${profile}" does not support artifact kind "${artifactKind}".`
|
|
194
217
|
);
|
|
195
218
|
}
|
|
219
|
+
const canonicalChecks = profileContract.requiredChecks;
|
|
196
220
|
const requiredChecks = Array.isArray(parity.requiredChecks) ? parity.requiredChecks : [];
|
|
197
221
|
if (stableJson(requiredChecks) !== stableJson(canonicalChecks)) {
|
|
198
222
|
throw new Error(`trainer artifact bridge: parity.requiredChecks must match profile "${profile}".`);
|
|
@@ -435,9 +459,7 @@ export function assertTrainerArtifactCandidateEntry(descriptor) {
|
|
|
435
459
|
if (normalized.selection.status !== 'selected' || !normalized.selection.receipt) {
|
|
436
460
|
throw new Error('trainer artifact bridge: candidate entry denied; selection receipt is required.');
|
|
437
461
|
}
|
|
438
|
-
const expectedAuthority = normalized.
|
|
439
|
-
? 'clocksmith/gamma'
|
|
440
|
-
: 'clocksmith/columbo';
|
|
462
|
+
const expectedAuthority = PARITY_PROFILES[normalized.parity.profile].selectionAuthority;
|
|
441
463
|
if (normalized.selection.authority !== expectedAuthority) {
|
|
442
464
|
throw new Error(
|
|
443
465
|
`trainer artifact bridge: candidate entry denied; selection authority must be "${expectedAuthority}".`
|
|
@@ -9,6 +9,8 @@ import {
|
|
|
9
9
|
recordSplitQG,
|
|
10
10
|
runRMSNorm,
|
|
11
11
|
recordRMSNorm,
|
|
12
|
+
runLayerNorm,
|
|
13
|
+
recordLayerNorm,
|
|
12
14
|
canUseRMSNormQK,
|
|
13
15
|
runRMSNormQK,
|
|
14
16
|
recordRMSNormQK,
|
|
@@ -22,6 +24,8 @@ import {
|
|
|
22
24
|
castF32ToF16,
|
|
23
25
|
recordCastF16ToF32,
|
|
24
26
|
recordCastF32ToF16,
|
|
27
|
+
runBiasAdd,
|
|
28
|
+
recordBiasAdd,
|
|
25
29
|
} from '../../../../gpu/kernel-selector.js';
|
|
26
30
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
27
31
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
@@ -29,7 +33,8 @@ import { QK_K, Q4K_BLOCK_BYTES } from '../../../../config/schema/index.js';
|
|
|
29
33
|
import { getKernelPathMatmulPrecision } from '../../../../config/kernel-path-loader.js';
|
|
30
34
|
import { applyLoRA } from '../lora-apply.js';
|
|
31
35
|
import { getLoRAModule } from '../lora.js';
|
|
32
|
-
import { getQKNormOnesBuffer } from './types.js';
|
|
36
|
+
import { getQKNormOnesBuffer, getQKNormZerosBuffer } from './types.js';
|
|
37
|
+
import { getVectorTensor } from '../weights.js';
|
|
33
38
|
|
|
34
39
|
function getMatmulRunner(recorder) {
|
|
35
40
|
if (!recorder) {
|
|
@@ -52,6 +57,20 @@ function getSplitQGRunner(recorder) {
|
|
|
52
57
|
return (qgTensor, options) => recordSplitQG(recorder, qgTensor, options);
|
|
53
58
|
}
|
|
54
59
|
|
|
60
|
+
function getBiasAddRunner(recorder) {
|
|
61
|
+
if (!recorder) {
|
|
62
|
+
return (data, bias, numTokens, dim, options) => runBiasAdd(data, bias, numTokens, dim, options);
|
|
63
|
+
}
|
|
64
|
+
return (data, bias, numTokens, dim, options) => recordBiasAdd(
|
|
65
|
+
recorder,
|
|
66
|
+
data,
|
|
67
|
+
bias,
|
|
68
|
+
numTokens,
|
|
69
|
+
dim,
|
|
70
|
+
options
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
|
|
55
74
|
function getRmsNormRunner(recorder) {
|
|
56
75
|
if (!recorder) {
|
|
57
76
|
return (input, weight, eps, options) => runRMSNorm(input, weight, eps, options);
|
|
@@ -59,6 +78,20 @@ function getRmsNormRunner(recorder) {
|
|
|
59
78
|
return (input, weight, eps, options) => recordRMSNorm(recorder, input, weight, eps, options);
|
|
60
79
|
}
|
|
61
80
|
|
|
81
|
+
function getLayerNormRunner(recorder) {
|
|
82
|
+
if (!recorder) {
|
|
83
|
+
return (input, weight, bias, eps, options) => runLayerNorm(input, weight, bias, eps, options);
|
|
84
|
+
}
|
|
85
|
+
return (input, weight, bias, eps, options) => recordLayerNorm(
|
|
86
|
+
recorder,
|
|
87
|
+
input,
|
|
88
|
+
weight,
|
|
89
|
+
bias,
|
|
90
|
+
eps,
|
|
91
|
+
options
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
62
95
|
function getRmsNormQKRunner(recorder) {
|
|
63
96
|
if (!recorder) {
|
|
64
97
|
return (q, k, qWeight, kWeight, eps, options) => runRMSNormQK(q, k, qWeight, kWeight, eps, options);
|
|
@@ -148,13 +181,24 @@ function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemp
|
|
|
148
181
|
releaseTemporary(buffer);
|
|
149
182
|
}
|
|
150
183
|
|
|
151
|
-
function
|
|
184
|
+
export function normBufferMatchesSize(buffer, expectedSize, layerWeight = null) {
|
|
185
|
+
// Logical shape proves how many elements a loaded weight represents; it does
|
|
186
|
+
// not prove that this invocation actually resolved a buffer. Reused-KV
|
|
187
|
+
// layers deliberately leave kNormBuf null, so accepting shape alone would
|
|
188
|
+
// dispatch K normalization with a missing binding.
|
|
152
189
|
if (!buffer || !Number.isFinite(buffer.size)) {
|
|
153
190
|
return false;
|
|
154
191
|
}
|
|
192
|
+
if (Array.isArray(layerWeight?.shape)) {
|
|
193
|
+
const logicalElements = layerWeight.shape.reduce((product, dimension) => product * dimension, 1);
|
|
194
|
+
return logicalElements === expectedSize;
|
|
195
|
+
}
|
|
196
|
+
if (ArrayBuffer.isView(layerWeight)) {
|
|
197
|
+
return layerWeight.length === expectedSize;
|
|
198
|
+
}
|
|
155
199
|
const elemsF32 = buffer.size / 4;
|
|
156
200
|
const elemsF16 = buffer.size / 2;
|
|
157
|
-
return elemsF32 ===
|
|
201
|
+
return elemsF32 === expectedSize || elemsF16 === expectedSize;
|
|
158
202
|
}
|
|
159
203
|
|
|
160
204
|
function ownsNormBuffer(layerWeight) {
|
|
@@ -367,6 +411,32 @@ async function projectSingleQkvTensor({
|
|
|
367
411
|
}
|
|
368
412
|
}
|
|
369
413
|
|
|
414
|
+
const biasWeight = layerWeights?.[`${weightKey}Bias`];
|
|
415
|
+
if (biasWeight) {
|
|
416
|
+
const { tensor: biasTensor, owned } = getVectorTensor(
|
|
417
|
+
biasWeight,
|
|
418
|
+
`${role}_bias`,
|
|
419
|
+
outputSize,
|
|
420
|
+
{},
|
|
421
|
+
{}
|
|
422
|
+
);
|
|
423
|
+
try {
|
|
424
|
+
projected = await getBiasAddRunner(recorder)(
|
|
425
|
+
projected,
|
|
426
|
+
biasTensor,
|
|
427
|
+
numTokens,
|
|
428
|
+
outputSize,
|
|
429
|
+
{
|
|
430
|
+
label: `L${layerIdx}.${role}_bias`,
|
|
431
|
+
layerIdx,
|
|
432
|
+
executionPolicies,
|
|
433
|
+
}
|
|
434
|
+
);
|
|
435
|
+
} finally {
|
|
436
|
+
if (owned) releaseTemporary(biasTensor.buffer);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
370
440
|
return projected;
|
|
371
441
|
}
|
|
372
442
|
|
|
@@ -674,6 +744,30 @@ export async function projectAttentionQKV({
|
|
|
674
744
|
rmsNormEps: fusedNormEps,
|
|
675
745
|
rmsNormOffset: fusedNormOffset,
|
|
676
746
|
});
|
|
747
|
+
if (layerWeights.qkvProjBias) {
|
|
748
|
+
const { tensor: qkvBiasTensor, owned: qkvBiasOwned } = getVectorTensor(
|
|
749
|
+
layerWeights.qkvProjBias,
|
|
750
|
+
`L${layerIdx}.qkv_proj_bias`,
|
|
751
|
+
qkvSizeTotal,
|
|
752
|
+
{},
|
|
753
|
+
{}
|
|
754
|
+
);
|
|
755
|
+
try {
|
|
756
|
+
qkvTensor = await getBiasAddRunner(recorder)(
|
|
757
|
+
qkvTensor,
|
|
758
|
+
qkvBiasTensor,
|
|
759
|
+
numTokens,
|
|
760
|
+
qkvSizeTotal,
|
|
761
|
+
{
|
|
762
|
+
label: `L${layerIdx}.qkv_proj_bias`,
|
|
763
|
+
layerIdx,
|
|
764
|
+
executionPolicies,
|
|
765
|
+
}
|
|
766
|
+
);
|
|
767
|
+
} finally {
|
|
768
|
+
if (qkvBiasOwned) releaseTemporary(qkvBiasTensor.buffer);
|
|
769
|
+
}
|
|
770
|
+
}
|
|
677
771
|
const qkNormRoPEOptions = qkNormRoPEFusion
|
|
678
772
|
? { ...qkNormRoPEFusion, headDim }
|
|
679
773
|
: null;
|
|
@@ -691,8 +785,8 @@ export async function projectAttentionQKV({
|
|
|
691
785
|
if (canFuseSplitQKNormAndRoPE) {
|
|
692
786
|
qNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
|
|
693
787
|
kNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
|
|
694
|
-
const qNormApplies =
|
|
695
|
-
const kNormApplies =
|
|
788
|
+
const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
|
|
789
|
+
const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
|
|
696
790
|
if (qNormApplies && kNormApplies) {
|
|
697
791
|
const fused = await runSplitQKVRMSNormRoPEQKForMode(
|
|
698
792
|
qkvTensor,
|
|
@@ -779,8 +873,8 @@ export async function projectAttentionQKV({
|
|
|
779
873
|
if (canFuseSplitAndQKNorm) {
|
|
780
874
|
qNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
|
|
781
875
|
kNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
|
|
782
|
-
const qNormApplies =
|
|
783
|
-
const kNormApplies =
|
|
876
|
+
const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
|
|
877
|
+
const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
|
|
784
878
|
if (qNormApplies && kNormApplies) {
|
|
785
879
|
const fused = await runSplitQKVRMSNormQKForMode(
|
|
786
880
|
qkvTensor,
|
|
@@ -1099,6 +1193,8 @@ export async function applyAttentionQKNorm({
|
|
|
1099
1193
|
numKVHeads,
|
|
1100
1194
|
headDim,
|
|
1101
1195
|
rmsNormWeightOffset = false,
|
|
1196
|
+
normalizationType = 'rmsnorm',
|
|
1197
|
+
normalizationAxis = 'head',
|
|
1102
1198
|
releaseTemporary,
|
|
1103
1199
|
onQNormApplied = null,
|
|
1104
1200
|
onKNormApplied = null,
|
|
@@ -1108,6 +1204,17 @@ export async function applyAttentionQKNorm({
|
|
|
1108
1204
|
}) {
|
|
1109
1205
|
const runRmsNormForMode = getRmsNormRunner(recorder);
|
|
1110
1206
|
const runRmsNormQKForMode = getRmsNormQKRunner(recorder);
|
|
1207
|
+
const runLayerNormForMode = getLayerNormRunner(recorder);
|
|
1208
|
+
if (normalizationType !== 'rmsnorm' && normalizationType !== 'layernorm') {
|
|
1209
|
+
throw new Error(`Unsupported Q/K normalization type "${normalizationType}".`);
|
|
1210
|
+
}
|
|
1211
|
+
if (normalizationAxis !== 'head' && normalizationAxis !== 'projection') {
|
|
1212
|
+
throw new Error(`Unsupported Q/K normalization axis "${normalizationAxis}".`);
|
|
1213
|
+
}
|
|
1214
|
+
const qNormSize = normalizationAxis === 'projection' ? numHeads * headDim : headDim;
|
|
1215
|
+
const kNormSize = normalizationAxis === 'projection' ? numKVHeads * headDim : headDim;
|
|
1216
|
+
const qNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numHeads;
|
|
1217
|
+
const kNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numKVHeads;
|
|
1111
1218
|
let nextQ = qTensor;
|
|
1112
1219
|
let nextK = kTensor;
|
|
1113
1220
|
let qNormBuf = null;
|
|
@@ -1121,18 +1228,32 @@ export async function applyAttentionQKNorm({
|
|
|
1121
1228
|
if (wantsQNorm) {
|
|
1122
1229
|
qNormBuf = layerWeights.qNorm && getNormWeightBuffer
|
|
1123
1230
|
? getNormWeightBuffer(layerWeights.qNorm, 'q_norm')
|
|
1124
|
-
: getQKNormOnesBuffer(
|
|
1231
|
+
: getQKNormOnesBuffer(qNormSize);
|
|
1125
1232
|
}
|
|
1126
1233
|
if (wantsKNorm) {
|
|
1127
1234
|
kNormBuf = layerWeights.kNorm && getNormWeightBuffer
|
|
1128
1235
|
? getNormWeightBuffer(layerWeights.kNorm, 'k_norm')
|
|
1129
|
-
: getQKNormOnesBuffer(
|
|
1236
|
+
: getQKNormOnesBuffer(kNormSize);
|
|
1130
1237
|
}
|
|
1131
1238
|
|
|
1132
|
-
const qNormApplies =
|
|
1133
|
-
const kNormApplies =
|
|
1239
|
+
const qNormApplies = normBufferMatchesSize(qNormBuf, qNormSize, layerWeights.qNorm);
|
|
1240
|
+
const kNormApplies = normBufferMatchesSize(kNormBuf, kNormSize, layerWeights.kNorm);
|
|
1241
|
+
if (wantsQNorm && !qNormApplies) {
|
|
1242
|
+
throw new Error(
|
|
1243
|
+
`Q normalization weight has ${String(qNormBuf?.size ?? 'unknown')} bytes; ` +
|
|
1244
|
+
`expected ${qNormSize} f16 or f32 elements.`
|
|
1245
|
+
);
|
|
1246
|
+
}
|
|
1247
|
+
if (wantsKNorm && !kNormApplies) {
|
|
1248
|
+
throw new Error(
|
|
1249
|
+
`K normalization weight has ${String(kNormBuf?.size ?? 'unknown')} bytes; ` +
|
|
1250
|
+
`expected ${kNormSize} f16 or f32 elements.`
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1134
1253
|
if (
|
|
1135
|
-
|
|
1254
|
+
normalizationType === 'rmsnorm'
|
|
1255
|
+
&& normalizationAxis === 'head'
|
|
1256
|
+
&& qNormApplies
|
|
1136
1257
|
&& kNormApplies
|
|
1137
1258
|
&& canUseRMSNormQK(nextQ, nextK, { skipKNorm })
|
|
1138
1259
|
) {
|
|
@@ -1159,12 +1280,20 @@ export async function applyAttentionQKNorm({
|
|
|
1159
1280
|
}
|
|
1160
1281
|
|
|
1161
1282
|
if (qNormApplies) {
|
|
1162
|
-
const qNormedTensor =
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1283
|
+
const qNormedTensor = normalizationType === 'layernorm'
|
|
1284
|
+
? await runLayerNormForMode(
|
|
1285
|
+
nextQ,
|
|
1286
|
+
qNormBuf,
|
|
1287
|
+
getQKNormZerosBuffer(qNormSize),
|
|
1288
|
+
rmsNormEps,
|
|
1289
|
+
{ batchSize: qNormBatchSize, hiddenSize: qNormSize, label: 'q_norm' }
|
|
1290
|
+
)
|
|
1291
|
+
: await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
|
|
1292
|
+
batchSize: qNormBatchSize,
|
|
1293
|
+
hiddenSize: qNormSize,
|
|
1294
|
+
rmsNormWeightOffset,
|
|
1295
|
+
label: 'q_norm',
|
|
1296
|
+
});
|
|
1168
1297
|
releaseTemporary(nextQ.buffer);
|
|
1169
1298
|
nextQ = qNormedTensor;
|
|
1170
1299
|
if (onQNormApplied) {
|
|
@@ -1173,12 +1302,20 @@ export async function applyAttentionQKNorm({
|
|
|
1173
1302
|
}
|
|
1174
1303
|
|
|
1175
1304
|
if (kNormApplies) {
|
|
1176
|
-
const kNormedTensor =
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1305
|
+
const kNormedTensor = normalizationType === 'layernorm'
|
|
1306
|
+
? await runLayerNormForMode(
|
|
1307
|
+
nextK,
|
|
1308
|
+
kNormBuf,
|
|
1309
|
+
getQKNormZerosBuffer(kNormSize),
|
|
1310
|
+
rmsNormEps,
|
|
1311
|
+
{ batchSize: kNormBatchSize, hiddenSize: kNormSize, label: 'k_norm' }
|
|
1312
|
+
)
|
|
1313
|
+
: await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
|
|
1314
|
+
batchSize: kNormBatchSize,
|
|
1315
|
+
hiddenSize: kNormSize,
|
|
1316
|
+
rmsNormWeightOffset,
|
|
1317
|
+
label: 'k_norm',
|
|
1318
|
+
});
|
|
1182
1319
|
if (!retainKInput) {
|
|
1183
1320
|
releaseTemporary(nextK.buffer);
|
|
1184
1321
|
}
|