doppler-gpu 0.4.10 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +40 -1
- package/models/catalog.json +637 -32
- package/package.json +24 -3
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +135 -13
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +69 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +73 -11
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +24 -4
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +205 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +196 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +33 -0
- package/src/inference/pipelines/text/types.d.ts +66 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +9 -2
- package/src/inference/pipelines/text.js +116 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -196,16 +196,28 @@ export function resolveMatmulBackwardDxVariant(weight) {
|
|
|
196
196
|
const dtype = String(weight?.dtype || 'f32').toLowerCase();
|
|
197
197
|
if (dtype === 'f16') return 'f16_weight';
|
|
198
198
|
if (dtype === 'f32') return 'default';
|
|
199
|
+
if (dtype === 'q4k') return 'q4k_weight';
|
|
199
200
|
throw new Error(`matmul backward dX does not support weight dtype "${dtype}".`);
|
|
200
201
|
}
|
|
201
202
|
|
|
203
|
+
function resolveMatmulBackwardDxDispatch(weight, K, transposeB) {
|
|
204
|
+
const variant = resolveMatmulBackwardDxVariant(weight);
|
|
205
|
+
if (variant === 'q4k_weight' && !transposeB) {
|
|
206
|
+
throw new Error('matmul backward dX requires transposeB=true for row-wise Q4K weights [N,K].');
|
|
207
|
+
}
|
|
208
|
+
return {
|
|
209
|
+
variant,
|
|
210
|
+
numBlocksPerRow: variant === 'q4k_weight' ? Math.ceil(K / 256) : 0,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
202
214
|
export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
|
|
203
215
|
const { alpha = 1.0, transposeB = false, outputBuffer = null } = options;
|
|
204
216
|
const device = getDevice();
|
|
205
217
|
const outputSize = M * K * 4;
|
|
206
218
|
const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
|
|
207
|
-
|
|
208
|
-
const pipeline = await createPipeline('matmul_backward',
|
|
219
|
+
const dispatchConfig = resolveMatmulBackwardDxDispatch(W, K, transposeB);
|
|
220
|
+
const pipeline = await createPipeline('matmul_backward', dispatchConfig.variant);
|
|
209
221
|
|
|
210
222
|
const uniformBuffer = createUniformBufferWithView(
|
|
211
223
|
'matmul_backward_uniforms',
|
|
@@ -216,6 +228,7 @@ export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
|
|
|
216
228
|
view.setUint32(8, K, true);
|
|
217
229
|
view.setFloat32(12, alpha, true);
|
|
218
230
|
view.setUint32(16, transposeB ? 1 : 0, true);
|
|
231
|
+
view.setUint32(20, dispatchConfig.numBlocksPerRow, true);
|
|
219
232
|
},
|
|
220
233
|
null,
|
|
221
234
|
device
|
|
@@ -250,8 +263,8 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
|
|
|
250
263
|
const device = recorder.device;
|
|
251
264
|
const outputSize = M * K * 4;
|
|
252
265
|
const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
|
|
253
|
-
|
|
254
|
-
const pipeline = await createPipeline('matmul_backward',
|
|
266
|
+
const dispatchConfig = resolveMatmulBackwardDxDispatch(W, K, transposeB);
|
|
267
|
+
const pipeline = await createPipeline('matmul_backward', dispatchConfig.variant);
|
|
255
268
|
|
|
256
269
|
const uniformBuffer = createUniformBufferWithView(
|
|
257
270
|
'matmul_backward_uniforms',
|
|
@@ -262,6 +275,7 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
|
|
|
262
275
|
view.setUint32(8, K, true);
|
|
263
276
|
view.setFloat32(12, alpha, true);
|
|
264
277
|
view.setUint32(16, transposeB ? 1 : 0, true);
|
|
278
|
+
view.setUint32(20, dispatchConfig.numBlocksPerRow, true);
|
|
265
279
|
},
|
|
266
280
|
recorder
|
|
267
281
|
);
|
|
@@ -9,6 +9,8 @@ import {
|
|
|
9
9
|
recordSplitQG,
|
|
10
10
|
runRMSNorm,
|
|
11
11
|
recordRMSNorm,
|
|
12
|
+
runLayerNorm,
|
|
13
|
+
recordLayerNorm,
|
|
12
14
|
canUseRMSNormQK,
|
|
13
15
|
runRMSNormQK,
|
|
14
16
|
recordRMSNormQK,
|
|
@@ -22,6 +24,8 @@ import {
|
|
|
22
24
|
castF32ToF16,
|
|
23
25
|
recordCastF16ToF32,
|
|
24
26
|
recordCastF32ToF16,
|
|
27
|
+
runBiasAdd,
|
|
28
|
+
recordBiasAdd,
|
|
25
29
|
} from '../../../../gpu/kernel-selector.js';
|
|
26
30
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
27
31
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
@@ -29,7 +33,8 @@ import { QK_K, Q4K_BLOCK_BYTES } from '../../../../config/schema/index.js';
|
|
|
29
33
|
import { getKernelPathMatmulPrecision } from '../../../../config/kernel-path-loader.js';
|
|
30
34
|
import { applyLoRA } from '../lora-apply.js';
|
|
31
35
|
import { getLoRAModule } from '../lora.js';
|
|
32
|
-
import { getQKNormOnesBuffer } from './types.js';
|
|
36
|
+
import { getQKNormOnesBuffer, getQKNormZerosBuffer } from './types.js';
|
|
37
|
+
import { getVectorTensor } from '../weights.js';
|
|
33
38
|
|
|
34
39
|
function getMatmulRunner(recorder) {
|
|
35
40
|
if (!recorder) {
|
|
@@ -52,6 +57,20 @@ function getSplitQGRunner(recorder) {
|
|
|
52
57
|
return (qgTensor, options) => recordSplitQG(recorder, qgTensor, options);
|
|
53
58
|
}
|
|
54
59
|
|
|
60
|
+
function getBiasAddRunner(recorder) {
|
|
61
|
+
if (!recorder) {
|
|
62
|
+
return (data, bias, numTokens, dim, options) => runBiasAdd(data, bias, numTokens, dim, options);
|
|
63
|
+
}
|
|
64
|
+
return (data, bias, numTokens, dim, options) => recordBiasAdd(
|
|
65
|
+
recorder,
|
|
66
|
+
data,
|
|
67
|
+
bias,
|
|
68
|
+
numTokens,
|
|
69
|
+
dim,
|
|
70
|
+
options
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
|
|
55
74
|
function getRmsNormRunner(recorder) {
|
|
56
75
|
if (!recorder) {
|
|
57
76
|
return (input, weight, eps, options) => runRMSNorm(input, weight, eps, options);
|
|
@@ -59,6 +78,20 @@ function getRmsNormRunner(recorder) {
|
|
|
59
78
|
return (input, weight, eps, options) => recordRMSNorm(recorder, input, weight, eps, options);
|
|
60
79
|
}
|
|
61
80
|
|
|
81
|
+
function getLayerNormRunner(recorder) {
|
|
82
|
+
if (!recorder) {
|
|
83
|
+
return (input, weight, bias, eps, options) => runLayerNorm(input, weight, bias, eps, options);
|
|
84
|
+
}
|
|
85
|
+
return (input, weight, bias, eps, options) => recordLayerNorm(
|
|
86
|
+
recorder,
|
|
87
|
+
input,
|
|
88
|
+
weight,
|
|
89
|
+
bias,
|
|
90
|
+
eps,
|
|
91
|
+
options
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
62
95
|
function getRmsNormQKRunner(recorder) {
|
|
63
96
|
if (!recorder) {
|
|
64
97
|
return (q, k, qWeight, kWeight, eps, options) => runRMSNormQK(q, k, qWeight, kWeight, eps, options);
|
|
@@ -148,13 +181,24 @@ function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemp
|
|
|
148
181
|
releaseTemporary(buffer);
|
|
149
182
|
}
|
|
150
183
|
|
|
151
|
-
function
|
|
184
|
+
export function normBufferMatchesSize(buffer, expectedSize, layerWeight = null) {
|
|
185
|
+
// Logical shape proves how many elements a loaded weight represents; it does
|
|
186
|
+
// not prove that this invocation actually resolved a buffer. Reused-KV
|
|
187
|
+
// layers deliberately leave kNormBuf null, so accepting shape alone would
|
|
188
|
+
// dispatch K normalization with a missing binding.
|
|
152
189
|
if (!buffer || !Number.isFinite(buffer.size)) {
|
|
153
190
|
return false;
|
|
154
191
|
}
|
|
192
|
+
if (Array.isArray(layerWeight?.shape)) {
|
|
193
|
+
const logicalElements = layerWeight.shape.reduce((product, dimension) => product * dimension, 1);
|
|
194
|
+
return logicalElements === expectedSize;
|
|
195
|
+
}
|
|
196
|
+
if (ArrayBuffer.isView(layerWeight)) {
|
|
197
|
+
return layerWeight.length === expectedSize;
|
|
198
|
+
}
|
|
155
199
|
const elemsF32 = buffer.size / 4;
|
|
156
200
|
const elemsF16 = buffer.size / 2;
|
|
157
|
-
return elemsF32 ===
|
|
201
|
+
return elemsF32 === expectedSize || elemsF16 === expectedSize;
|
|
158
202
|
}
|
|
159
203
|
|
|
160
204
|
function ownsNormBuffer(layerWeight) {
|
|
@@ -367,6 +411,32 @@ async function projectSingleQkvTensor({
|
|
|
367
411
|
}
|
|
368
412
|
}
|
|
369
413
|
|
|
414
|
+
const biasWeight = layerWeights?.[`${weightKey}Bias`];
|
|
415
|
+
if (biasWeight) {
|
|
416
|
+
const { tensor: biasTensor, owned } = getVectorTensor(
|
|
417
|
+
biasWeight,
|
|
418
|
+
`${role}_bias`,
|
|
419
|
+
outputSize,
|
|
420
|
+
{},
|
|
421
|
+
{}
|
|
422
|
+
);
|
|
423
|
+
try {
|
|
424
|
+
projected = await getBiasAddRunner(recorder)(
|
|
425
|
+
projected,
|
|
426
|
+
biasTensor,
|
|
427
|
+
numTokens,
|
|
428
|
+
outputSize,
|
|
429
|
+
{
|
|
430
|
+
label: `L${layerIdx}.${role}_bias`,
|
|
431
|
+
layerIdx,
|
|
432
|
+
executionPolicies,
|
|
433
|
+
}
|
|
434
|
+
);
|
|
435
|
+
} finally {
|
|
436
|
+
if (owned) releaseTemporary(biasTensor.buffer);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
370
440
|
return projected;
|
|
371
441
|
}
|
|
372
442
|
|
|
@@ -674,6 +744,30 @@ export async function projectAttentionQKV({
|
|
|
674
744
|
rmsNormEps: fusedNormEps,
|
|
675
745
|
rmsNormOffset: fusedNormOffset,
|
|
676
746
|
});
|
|
747
|
+
if (layerWeights.qkvProjBias) {
|
|
748
|
+
const { tensor: qkvBiasTensor, owned: qkvBiasOwned } = getVectorTensor(
|
|
749
|
+
layerWeights.qkvProjBias,
|
|
750
|
+
`L${layerIdx}.qkv_proj_bias`,
|
|
751
|
+
qkvSizeTotal,
|
|
752
|
+
{},
|
|
753
|
+
{}
|
|
754
|
+
);
|
|
755
|
+
try {
|
|
756
|
+
qkvTensor = await getBiasAddRunner(recorder)(
|
|
757
|
+
qkvTensor,
|
|
758
|
+
qkvBiasTensor,
|
|
759
|
+
numTokens,
|
|
760
|
+
qkvSizeTotal,
|
|
761
|
+
{
|
|
762
|
+
label: `L${layerIdx}.qkv_proj_bias`,
|
|
763
|
+
layerIdx,
|
|
764
|
+
executionPolicies,
|
|
765
|
+
}
|
|
766
|
+
);
|
|
767
|
+
} finally {
|
|
768
|
+
if (qkvBiasOwned) releaseTemporary(qkvBiasTensor.buffer);
|
|
769
|
+
}
|
|
770
|
+
}
|
|
677
771
|
const qkNormRoPEOptions = qkNormRoPEFusion
|
|
678
772
|
? { ...qkNormRoPEFusion, headDim }
|
|
679
773
|
: null;
|
|
@@ -691,8 +785,8 @@ export async function projectAttentionQKV({
|
|
|
691
785
|
if (canFuseSplitQKNormAndRoPE) {
|
|
692
786
|
qNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
|
|
693
787
|
kNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
|
|
694
|
-
const qNormApplies =
|
|
695
|
-
const kNormApplies =
|
|
788
|
+
const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
|
|
789
|
+
const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
|
|
696
790
|
if (qNormApplies && kNormApplies) {
|
|
697
791
|
const fused = await runSplitQKVRMSNormRoPEQKForMode(
|
|
698
792
|
qkvTensor,
|
|
@@ -779,8 +873,8 @@ export async function projectAttentionQKV({
|
|
|
779
873
|
if (canFuseSplitAndQKNorm) {
|
|
780
874
|
qNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
|
|
781
875
|
kNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
|
|
782
|
-
const qNormApplies =
|
|
783
|
-
const kNormApplies =
|
|
876
|
+
const qNormApplies = normBufferMatchesSize(qNormBuf, headDim, layerWeights.qNorm);
|
|
877
|
+
const kNormApplies = normBufferMatchesSize(kNormBuf, headDim, layerWeights.kNorm);
|
|
784
878
|
if (qNormApplies && kNormApplies) {
|
|
785
879
|
const fused = await runSplitQKVRMSNormQKForMode(
|
|
786
880
|
qkvTensor,
|
|
@@ -1099,6 +1193,8 @@ export async function applyAttentionQKNorm({
|
|
|
1099
1193
|
numKVHeads,
|
|
1100
1194
|
headDim,
|
|
1101
1195
|
rmsNormWeightOffset = false,
|
|
1196
|
+
normalizationType = 'rmsnorm',
|
|
1197
|
+
normalizationAxis = 'head',
|
|
1102
1198
|
releaseTemporary,
|
|
1103
1199
|
onQNormApplied = null,
|
|
1104
1200
|
onKNormApplied = null,
|
|
@@ -1108,6 +1204,17 @@ export async function applyAttentionQKNorm({
|
|
|
1108
1204
|
}) {
|
|
1109
1205
|
const runRmsNormForMode = getRmsNormRunner(recorder);
|
|
1110
1206
|
const runRmsNormQKForMode = getRmsNormQKRunner(recorder);
|
|
1207
|
+
const runLayerNormForMode = getLayerNormRunner(recorder);
|
|
1208
|
+
if (normalizationType !== 'rmsnorm' && normalizationType !== 'layernorm') {
|
|
1209
|
+
throw new Error(`Unsupported Q/K normalization type "${normalizationType}".`);
|
|
1210
|
+
}
|
|
1211
|
+
if (normalizationAxis !== 'head' && normalizationAxis !== 'projection') {
|
|
1212
|
+
throw new Error(`Unsupported Q/K normalization axis "${normalizationAxis}".`);
|
|
1213
|
+
}
|
|
1214
|
+
const qNormSize = normalizationAxis === 'projection' ? numHeads * headDim : headDim;
|
|
1215
|
+
const kNormSize = normalizationAxis === 'projection' ? numKVHeads * headDim : headDim;
|
|
1216
|
+
const qNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numHeads;
|
|
1217
|
+
const kNormBatchSize = normalizationAxis === 'projection' ? numTokens : numTokens * numKVHeads;
|
|
1111
1218
|
let nextQ = qTensor;
|
|
1112
1219
|
let nextK = kTensor;
|
|
1113
1220
|
let qNormBuf = null;
|
|
@@ -1121,18 +1228,32 @@ export async function applyAttentionQKNorm({
|
|
|
1121
1228
|
if (wantsQNorm) {
|
|
1122
1229
|
qNormBuf = layerWeights.qNorm && getNormWeightBuffer
|
|
1123
1230
|
? getNormWeightBuffer(layerWeights.qNorm, 'q_norm')
|
|
1124
|
-
: getQKNormOnesBuffer(
|
|
1231
|
+
: getQKNormOnesBuffer(qNormSize);
|
|
1125
1232
|
}
|
|
1126
1233
|
if (wantsKNorm) {
|
|
1127
1234
|
kNormBuf = layerWeights.kNorm && getNormWeightBuffer
|
|
1128
1235
|
? getNormWeightBuffer(layerWeights.kNorm, 'k_norm')
|
|
1129
|
-
: getQKNormOnesBuffer(
|
|
1236
|
+
: getQKNormOnesBuffer(kNormSize);
|
|
1130
1237
|
}
|
|
1131
1238
|
|
|
1132
|
-
const qNormApplies =
|
|
1133
|
-
const kNormApplies =
|
|
1239
|
+
const qNormApplies = normBufferMatchesSize(qNormBuf, qNormSize, layerWeights.qNorm);
|
|
1240
|
+
const kNormApplies = normBufferMatchesSize(kNormBuf, kNormSize, layerWeights.kNorm);
|
|
1241
|
+
if (wantsQNorm && !qNormApplies) {
|
|
1242
|
+
throw new Error(
|
|
1243
|
+
`Q normalization weight has ${String(qNormBuf?.size ?? 'unknown')} bytes; ` +
|
|
1244
|
+
`expected ${qNormSize} f16 or f32 elements.`
|
|
1245
|
+
);
|
|
1246
|
+
}
|
|
1247
|
+
if (wantsKNorm && !kNormApplies) {
|
|
1248
|
+
throw new Error(
|
|
1249
|
+
`K normalization weight has ${String(kNormBuf?.size ?? 'unknown')} bytes; ` +
|
|
1250
|
+
`expected ${kNormSize} f16 or f32 elements.`
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1134
1253
|
if (
|
|
1135
|
-
|
|
1254
|
+
normalizationType === 'rmsnorm'
|
|
1255
|
+
&& normalizationAxis === 'head'
|
|
1256
|
+
&& qNormApplies
|
|
1136
1257
|
&& kNormApplies
|
|
1137
1258
|
&& canUseRMSNormQK(nextQ, nextK, { skipKNorm })
|
|
1138
1259
|
) {
|
|
@@ -1159,12 +1280,20 @@ export async function applyAttentionQKNorm({
|
|
|
1159
1280
|
}
|
|
1160
1281
|
|
|
1161
1282
|
if (qNormApplies) {
|
|
1162
|
-
const qNormedTensor =
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1283
|
+
const qNormedTensor = normalizationType === 'layernorm'
|
|
1284
|
+
? await runLayerNormForMode(
|
|
1285
|
+
nextQ,
|
|
1286
|
+
qNormBuf,
|
|
1287
|
+
getQKNormZerosBuffer(qNormSize),
|
|
1288
|
+
rmsNormEps,
|
|
1289
|
+
{ batchSize: qNormBatchSize, hiddenSize: qNormSize, label: 'q_norm' }
|
|
1290
|
+
)
|
|
1291
|
+
: await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
|
|
1292
|
+
batchSize: qNormBatchSize,
|
|
1293
|
+
hiddenSize: qNormSize,
|
|
1294
|
+
rmsNormWeightOffset,
|
|
1295
|
+
label: 'q_norm',
|
|
1296
|
+
});
|
|
1168
1297
|
releaseTemporary(nextQ.buffer);
|
|
1169
1298
|
nextQ = qNormedTensor;
|
|
1170
1299
|
if (onQNormApplied) {
|
|
@@ -1173,12 +1302,20 @@ export async function applyAttentionQKNorm({
|
|
|
1173
1302
|
}
|
|
1174
1303
|
|
|
1175
1304
|
if (kNormApplies) {
|
|
1176
|
-
const kNormedTensor =
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1305
|
+
const kNormedTensor = normalizationType === 'layernorm'
|
|
1306
|
+
? await runLayerNormForMode(
|
|
1307
|
+
nextK,
|
|
1308
|
+
kNormBuf,
|
|
1309
|
+
getQKNormZerosBuffer(kNormSize),
|
|
1310
|
+
rmsNormEps,
|
|
1311
|
+
{ batchSize: kNormBatchSize, hiddenSize: kNormSize, label: 'k_norm' }
|
|
1312
|
+
)
|
|
1313
|
+
: await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
|
|
1314
|
+
batchSize: kNormBatchSize,
|
|
1315
|
+
hiddenSize: kNormSize,
|
|
1316
|
+
rmsNormWeightOffset,
|
|
1317
|
+
label: 'k_norm',
|
|
1318
|
+
});
|
|
1182
1319
|
if (!retainKInput) {
|
|
1183
1320
|
releaseTemporary(nextK.buffer);
|
|
1184
1321
|
}
|
|
@@ -6,6 +6,8 @@ import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
|
6
6
|
import {
|
|
7
7
|
recordMatmul,
|
|
8
8
|
recordRMSNorm,
|
|
9
|
+
recordLayerNorm,
|
|
10
|
+
recordBiasAdd,
|
|
9
11
|
recordRoPE,
|
|
10
12
|
canUseRoPEQK,
|
|
11
13
|
recordRoPEQK,
|
|
@@ -63,6 +65,7 @@ import {
|
|
|
63
65
|
} from './precision-contract.js';
|
|
64
66
|
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
65
67
|
import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
|
|
68
|
+
import { getVectorTensor } from '../weights.js';
|
|
66
69
|
|
|
67
70
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
68
71
|
|
|
@@ -284,6 +287,7 @@ export async function recordLayerAttentionGPU(
|
|
|
284
287
|
sharedKVSourceLayerIdx != null
|
|
285
288
|
);
|
|
286
289
|
const canFuseInputNormProjRec = rmsNormFusionFlagRec
|
|
290
|
+
&& config.normalizationType === 'rmsnorm'
|
|
287
291
|
&& canSelectFusedRmsNormProjectionRec
|
|
288
292
|
&& !skipInputNorm
|
|
289
293
|
&& layerWeights.inputNorm
|
|
@@ -319,12 +323,36 @@ export async function recordLayerAttentionGPU(
|
|
|
319
323
|
// Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
|
|
320
324
|
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
321
325
|
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
326
|
+
let normBiasTensor = null;
|
|
327
|
+
let normBiasOwned = false;
|
|
328
|
+
if (config.normalizationType === 'layernorm') {
|
|
329
|
+
if (!layerWeights.inputNormBias) {
|
|
330
|
+
throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
|
|
331
|
+
}
|
|
332
|
+
({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
|
|
333
|
+
layerWeights.inputNormBias,
|
|
334
|
+
`L${layerIdx}.input_norm_bias`,
|
|
335
|
+
hiddenSize,
|
|
336
|
+
{},
|
|
337
|
+
debugFlags
|
|
338
|
+
));
|
|
339
|
+
normed = await recordLayerNorm(
|
|
340
|
+
recorder,
|
|
341
|
+
attentionInput,
|
|
342
|
+
normWeightBuf,
|
|
343
|
+
normBiasTensor.buffer,
|
|
344
|
+
rmsNormEps,
|
|
345
|
+
{ batchSize: numTokens, hiddenSize, label: 'input_norm' }
|
|
346
|
+
);
|
|
347
|
+
} else {
|
|
348
|
+
normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
|
|
349
|
+
batchSize: numTokens,
|
|
350
|
+
hiddenSize,
|
|
351
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
352
|
+
label: 'input_norm',
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
if (normBiasOwned && normBiasTensor) releaseOrTrack(recorder, normBiasTensor.buffer);
|
|
328
356
|
if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
|
|
329
357
|
await runProbes('post_input_norm', normed.buffer, {
|
|
330
358
|
layerIdx, numTokens, hiddenSize,
|
|
@@ -438,7 +466,10 @@ export async function recordLayerAttentionGPU(
|
|
|
438
466
|
fusedNormEps: fusedNormEpsRec,
|
|
439
467
|
fusedNormOffset: fusedNormOffsetRec,
|
|
440
468
|
qkNormFusion: {
|
|
441
|
-
enabled: qkNormFusionFlag
|
|
469
|
+
enabled: qkNormFusionFlag
|
|
470
|
+
&& qkNormState.wantsQKNorm
|
|
471
|
+
&& config.queryKeyNormType === 'rmsnorm'
|
|
472
|
+
&& config.queryKeyNormAxis === 'head',
|
|
442
473
|
getNormWeightBuffer,
|
|
443
474
|
rmsNormEps,
|
|
444
475
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
@@ -449,6 +480,8 @@ export async function recordLayerAttentionGPU(
|
|
|
449
480
|
qkNormRoPEFusion: {
|
|
450
481
|
enabled: qkNormRoPEFusionFlag
|
|
451
482
|
&& qkNormState.wantsQKNorm
|
|
483
|
+
&& config.queryKeyNormType === 'rmsnorm'
|
|
484
|
+
&& config.queryKeyNormAxis === 'head'
|
|
452
485
|
&& !disableRoPE
|
|
453
486
|
&& !!state.ropeFreqsCos
|
|
454
487
|
&& !!state.ropeFreqsSin,
|
|
@@ -520,8 +553,8 @@ export async function recordLayerAttentionGPU(
|
|
|
520
553
|
);
|
|
521
554
|
}
|
|
522
555
|
|
|
523
|
-
// Optional
|
|
524
|
-
//
|
|
556
|
+
// Optional Q/K normalization. The manifest owns both its numerical family
|
|
557
|
+
// and whether it applies per head or across the complete projection.
|
|
525
558
|
if (qkNormState.wantsQKNorm && !qkNormApplied) {
|
|
526
559
|
({ qTensor, kTensor } = await applyAttentionQKNorm({
|
|
527
560
|
recorder,
|
|
@@ -535,6 +568,8 @@ export async function recordLayerAttentionGPU(
|
|
|
535
568
|
numKVHeads,
|
|
536
569
|
headDim,
|
|
537
570
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
571
|
+
normalizationType: config.queryKeyNormType,
|
|
572
|
+
normalizationAxis: config.queryKeyNormAxis,
|
|
538
573
|
releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
|
|
539
574
|
skipKNorm: qkNormState.skipKNorm,
|
|
540
575
|
retainKInput: valueAliasesKey,
|
|
@@ -1113,6 +1148,29 @@ export async function recordLayerAttentionGPU(
|
|
|
1113
1148
|
}
|
|
1114
1149
|
}
|
|
1115
1150
|
|
|
1151
|
+
if (layerWeights.oProjBias) {
|
|
1152
|
+
const { tensor: oProjBiasTensor, owned } = getVectorTensor(
|
|
1153
|
+
layerWeights.oProjBias,
|
|
1154
|
+
`L${layerIdx}.o_proj_bias`,
|
|
1155
|
+
hiddenSize,
|
|
1156
|
+
{},
|
|
1157
|
+
debugFlags
|
|
1158
|
+
);
|
|
1159
|
+
output = await recordBiasAdd(
|
|
1160
|
+
recorder,
|
|
1161
|
+
output,
|
|
1162
|
+
oProjBiasTensor,
|
|
1163
|
+
numTokens,
|
|
1164
|
+
hiddenSize,
|
|
1165
|
+
{
|
|
1166
|
+
label: `L${layerIdx}.o_proj_bias`,
|
|
1167
|
+
layerIdx,
|
|
1168
|
+
executionPolicies: state.executionPolicies ?? null,
|
|
1169
|
+
}
|
|
1170
|
+
);
|
|
1171
|
+
if (owned) releaseOrTrack(recorder, oProjBiasTensor.buffer);
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1116
1174
|
finalOutput = output;
|
|
1117
1175
|
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
1118
1176
|
enqueueRecordedTensorHealth(
|
|
@@ -7,6 +7,8 @@ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js'
|
|
|
7
7
|
import {
|
|
8
8
|
runMatmul,
|
|
9
9
|
runRMSNorm,
|
|
10
|
+
runLayerNorm,
|
|
11
|
+
runBiasAdd,
|
|
10
12
|
runRoPE,
|
|
11
13
|
canUseRoPEQK,
|
|
12
14
|
runRoPEQK,
|
|
@@ -67,6 +69,7 @@ import {
|
|
|
67
69
|
} from './precision-contract.js';
|
|
68
70
|
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
69
71
|
import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
|
|
72
|
+
import { getVectorTensor } from '../weights.js';
|
|
70
73
|
|
|
71
74
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
72
75
|
|
|
@@ -244,6 +247,7 @@ export async function runLayerAttentionGPU(
|
|
|
244
247
|
const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
|
|
245
248
|
const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
|
|
246
249
|
const canFuseInputNormIntoProjections = rmsNormFusionFlag
|
|
250
|
+
&& config.normalizationType === 'rmsnorm'
|
|
247
251
|
&& hasInputNormWeight
|
|
248
252
|
&& canSelectFusedRmsNormProjection
|
|
249
253
|
&& !skipInputNorm
|
|
@@ -283,18 +287,41 @@ export async function runLayerAttentionGPU(
|
|
|
283
287
|
// norm internally via the fused kernel variant.
|
|
284
288
|
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
285
289
|
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
290
|
+
let normBiasTensor = null;
|
|
291
|
+
let normBiasOwned = false;
|
|
286
292
|
try {
|
|
287
293
|
// Debug: norm weights for configured layers
|
|
288
294
|
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'norm_weights', debugFlags) && debugCheckBuffer) {
|
|
289
295
|
await debugCheckBuffer(normWeightBuf, `L${layerIdx} input norm weights (GPU)`, 1, hiddenSize);
|
|
290
296
|
}
|
|
291
297
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
298
|
+
if (config.normalizationType === 'layernorm') {
|
|
299
|
+
if (!layerWeights.inputNormBias) {
|
|
300
|
+
throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
|
|
301
|
+
}
|
|
302
|
+
({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
|
|
303
|
+
layerWeights.inputNormBias,
|
|
304
|
+
`L${layerIdx}.input_norm_bias`,
|
|
305
|
+
hiddenSize,
|
|
306
|
+
{},
|
|
307
|
+
debugFlags
|
|
308
|
+
));
|
|
309
|
+
normed = await runLayerNorm(
|
|
310
|
+
attentionInput,
|
|
311
|
+
normWeightBuf,
|
|
312
|
+
normBiasTensor.buffer,
|
|
313
|
+
rmsNormEps,
|
|
314
|
+
{ batchSize: numTokens, hiddenSize }
|
|
315
|
+
);
|
|
316
|
+
} else {
|
|
317
|
+
normed = await runRMSNorm(attentionInput, normWeightBuf, rmsNormEps, {
|
|
318
|
+
batchSize: numTokens,
|
|
319
|
+
hiddenSize,
|
|
320
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
321
|
+
});
|
|
322
|
+
}
|
|
297
323
|
} finally {
|
|
324
|
+
if (normBiasOwned && normBiasTensor) releaseBuffer(normBiasTensor.buffer);
|
|
298
325
|
if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseBuffer(normWeightBuf);
|
|
299
326
|
}
|
|
300
327
|
|
|
@@ -408,7 +435,10 @@ export async function runLayerAttentionGPU(
|
|
|
408
435
|
fusedNormEps,
|
|
409
436
|
fusedNormOffset,
|
|
410
437
|
qkNormFusion: {
|
|
411
|
-
enabled: qkNormFusionFlag
|
|
438
|
+
enabled: qkNormFusionFlag
|
|
439
|
+
&& qkNormState.wantsQKNorm
|
|
440
|
+
&& config.queryKeyNormType === 'rmsnorm'
|
|
441
|
+
&& config.queryKeyNormAxis === 'head',
|
|
412
442
|
getNormWeightBuffer,
|
|
413
443
|
rmsNormEps,
|
|
414
444
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
@@ -419,6 +449,8 @@ export async function runLayerAttentionGPU(
|
|
|
419
449
|
qkNormRoPEFusion: {
|
|
420
450
|
enabled: qkNormRoPEFusionFlag
|
|
421
451
|
&& qkNormState.wantsQKNorm
|
|
452
|
+
&& config.queryKeyNormType === 'rmsnorm'
|
|
453
|
+
&& config.queryKeyNormAxis === 'head'
|
|
422
454
|
&& !disableRoPE
|
|
423
455
|
&& !!state.ropeFreqsCos
|
|
424
456
|
&& !!state.ropeFreqsSin,
|
|
@@ -508,7 +540,7 @@ export async function runLayerAttentionGPU(
|
|
|
508
540
|
await debugCheckBuffer(vTensor.buffer, `L${layerIdx} V after proj (GPU)`, numTokens, numKVHeads * headDim);
|
|
509
541
|
}
|
|
510
542
|
|
|
511
|
-
// Optional
|
|
543
|
+
// Optional Q/K normalization; the manifest owns the axis and norm family.
|
|
512
544
|
if (isKernelDebugEnabled(layerIdx)) {
|
|
513
545
|
logKernelStep(
|
|
514
546
|
'qk_norm',
|
|
@@ -533,6 +565,8 @@ export async function runLayerAttentionGPU(
|
|
|
533
565
|
numKVHeads,
|
|
534
566
|
headDim,
|
|
535
567
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
568
|
+
normalizationType: config.queryKeyNormType,
|
|
569
|
+
normalizationAxis: config.queryKeyNormAxis,
|
|
536
570
|
releaseTemporary: (buffer) => releaseBuffer(buffer),
|
|
537
571
|
skipKNorm: qkNormState.skipKNorm,
|
|
538
572
|
onQNormApplied: isKernelDebugEnabled(layerIdx)
|
|
@@ -1229,6 +1263,25 @@ export async function runLayerAttentionGPU(
|
|
|
1229
1263
|
}
|
|
1230
1264
|
}
|
|
1231
1265
|
|
|
1266
|
+
if (layerWeights.oProjBias) {
|
|
1267
|
+
const { tensor: oProjBiasTensor, owned } = getVectorTensor(
|
|
1268
|
+
layerWeights.oProjBias,
|
|
1269
|
+
`L${layerIdx}.o_proj_bias`,
|
|
1270
|
+
hiddenSize,
|
|
1271
|
+
{},
|
|
1272
|
+
debugFlags
|
|
1273
|
+
);
|
|
1274
|
+
try {
|
|
1275
|
+
output = await runBiasAdd(output, oProjBiasTensor, numTokens, hiddenSize, {
|
|
1276
|
+
label: `L${layerIdx}.o_proj_bias`,
|
|
1277
|
+
layerIdx,
|
|
1278
|
+
executionPolicies: state.executionPolicies ?? null,
|
|
1279
|
+
});
|
|
1280
|
+
} finally {
|
|
1281
|
+
if (owned) releaseBuffer(oProjBiasTensor.buffer);
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1232
1285
|
if (oProjInputTemp) {
|
|
1233
1286
|
releaseBuffer(oProjInputTemp.buffer);
|
|
1234
1287
|
}
|
|
@@ -45,28 +45,37 @@ export function releaseOrTrack(recorder, buffer) {
|
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
const qkNormOnesCache = new WeakMap();
|
|
48
|
+
const qkNormZerosCache = new WeakMap();
|
|
48
49
|
|
|
49
|
-
|
|
50
|
-
export function getQKNormOnesBuffer(headDim) {
|
|
50
|
+
function getCachedQKNormConstantBuffer(cache, size, value, label) {
|
|
51
51
|
const device = getDevice();
|
|
52
52
|
if (!device) {
|
|
53
53
|
throw new Error('No GPU device available for Q/K norm buffer');
|
|
54
54
|
}
|
|
55
|
-
let perDeviceCache =
|
|
55
|
+
let perDeviceCache = cache.get(device);
|
|
56
56
|
if (!perDeviceCache) {
|
|
57
57
|
perDeviceCache = new Map();
|
|
58
|
-
|
|
58
|
+
cache.set(device, perDeviceCache);
|
|
59
59
|
}
|
|
60
|
-
const cached = perDeviceCache.get(
|
|
60
|
+
const cached = perDeviceCache.get(size);
|
|
61
61
|
if (cached) return cached;
|
|
62
|
-
const data = new Float32Array(
|
|
63
|
-
data.fill(
|
|
62
|
+
const data = new Float32Array(size);
|
|
63
|
+
data.fill(value);
|
|
64
64
|
const buffer = device.createBuffer({
|
|
65
|
-
label:
|
|
65
|
+
label: `${label}_${size}`,
|
|
66
66
|
size: data.byteLength,
|
|
67
67
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
68
68
|
});
|
|
69
69
|
device.queue.writeBuffer(buffer, 0, data);
|
|
70
|
-
perDeviceCache.set(
|
|
70
|
+
perDeviceCache.set(size, buffer);
|
|
71
71
|
return buffer;
|
|
72
72
|
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
export function getQKNormOnesBuffer(headDim) {
|
|
76
|
+
return getCachedQKNormConstantBuffer(qkNormOnesCache, headDim, 1, 'qk_norm_ones');
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function getQKNormZerosBuffer(size) {
|
|
80
|
+
return getCachedQKNormConstantBuffer(qkNormZerosCache, size, 0, 'qk_norm_zeros');
|
|
81
|
+
}
|