doppler-gpu 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +27 -0
- package/models/catalog.json +612 -7
- package/package.json +13 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +126 -4
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +64 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/kernels/registry.json +52 -9
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/lora-loader.js +15 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +189 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.js +90 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/ops.js +32 -0
- package/src/inference/pipelines/text/types.d.ts +38 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +4 -2
- package/src/inference/pipelines/text.js +112 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/version.js +1 -1
|
@@ -6,6 +6,8 @@ import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
|
6
6
|
import {
|
|
7
7
|
recordMatmul,
|
|
8
8
|
recordRMSNorm,
|
|
9
|
+
recordLayerNorm,
|
|
10
|
+
recordBiasAdd,
|
|
9
11
|
recordRoPE,
|
|
10
12
|
canUseRoPEQK,
|
|
11
13
|
recordRoPEQK,
|
|
@@ -63,6 +65,7 @@ import {
|
|
|
63
65
|
} from './precision-contract.js';
|
|
64
66
|
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
65
67
|
import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
|
|
68
|
+
import { getVectorTensor } from '../weights.js';
|
|
66
69
|
|
|
67
70
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
68
71
|
|
|
@@ -284,6 +287,7 @@ export async function recordLayerAttentionGPU(
|
|
|
284
287
|
sharedKVSourceLayerIdx != null
|
|
285
288
|
);
|
|
286
289
|
const canFuseInputNormProjRec = rmsNormFusionFlagRec
|
|
290
|
+
&& config.normalizationType === 'rmsnorm'
|
|
287
291
|
&& canSelectFusedRmsNormProjectionRec
|
|
288
292
|
&& !skipInputNorm
|
|
289
293
|
&& layerWeights.inputNorm
|
|
@@ -319,12 +323,36 @@ export async function recordLayerAttentionGPU(
|
|
|
319
323
|
// Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
|
|
320
324
|
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
321
325
|
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
326
|
+
let normBiasTensor = null;
|
|
327
|
+
let normBiasOwned = false;
|
|
328
|
+
if (config.normalizationType === 'layernorm') {
|
|
329
|
+
if (!layerWeights.inputNormBias) {
|
|
330
|
+
throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
|
|
331
|
+
}
|
|
332
|
+
({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
|
|
333
|
+
layerWeights.inputNormBias,
|
|
334
|
+
`L${layerIdx}.input_norm_bias`,
|
|
335
|
+
hiddenSize,
|
|
336
|
+
{},
|
|
337
|
+
debugFlags
|
|
338
|
+
));
|
|
339
|
+
normed = await recordLayerNorm(
|
|
340
|
+
recorder,
|
|
341
|
+
attentionInput,
|
|
342
|
+
normWeightBuf,
|
|
343
|
+
normBiasTensor.buffer,
|
|
344
|
+
rmsNormEps,
|
|
345
|
+
{ batchSize: numTokens, hiddenSize, label: 'input_norm' }
|
|
346
|
+
);
|
|
347
|
+
} else {
|
|
348
|
+
normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
|
|
349
|
+
batchSize: numTokens,
|
|
350
|
+
hiddenSize,
|
|
351
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
352
|
+
label: 'input_norm',
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
if (normBiasOwned && normBiasTensor) releaseOrTrack(recorder, normBiasTensor.buffer);
|
|
328
356
|
if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
|
|
329
357
|
await runProbes('post_input_norm', normed.buffer, {
|
|
330
358
|
layerIdx, numTokens, hiddenSize,
|
|
@@ -438,7 +466,10 @@ export async function recordLayerAttentionGPU(
|
|
|
438
466
|
fusedNormEps: fusedNormEpsRec,
|
|
439
467
|
fusedNormOffset: fusedNormOffsetRec,
|
|
440
468
|
qkNormFusion: {
|
|
441
|
-
enabled: qkNormFusionFlag
|
|
469
|
+
enabled: qkNormFusionFlag
|
|
470
|
+
&& qkNormState.wantsQKNorm
|
|
471
|
+
&& config.queryKeyNormType === 'rmsnorm'
|
|
472
|
+
&& config.queryKeyNormAxis === 'head',
|
|
442
473
|
getNormWeightBuffer,
|
|
443
474
|
rmsNormEps,
|
|
444
475
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
@@ -449,6 +480,8 @@ export async function recordLayerAttentionGPU(
|
|
|
449
480
|
qkNormRoPEFusion: {
|
|
450
481
|
enabled: qkNormRoPEFusionFlag
|
|
451
482
|
&& qkNormState.wantsQKNorm
|
|
483
|
+
&& config.queryKeyNormType === 'rmsnorm'
|
|
484
|
+
&& config.queryKeyNormAxis === 'head'
|
|
452
485
|
&& !disableRoPE
|
|
453
486
|
&& !!state.ropeFreqsCos
|
|
454
487
|
&& !!state.ropeFreqsSin,
|
|
@@ -520,8 +553,8 @@ export async function recordLayerAttentionGPU(
|
|
|
520
553
|
);
|
|
521
554
|
}
|
|
522
555
|
|
|
523
|
-
// Optional
|
|
524
|
-
//
|
|
556
|
+
// Optional Q/K normalization. The manifest owns both its numerical family
|
|
557
|
+
// and whether it applies per head or across the complete projection.
|
|
525
558
|
if (qkNormState.wantsQKNorm && !qkNormApplied) {
|
|
526
559
|
({ qTensor, kTensor } = await applyAttentionQKNorm({
|
|
527
560
|
recorder,
|
|
@@ -535,6 +568,8 @@ export async function recordLayerAttentionGPU(
|
|
|
535
568
|
numKVHeads,
|
|
536
569
|
headDim,
|
|
537
570
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
571
|
+
normalizationType: config.queryKeyNormType,
|
|
572
|
+
normalizationAxis: config.queryKeyNormAxis,
|
|
538
573
|
releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
|
|
539
574
|
skipKNorm: qkNormState.skipKNorm,
|
|
540
575
|
retainKInput: valueAliasesKey,
|
|
@@ -1113,6 +1148,29 @@ export async function recordLayerAttentionGPU(
|
|
|
1113
1148
|
}
|
|
1114
1149
|
}
|
|
1115
1150
|
|
|
1151
|
+
if (layerWeights.oProjBias) {
|
|
1152
|
+
const { tensor: oProjBiasTensor, owned } = getVectorTensor(
|
|
1153
|
+
layerWeights.oProjBias,
|
|
1154
|
+
`L${layerIdx}.o_proj_bias`,
|
|
1155
|
+
hiddenSize,
|
|
1156
|
+
{},
|
|
1157
|
+
debugFlags
|
|
1158
|
+
);
|
|
1159
|
+
output = await recordBiasAdd(
|
|
1160
|
+
recorder,
|
|
1161
|
+
output,
|
|
1162
|
+
oProjBiasTensor,
|
|
1163
|
+
numTokens,
|
|
1164
|
+
hiddenSize,
|
|
1165
|
+
{
|
|
1166
|
+
label: `L${layerIdx}.o_proj_bias`,
|
|
1167
|
+
layerIdx,
|
|
1168
|
+
executionPolicies: state.executionPolicies ?? null,
|
|
1169
|
+
}
|
|
1170
|
+
);
|
|
1171
|
+
if (owned) releaseOrTrack(recorder, oProjBiasTensor.buffer);
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1116
1174
|
finalOutput = output;
|
|
1117
1175
|
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
1118
1176
|
enqueueRecordedTensorHealth(
|
|
@@ -7,6 +7,8 @@ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js'
|
|
|
7
7
|
import {
|
|
8
8
|
runMatmul,
|
|
9
9
|
runRMSNorm,
|
|
10
|
+
runLayerNorm,
|
|
11
|
+
runBiasAdd,
|
|
10
12
|
runRoPE,
|
|
11
13
|
canUseRoPEQK,
|
|
12
14
|
runRoPEQK,
|
|
@@ -67,6 +69,7 @@ import {
|
|
|
67
69
|
} from './precision-contract.js';
|
|
68
70
|
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
69
71
|
import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
|
|
72
|
+
import { getVectorTensor } from '../weights.js';
|
|
70
73
|
|
|
71
74
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
72
75
|
|
|
@@ -244,6 +247,7 @@ export async function runLayerAttentionGPU(
|
|
|
244
247
|
const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
|
|
245
248
|
const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
|
|
246
249
|
const canFuseInputNormIntoProjections = rmsNormFusionFlag
|
|
250
|
+
&& config.normalizationType === 'rmsnorm'
|
|
247
251
|
&& hasInputNormWeight
|
|
248
252
|
&& canSelectFusedRmsNormProjection
|
|
249
253
|
&& !skipInputNorm
|
|
@@ -283,18 +287,41 @@ export async function runLayerAttentionGPU(
|
|
|
283
287
|
// norm internally via the fused kernel variant.
|
|
284
288
|
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
285
289
|
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
290
|
+
let normBiasTensor = null;
|
|
291
|
+
let normBiasOwned = false;
|
|
286
292
|
try {
|
|
287
293
|
// Debug: norm weights for configured layers
|
|
288
294
|
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'norm_weights', debugFlags) && debugCheckBuffer) {
|
|
289
295
|
await debugCheckBuffer(normWeightBuf, `L${layerIdx} input norm weights (GPU)`, 1, hiddenSize);
|
|
290
296
|
}
|
|
291
297
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
298
|
+
if (config.normalizationType === 'layernorm') {
|
|
299
|
+
if (!layerWeights.inputNormBias) {
|
|
300
|
+
throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
|
|
301
|
+
}
|
|
302
|
+
({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
|
|
303
|
+
layerWeights.inputNormBias,
|
|
304
|
+
`L${layerIdx}.input_norm_bias`,
|
|
305
|
+
hiddenSize,
|
|
306
|
+
{},
|
|
307
|
+
debugFlags
|
|
308
|
+
));
|
|
309
|
+
normed = await runLayerNorm(
|
|
310
|
+
attentionInput,
|
|
311
|
+
normWeightBuf,
|
|
312
|
+
normBiasTensor.buffer,
|
|
313
|
+
rmsNormEps,
|
|
314
|
+
{ batchSize: numTokens, hiddenSize }
|
|
315
|
+
);
|
|
316
|
+
} else {
|
|
317
|
+
normed = await runRMSNorm(attentionInput, normWeightBuf, rmsNormEps, {
|
|
318
|
+
batchSize: numTokens,
|
|
319
|
+
hiddenSize,
|
|
320
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
321
|
+
});
|
|
322
|
+
}
|
|
297
323
|
} finally {
|
|
324
|
+
if (normBiasOwned && normBiasTensor) releaseBuffer(normBiasTensor.buffer);
|
|
298
325
|
if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseBuffer(normWeightBuf);
|
|
299
326
|
}
|
|
300
327
|
|
|
@@ -408,7 +435,10 @@ export async function runLayerAttentionGPU(
|
|
|
408
435
|
fusedNormEps,
|
|
409
436
|
fusedNormOffset,
|
|
410
437
|
qkNormFusion: {
|
|
411
|
-
enabled: qkNormFusionFlag
|
|
438
|
+
enabled: qkNormFusionFlag
|
|
439
|
+
&& qkNormState.wantsQKNorm
|
|
440
|
+
&& config.queryKeyNormType === 'rmsnorm'
|
|
441
|
+
&& config.queryKeyNormAxis === 'head',
|
|
412
442
|
getNormWeightBuffer,
|
|
413
443
|
rmsNormEps,
|
|
414
444
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
@@ -419,6 +449,8 @@ export async function runLayerAttentionGPU(
|
|
|
419
449
|
qkNormRoPEFusion: {
|
|
420
450
|
enabled: qkNormRoPEFusionFlag
|
|
421
451
|
&& qkNormState.wantsQKNorm
|
|
452
|
+
&& config.queryKeyNormType === 'rmsnorm'
|
|
453
|
+
&& config.queryKeyNormAxis === 'head'
|
|
422
454
|
&& !disableRoPE
|
|
423
455
|
&& !!state.ropeFreqsCos
|
|
424
456
|
&& !!state.ropeFreqsSin,
|
|
@@ -508,7 +540,7 @@ export async function runLayerAttentionGPU(
|
|
|
508
540
|
await debugCheckBuffer(vTensor.buffer, `L${layerIdx} V after proj (GPU)`, numTokens, numKVHeads * headDim);
|
|
509
541
|
}
|
|
510
542
|
|
|
511
|
-
// Optional
|
|
543
|
+
// Optional Q/K normalization; the manifest owns the axis and norm family.
|
|
512
544
|
if (isKernelDebugEnabled(layerIdx)) {
|
|
513
545
|
logKernelStep(
|
|
514
546
|
'qk_norm',
|
|
@@ -533,6 +565,8 @@ export async function runLayerAttentionGPU(
|
|
|
533
565
|
numKVHeads,
|
|
534
566
|
headDim,
|
|
535
567
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
568
|
+
normalizationType: config.queryKeyNormType,
|
|
569
|
+
normalizationAxis: config.queryKeyNormAxis,
|
|
536
570
|
releaseTemporary: (buffer) => releaseBuffer(buffer),
|
|
537
571
|
skipKNorm: qkNormState.skipKNorm,
|
|
538
572
|
onQNormApplied: isKernelDebugEnabled(layerIdx)
|
|
@@ -1229,6 +1263,25 @@ export async function runLayerAttentionGPU(
|
|
|
1229
1263
|
}
|
|
1230
1264
|
}
|
|
1231
1265
|
|
|
1266
|
+
if (layerWeights.oProjBias) {
|
|
1267
|
+
const { tensor: oProjBiasTensor, owned } = getVectorTensor(
|
|
1268
|
+
layerWeights.oProjBias,
|
|
1269
|
+
`L${layerIdx}.o_proj_bias`,
|
|
1270
|
+
hiddenSize,
|
|
1271
|
+
{},
|
|
1272
|
+
debugFlags
|
|
1273
|
+
);
|
|
1274
|
+
try {
|
|
1275
|
+
output = await runBiasAdd(output, oProjBiasTensor, numTokens, hiddenSize, {
|
|
1276
|
+
label: `L${layerIdx}.o_proj_bias`,
|
|
1277
|
+
layerIdx,
|
|
1278
|
+
executionPolicies: state.executionPolicies ?? null,
|
|
1279
|
+
});
|
|
1280
|
+
} finally {
|
|
1281
|
+
if (owned) releaseBuffer(oProjBiasTensor.buffer);
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1232
1285
|
if (oProjInputTemp) {
|
|
1233
1286
|
releaseBuffer(oProjInputTemp.buffer);
|
|
1234
1287
|
}
|
|
@@ -45,28 +45,37 @@ export function releaseOrTrack(recorder, buffer) {
|
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
const qkNormOnesCache = new WeakMap();
|
|
48
|
+
const qkNormZerosCache = new WeakMap();
|
|
48
49
|
|
|
49
|
-
|
|
50
|
-
export function getQKNormOnesBuffer(headDim) {
|
|
50
|
+
function getCachedQKNormConstantBuffer(cache, size, value, label) {
|
|
51
51
|
const device = getDevice();
|
|
52
52
|
if (!device) {
|
|
53
53
|
throw new Error('No GPU device available for Q/K norm buffer');
|
|
54
54
|
}
|
|
55
|
-
let perDeviceCache =
|
|
55
|
+
let perDeviceCache = cache.get(device);
|
|
56
56
|
if (!perDeviceCache) {
|
|
57
57
|
perDeviceCache = new Map();
|
|
58
|
-
|
|
58
|
+
cache.set(device, perDeviceCache);
|
|
59
59
|
}
|
|
60
|
-
const cached = perDeviceCache.get(
|
|
60
|
+
const cached = perDeviceCache.get(size);
|
|
61
61
|
if (cached) return cached;
|
|
62
|
-
const data = new Float32Array(
|
|
63
|
-
data.fill(
|
|
62
|
+
const data = new Float32Array(size);
|
|
63
|
+
data.fill(value);
|
|
64
64
|
const buffer = device.createBuffer({
|
|
65
|
-
label:
|
|
65
|
+
label: `${label}_${size}`,
|
|
66
66
|
size: data.byteLength,
|
|
67
67
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
68
68
|
});
|
|
69
69
|
device.queue.writeBuffer(buffer, 0, data);
|
|
70
|
-
perDeviceCache.set(
|
|
70
|
+
perDeviceCache.set(size, buffer);
|
|
71
71
|
return buffer;
|
|
72
72
|
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
export function getQKNormOnesBuffer(headDim) {
|
|
76
|
+
return getCachedQKNormConstantBuffer(qkNormOnesCache, headDim, 1, 'qk_norm_ones');
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function getQKNormZerosBuffer(size) {
|
|
80
|
+
return getCachedQKNormConstantBuffer(qkNormZerosCache, size, 0, 'qk_norm_zeros');
|
|
81
|
+
}
|
|
@@ -186,6 +186,8 @@ export interface ParsedModelConfig {
|
|
|
186
186
|
ropeLocalScaling: RopeScalingConfig | null;
|
|
187
187
|
quantization: string;
|
|
188
188
|
quantMethod: string | null;
|
|
189
|
+
normalizationType: 'rmsnorm' | 'layernorm';
|
|
190
|
+
finalNormBiasTensor: string | null;
|
|
189
191
|
rmsNormEps: number;
|
|
190
192
|
rmsNormWeightOffset: boolean;
|
|
191
193
|
postAttentionNorm: boolean;
|
|
@@ -200,6 +202,7 @@ export interface ParsedModelConfig {
|
|
|
200
202
|
embeddingVocabSize: number | null;
|
|
201
203
|
embeddingPostprocessor: ManifestEmbeddingPostprocessorSchema | null;
|
|
202
204
|
hiddenActivation: ActivationType;
|
|
205
|
+
gatedActivation: boolean;
|
|
203
206
|
ffnBranchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
|
|
204
207
|
useDoubleWideMlp: boolean;
|
|
205
208
|
swigluLimit: number | null;
|
|
@@ -216,6 +219,8 @@ export interface ParsedModelConfig {
|
|
|
216
219
|
finalLogitSoftcapping: number | null;
|
|
217
220
|
attnLogitSoftcapping: number | null;
|
|
218
221
|
queryKeyNorm: boolean;
|
|
222
|
+
queryKeyNormType: 'rmsnorm' | 'layernorm';
|
|
223
|
+
queryKeyNormAxis: 'head' | 'projection';
|
|
219
224
|
queryKeyNormLayers: number[] | null;
|
|
220
225
|
queryKeyNormWeightLayers: number[] | null;
|
|
221
226
|
valueNorm: boolean;
|
|
@@ -458,6 +458,9 @@ export function getStopTokenIds(manifest) {
|
|
|
458
458
|
if (typeof eosTokenId === 'number') return [eosTokenId];
|
|
459
459
|
const modelId = manifest?.modelId ?? 'unknown';
|
|
460
460
|
if (eosTokenId == null) {
|
|
461
|
+
if (manifest?.modelType === 'embedding') {
|
|
462
|
+
return [];
|
|
463
|
+
}
|
|
461
464
|
throw new Error(
|
|
462
465
|
`Manifest "${modelId}" is missing eos_token_id. Re-convert the model with tokenizer metadata.`
|
|
463
466
|
);
|
|
@@ -708,6 +711,18 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
708
711
|
if (inf.attention.queryKeyNorm == null) {
|
|
709
712
|
errors.push('attention.queryKeyNorm is required');
|
|
710
713
|
}
|
|
714
|
+
if (
|
|
715
|
+
inf.attention.queryKeyNormType !== undefined
|
|
716
|
+
&& !['rmsnorm', 'layernorm'].includes(inf.attention.queryKeyNormType)
|
|
717
|
+
) {
|
|
718
|
+
errors.push('attention.queryKeyNormType must be "rmsnorm" or "layernorm"');
|
|
719
|
+
}
|
|
720
|
+
if (
|
|
721
|
+
inf.attention.queryKeyNormAxis !== undefined
|
|
722
|
+
&& !['head', 'projection'].includes(inf.attention.queryKeyNormAxis)
|
|
723
|
+
) {
|
|
724
|
+
errors.push('attention.queryKeyNormAxis must be "head" or "projection"');
|
|
725
|
+
}
|
|
711
726
|
if (
|
|
712
727
|
inf.attention.queryKeyNormLayers !== undefined
|
|
713
728
|
&& inf.attention.queryKeyNormLayers !== null
|
|
@@ -756,6 +771,12 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
756
771
|
}
|
|
757
772
|
|
|
758
773
|
// Normalization fields
|
|
774
|
+
if (
|
|
775
|
+
inf.normalization.type !== undefined
|
|
776
|
+
&& !['rmsnorm', 'layernorm'].includes(inf.normalization.type)
|
|
777
|
+
) {
|
|
778
|
+
errors.push('normalization.type must be rmsnorm or layernorm');
|
|
779
|
+
}
|
|
759
780
|
if (inf.normalization.rmsNormWeightOffset == null) {
|
|
760
781
|
errors.push('normalization.rmsNormWeightOffset is required');
|
|
761
782
|
}
|
|
@@ -771,6 +792,23 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
771
792
|
if (inf.normalization.postFeedforwardNorm == null) {
|
|
772
793
|
errors.push('normalization.postFeedforwardNorm is required');
|
|
773
794
|
}
|
|
795
|
+
if (
|
|
796
|
+
inf.normalization.finalNormBiasTensor !== undefined
|
|
797
|
+
&&
|
|
798
|
+
inf.normalization.finalNormBiasTensor !== null
|
|
799
|
+
&& (
|
|
800
|
+
typeof inf.normalization.finalNormBiasTensor !== 'string'
|
|
801
|
+
|| inf.normalization.finalNormBiasTensor.trim() === ''
|
|
802
|
+
)
|
|
803
|
+
) {
|
|
804
|
+
errors.push('normalization.finalNormBiasTensor must be null or a non-empty string');
|
|
805
|
+
}
|
|
806
|
+
if (
|
|
807
|
+
(inf.normalization.type ?? 'rmsnorm') === 'rmsnorm'
|
|
808
|
+
&& (inf.normalization.finalNormBiasTensor ?? null) !== null
|
|
809
|
+
) {
|
|
810
|
+
errors.push('normalization.finalNormBiasTensor requires normalization.type="layernorm"');
|
|
811
|
+
}
|
|
774
812
|
|
|
775
813
|
// FFN fields
|
|
776
814
|
if (inf.ffn.activation == null) {
|
|
@@ -944,6 +982,54 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
944
982
|
}
|
|
945
983
|
}
|
|
946
984
|
}
|
|
985
|
+
if (
|
|
986
|
+
inf.output.lmHeadBiasTensor !== undefined
|
|
987
|
+
&& inf.output.lmHeadBiasTensor !== null
|
|
988
|
+
&& (typeof inf.output.lmHeadBiasTensor !== 'string' || inf.output.lmHeadBiasTensor.trim() === '')
|
|
989
|
+
) {
|
|
990
|
+
errors.push('output.lmHeadBiasTensor must be null or a non-empty tensor name');
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
if (inf.supportsSequence !== undefined && typeof inf.supportsSequence !== 'boolean') {
|
|
994
|
+
errors.push('supportsSequence must be boolean when provided');
|
|
995
|
+
}
|
|
996
|
+
if (inf.sequence !== undefined && inf.sequence !== null) {
|
|
997
|
+
const sequence = inf.sequence;
|
|
998
|
+
if (!sequence || typeof sequence !== 'object' || Array.isArray(sequence)) {
|
|
999
|
+
errors.push('sequence must be null or an object');
|
|
1000
|
+
} else {
|
|
1001
|
+
if (sequence.alphabet !== 'amino_acid' && sequence.alphabet !== 'nucleotide') {
|
|
1002
|
+
errors.push('sequence.alphabet must be "amino_acid" or "nucleotide"');
|
|
1003
|
+
}
|
|
1004
|
+
if (typeof sequence.tokenEmbeddings !== 'boolean') {
|
|
1005
|
+
errors.push('sequence.tokenEmbeddings must be boolean');
|
|
1006
|
+
}
|
|
1007
|
+
if (typeof sequence.logits !== 'boolean') {
|
|
1008
|
+
errors.push('sequence.logits must be boolean');
|
|
1009
|
+
}
|
|
1010
|
+
if (sequence.pooledEmbedding !== null) {
|
|
1011
|
+
const pooling = sequence.pooledEmbedding;
|
|
1012
|
+
if (!pooling || typeof pooling !== 'object' || Array.isArray(pooling)) {
|
|
1013
|
+
errors.push('sequence.pooledEmbedding must be null or an object');
|
|
1014
|
+
} else {
|
|
1015
|
+
if (pooling.mode !== 'mean' && pooling.mode !== 'last') {
|
|
1016
|
+
errors.push('sequence.pooledEmbedding.mode must be "mean" or "last"');
|
|
1017
|
+
}
|
|
1018
|
+
if (!Array.isArray(pooling.excludeTokenIds) || pooling.excludeTokenIds.some(
|
|
1019
|
+
(tokenId) => !Number.isInteger(tokenId) || tokenId < 0
|
|
1020
|
+
)) {
|
|
1021
|
+
errors.push('sequence.pooledEmbedding.excludeTokenIds must be an array of non-negative integers');
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
if (inf.supportsSequence === true && (inf.sequence == null || typeof inf.sequence !== 'object')) {
|
|
1028
|
+
errors.push('sequence is required when supportsSequence=true');
|
|
1029
|
+
}
|
|
1030
|
+
if (inf.supportsSequence !== true && inf.sequence != null) {
|
|
1031
|
+
errors.push('supportsSequence must be true when sequence is configured');
|
|
1032
|
+
}
|
|
947
1033
|
|
|
948
1034
|
// Layer pattern fields
|
|
949
1035
|
if (inf.layerPattern?.type == null) {
|
|
@@ -1843,6 +1929,8 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1843
1929
|
ropeLocalScaling,
|
|
1844
1930
|
quantization: manifest.quantization,
|
|
1845
1931
|
quantMethod: config.quantization_config?.quant_method ?? null,
|
|
1932
|
+
normalizationType: inf.normalization.type ?? 'rmsnorm',
|
|
1933
|
+
finalNormBiasTensor: inf.normalization.finalNormBiasTensor ?? null,
|
|
1846
1934
|
rmsNormEps: inf.normalization.rmsNormEps,
|
|
1847
1935
|
rmsNormWeightOffset: inf.normalization.rmsNormWeightOffset,
|
|
1848
1936
|
postAttentionNorm: inf.normalization.postAttentionNorm,
|
|
@@ -1857,6 +1945,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1857
1945
|
embeddingVocabSize: inf.output.embeddingVocabSize,
|
|
1858
1946
|
embeddingPostprocessor,
|
|
1859
1947
|
hiddenActivation,
|
|
1948
|
+
gatedActivation: inf.ffn.gatedActivation,
|
|
1860
1949
|
ffnBranchMode: inf.ffn.branchMode,
|
|
1861
1950
|
useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
|
|
1862
1951
|
swigluLimit: inf.ffn.swigluLimit,
|
|
@@ -1873,6 +1962,8 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1873
1962
|
finalLogitSoftcapping: inf.output.finalLogitSoftcapping,
|
|
1874
1963
|
attnLogitSoftcapping: inf.attention.attnLogitSoftcapping,
|
|
1875
1964
|
queryKeyNorm: inf.attention.queryKeyNorm,
|
|
1965
|
+
queryKeyNormType: inf.attention.queryKeyNormType ?? 'rmsnorm',
|
|
1966
|
+
queryKeyNormAxis: inf.attention.queryKeyNormAxis ?? 'head',
|
|
1876
1967
|
queryKeyNormLayers: inf.attention.queryKeyNormLayers ?? null,
|
|
1877
1968
|
queryKeyNormWeightLayers: inf.attention.queryKeyNormWeightLayers ?? null,
|
|
1878
1969
|
valueNorm: inf.attention.valueNorm,
|