doppler-gpu 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +27 -0
- package/models/catalog.json +612 -7
- package/package.json +13 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +126 -4
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +64 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/kernels/registry.json +52 -9
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/lora-loader.js +15 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +189 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.js +90 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/ops.js +32 -0
- package/src/inference/pipelines/text/types.d.ts +38 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +4 -2
- package/src/inference/pipelines/text.js +112 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/version.js +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
|
-
doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
|
|
4
|
+
doBiasAdd, doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
|
|
5
5
|
releaseOrTrack
|
|
6
6
|
} from '../ops.js';
|
|
7
7
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
@@ -29,7 +29,7 @@ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
|
|
|
29
29
|
import { isKernelDebugEnabled, dumpTokenVector, decodeReadback, getLogitsHealth, shouldDebugLayerOutput } from '../debug-utils.js';
|
|
30
30
|
import { applyLoRA } from '../lora-apply.js';
|
|
31
31
|
import { getLoRAModule } from '../lora.js';
|
|
32
|
-
import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
|
|
32
|
+
import { getWeightBuffer, getNormWeightBuffer, getVectorTensor } from '../weights.js';
|
|
33
33
|
import { runProbes } from '../probes.js';
|
|
34
34
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
35
35
|
import {
|
|
@@ -527,6 +527,189 @@ async function dispatchFusedGateUp({
|
|
|
527
527
|
return fusedOutput;
|
|
528
528
|
}
|
|
529
529
|
|
|
530
|
+
async function applyDenseProjectionBias(
|
|
531
|
+
tensor,
|
|
532
|
+
biasWeight,
|
|
533
|
+
label,
|
|
534
|
+
numTokens,
|
|
535
|
+
dim,
|
|
536
|
+
context,
|
|
537
|
+
layerIdx
|
|
538
|
+
) {
|
|
539
|
+
if (!biasWeight) return tensor;
|
|
540
|
+
const { tensor: biasTensor, owned } = getVectorTensor(
|
|
541
|
+
biasWeight,
|
|
542
|
+
label,
|
|
543
|
+
dim,
|
|
544
|
+
context.weightConfig,
|
|
545
|
+
context.debugFlags
|
|
546
|
+
);
|
|
547
|
+
try {
|
|
548
|
+
return await doBiasAdd(tensor, biasTensor, numTokens, dim, {
|
|
549
|
+
label,
|
|
550
|
+
layerIdx,
|
|
551
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
552
|
+
}, context.recorder);
|
|
553
|
+
} finally {
|
|
554
|
+
if (owned) releaseOrTrack(context.recorder, biasTensor.buffer, context.decodeBuffers);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
async function runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, context, layerWeights) {
|
|
559
|
+
const { config, recorder } = context;
|
|
560
|
+
const { hiddenSize, hiddenActivation } = config;
|
|
561
|
+
const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
|
|
562
|
+
if (layerWeights.gate || layerWeights.gateUp) {
|
|
563
|
+
throw new Error(`Layer ${layerIdx} declares an ungated FFN but contains gate weights.`);
|
|
564
|
+
}
|
|
565
|
+
if (!layerWeights.up || !layerWeights.down) {
|
|
566
|
+
throw new Error(`Layer ${layerIdx} ungated FFN requires up_proj and down_proj weights.`);
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
const kernelPath = context.kernelPath ?? null;
|
|
570
|
+
const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
|
|
571
|
+
const ffnStepPrecision = context.ffnStepPrecision ?? null;
|
|
572
|
+
const upInputDtype = resolveMatmulStepDtype(
|
|
573
|
+
'ffn_up', phase, layerIdx, kernelPath, inputTensor.dtype, 'inputDtype', ffnStepPrecision
|
|
574
|
+
);
|
|
575
|
+
const upOutputDtype = resolveMatmulStepDtype(
|
|
576
|
+
'ffn_up', phase, layerIdx, kernelPath, inputTensor.dtype, 'outputDtype', ffnStepPrecision
|
|
577
|
+
);
|
|
578
|
+
const downInputDtype = resolveMatmulStepDtype(
|
|
579
|
+
'ffn_down', phase, layerIdx, kernelPath, upOutputDtype, 'inputDtype', ffnStepPrecision
|
|
580
|
+
);
|
|
581
|
+
const downOutputDtype = resolveMatmulStepDtype(
|
|
582
|
+
'ffn_down', phase, layerIdx, kernelPath, inputTensor.dtype, 'outputDtype', ffnStepPrecision
|
|
583
|
+
);
|
|
584
|
+
|
|
585
|
+
let upInput = inputTensor;
|
|
586
|
+
let upInputOwned = false;
|
|
587
|
+
if (upInputDtype && upInputDtype !== inputTensor.dtype) {
|
|
588
|
+
upInput = await coerceTensorDtype(inputTensor, upInputDtype, recorder, {
|
|
589
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
590
|
+
op: 'ffn_up_input',
|
|
591
|
+
transitionDeclaredBy: 'step_precision',
|
|
592
|
+
});
|
|
593
|
+
upInputOwned = upInput !== inputTensor;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
const upWeight = getWeightBuffer(layerWeights.up, 'ffn_up');
|
|
597
|
+
let upOutput = await doMatmul(
|
|
598
|
+
upInput,
|
|
599
|
+
upWeight,
|
|
600
|
+
numTokens,
|
|
601
|
+
intermediateSize,
|
|
602
|
+
hiddenSize,
|
|
603
|
+
{
|
|
604
|
+
transposeB: 'auto',
|
|
605
|
+
label: `L${layerIdx}.ffn_up`,
|
|
606
|
+
layerIdx,
|
|
607
|
+
kernelPath,
|
|
608
|
+
outputDtype: upOutputDtype,
|
|
609
|
+
role: 'ffn_up',
|
|
610
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
611
|
+
},
|
|
612
|
+
recorder
|
|
613
|
+
);
|
|
614
|
+
const loraUp = getLoRAModule(context.lora ?? null, layerIdx, 'up_proj');
|
|
615
|
+
if (loraUp) {
|
|
616
|
+
const combined = await applyLoRA(
|
|
617
|
+
upInput,
|
|
618
|
+
upOutput,
|
|
619
|
+
loraUp,
|
|
620
|
+
{ M: numTokens, N: intermediateSize, K: hiddenSize },
|
|
621
|
+
getWeightBuffer,
|
|
622
|
+
recorder,
|
|
623
|
+
{ kernelPath }
|
|
624
|
+
);
|
|
625
|
+
if (combined.buffer !== upOutput.buffer) {
|
|
626
|
+
releaseOrTrack(recorder, upOutput.buffer, context.decodeBuffers);
|
|
627
|
+
upOutput = combined;
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
upOutput = await applyDenseProjectionBias(
|
|
631
|
+
upOutput,
|
|
632
|
+
layerWeights.upBias,
|
|
633
|
+
`L${layerIdx}.ffn_up_bias`,
|
|
634
|
+
numTokens,
|
|
635
|
+
intermediateSize,
|
|
636
|
+
context,
|
|
637
|
+
layerIdx
|
|
638
|
+
);
|
|
639
|
+
const activated = await dispatchActivation(hiddenActivation, upOutput, {
|
|
640
|
+
size: numTokens * intermediateSize,
|
|
641
|
+
gate: null,
|
|
642
|
+
label: `L${layerIdx}.ffn_activation`,
|
|
643
|
+
layerIdx,
|
|
644
|
+
}, recorder);
|
|
645
|
+
|
|
646
|
+
let downInput = activated;
|
|
647
|
+
let downInputOwned = false;
|
|
648
|
+
if (downInputDtype && activated.dtype !== downInputDtype) {
|
|
649
|
+
downInput = await coerceTensorDtype(activated, downInputDtype, recorder, {
|
|
650
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
651
|
+
op: 'ffn_down_input',
|
|
652
|
+
transitionDeclaredBy: 'step_precision',
|
|
653
|
+
});
|
|
654
|
+
downInputOwned = downInput !== activated;
|
|
655
|
+
}
|
|
656
|
+
const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
|
|
657
|
+
let output = await doMatmul(
|
|
658
|
+
downInput,
|
|
659
|
+
downWeight,
|
|
660
|
+
numTokens,
|
|
661
|
+
hiddenSize,
|
|
662
|
+
intermediateSize,
|
|
663
|
+
{
|
|
664
|
+
transposeB: 'auto',
|
|
665
|
+
label: `L${layerIdx}.ffn_down`,
|
|
666
|
+
layerIdx,
|
|
667
|
+
kernelPath,
|
|
668
|
+
outputDtype: downOutputDtype,
|
|
669
|
+
role: 'ffn_down',
|
|
670
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
671
|
+
},
|
|
672
|
+
recorder
|
|
673
|
+
);
|
|
674
|
+
const loraDown = getLoRAModule(context.lora ?? null, layerIdx, 'down_proj');
|
|
675
|
+
if (loraDown) {
|
|
676
|
+
const combined = await applyLoRA(
|
|
677
|
+
downInput,
|
|
678
|
+
output,
|
|
679
|
+
loraDown,
|
|
680
|
+
{ M: numTokens, N: hiddenSize, K: intermediateSize },
|
|
681
|
+
getWeightBuffer,
|
|
682
|
+
recorder,
|
|
683
|
+
{ kernelPath }
|
|
684
|
+
);
|
|
685
|
+
if (combined.buffer !== output.buffer) {
|
|
686
|
+
releaseOrTrack(recorder, output.buffer, context.decodeBuffers);
|
|
687
|
+
output = combined;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
output = await applyDenseProjectionBias(
|
|
691
|
+
output,
|
|
692
|
+
layerWeights.downBias,
|
|
693
|
+
`L${layerIdx}.ffn_down_bias`,
|
|
694
|
+
numTokens,
|
|
695
|
+
hiddenSize,
|
|
696
|
+
context,
|
|
697
|
+
layerIdx
|
|
698
|
+
);
|
|
699
|
+
|
|
700
|
+
if (!isGpuBufferInstance(layerWeights.up) && !isWeightBuffer(layerWeights.up)) {
|
|
701
|
+
releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight, context.decodeBuffers);
|
|
702
|
+
}
|
|
703
|
+
if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
|
|
704
|
+
releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight, context.decodeBuffers);
|
|
705
|
+
}
|
|
706
|
+
if (upInputOwned) releaseOrTrack(recorder, upInput.buffer, context.decodeBuffers);
|
|
707
|
+
if (downInputOwned) releaseOrTrack(recorder, downInput.buffer, context.decodeBuffers);
|
|
708
|
+
releaseOrTrack(recorder, upOutput.buffer, context.decodeBuffers);
|
|
709
|
+
releaseOrTrack(recorder, activated.buffer, context.decodeBuffers);
|
|
710
|
+
return output;
|
|
711
|
+
}
|
|
712
|
+
|
|
530
713
|
|
|
531
714
|
export async function runDenseFFNGPU(
|
|
532
715
|
layerIdx,
|
|
@@ -548,6 +731,10 @@ export async function runDenseFFNGPU(
|
|
|
548
731
|
const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
|
|
549
732
|
const gateUpPathMode = resolveGateUpPathMode({ kernelPath, phase, layerIdx });
|
|
550
733
|
|
|
734
|
+
if (config.gatedActivation === false) {
|
|
735
|
+
return runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, context, layerWeights);
|
|
736
|
+
}
|
|
737
|
+
|
|
551
738
|
if (layerWeights?.gateUp && layerWeights?.down) {
|
|
552
739
|
const gateUpWeight = getWeightBuffer(layerWeights.gateUp, 'ffn_gate_up');
|
|
553
740
|
const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
|
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import {
|
|
4
|
+
doCast,
|
|
5
|
+
doLayerNorm,
|
|
6
|
+
doRMSNorm,
|
|
7
|
+
doRMSNormStats,
|
|
8
|
+
doResidualAdd,
|
|
9
|
+
doResidualNextRMSNormPair,
|
|
10
|
+
releaseOrTrack,
|
|
11
|
+
} from '../ops.js';
|
|
12
|
+
import { getNormWeightBuffer, getVectorTensor } from '../weights.js';
|
|
5
13
|
import { runProbes } from '../probes.js';
|
|
6
14
|
import { isMoELayerLocal } from './types.js';
|
|
7
15
|
import { runDenseFFNGPU } from './dense.js';
|
|
@@ -15,6 +23,7 @@ import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
|
15
23
|
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
16
24
|
|
|
17
25
|
function shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context }) {
|
|
26
|
+
if (context.config.normalizationType !== 'rmsnorm') return false;
|
|
18
27
|
if (session?.usePostAttnNormFusedGateUp !== true) return false;
|
|
19
28
|
if (numTokens !== 1) return false;
|
|
20
29
|
if (postAttn?.dtype !== 'f32' || fusedResidualInput?.dtype !== 'f32') return false;
|
|
@@ -92,7 +101,44 @@ export async function processFFNStandard(
|
|
|
92
101
|
releasePostAttnNormWeight = !isGpuBufferInstance(layerWeights.postAttnNorm) && !isWeightBuffer(layerWeights.postAttnNorm);
|
|
93
102
|
const session = getRuntimeConfig().inference?.session;
|
|
94
103
|
|
|
95
|
-
if (
|
|
104
|
+
if (config.normalizationType === 'layernorm') {
|
|
105
|
+
if (!layerWeights.postAttentionNormBias) {
|
|
106
|
+
throw new Error(`Layer ${layerIdx} requires post_attention_layernorm.bias for LayerNorm execution.`);
|
|
107
|
+
}
|
|
108
|
+
let normInput = postAttn;
|
|
109
|
+
if (fusedResidualInput) {
|
|
110
|
+
normInput = await doResidualAdd(postAttn, fusedResidualInput, size, recorder, {
|
|
111
|
+
label: `L${layerIdx}.attention_residual`,
|
|
112
|
+
layerIdx,
|
|
113
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
114
|
+
});
|
|
115
|
+
prenormSumBuffer = normInput.buffer;
|
|
116
|
+
}
|
|
117
|
+
const { tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
|
|
118
|
+
layerWeights.postAttentionNormBias,
|
|
119
|
+
`L${layerIdx}.post_attn_norm_bias`,
|
|
120
|
+
hiddenSize,
|
|
121
|
+
weightConfig,
|
|
122
|
+
debugFlags
|
|
123
|
+
);
|
|
124
|
+
try {
|
|
125
|
+
normedTensor = await doLayerNorm(
|
|
126
|
+
normInput,
|
|
127
|
+
normWeightBuf,
|
|
128
|
+
normBiasTensor.buffer,
|
|
129
|
+
rmsNormEps,
|
|
130
|
+
{
|
|
131
|
+
batchSize: numTokens,
|
|
132
|
+
hiddenSize,
|
|
133
|
+
label: `L${layerIdx}.post_attn_norm`,
|
|
134
|
+
layerIdx,
|
|
135
|
+
},
|
|
136
|
+
recorder
|
|
137
|
+
);
|
|
138
|
+
} finally {
|
|
139
|
+
if (normBiasOwned) releaseOrTrack(recorder, normBiasTensor.buffer, decodeBuffers);
|
|
140
|
+
}
|
|
141
|
+
} else if (fusedResidualInput) {
|
|
96
142
|
const bytesPerElement = postAttn.dtype === 'f16' ? 2 : 4;
|
|
97
143
|
prenormSumBuffer = acquireBuffer(size * bytesPerElement, undefined, 'fused_prenorm_sum');
|
|
98
144
|
if (shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context })) {
|
|
@@ -226,6 +272,7 @@ export async function processFFNStandard(
|
|
|
226
272
|
: 1;
|
|
227
273
|
if (
|
|
228
274
|
nextInputNormFusion
|
|
275
|
+
&& config.normalizationType === 'rmsnorm'
|
|
229
276
|
&& residualInput.dtype === 'f32'
|
|
230
277
|
&& residualTensor.dtype === 'f32'
|
|
231
278
|
) {
|
|
@@ -204,13 +204,21 @@ function getWeightBufferConfig(state) {
|
|
|
204
204
|
export function getLogitsWeights(state) {
|
|
205
205
|
const finalNorm = state.weights.get('final_norm');
|
|
206
206
|
const lmHead = state.weights.get('lm_head');
|
|
207
|
+
const lmHeadBias = state.weights.get('lm_head_bias') ?? null;
|
|
208
|
+
const finalNormBias = state.weights.get('final_norm_bias') ?? null;
|
|
207
209
|
if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array || isWeightBuffer(finalNorm))) {
|
|
208
210
|
throw new Error('Final norm not found or invalid type');
|
|
209
211
|
}
|
|
210
212
|
if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
|
|
211
213
|
throw new Error('LM head not found or invalid type');
|
|
212
214
|
}
|
|
213
|
-
|
|
215
|
+
if (lmHeadBias !== null && !(lmHeadBias instanceof Float32Array)) {
|
|
216
|
+
throw new Error('LM head bias is not a Float32Array');
|
|
217
|
+
}
|
|
218
|
+
if (finalNormBias !== null && !(finalNormBias instanceof Float32Array)) {
|
|
219
|
+
throw new Error('Final norm bias is not a Float32Array');
|
|
220
|
+
}
|
|
221
|
+
return { finalNorm, finalNormBias, lmHead, lmHeadBias };
|
|
214
222
|
}
|
|
215
223
|
|
|
216
224
|
|
|
@@ -222,6 +230,8 @@ export function getLogitsConfig(state) {
|
|
|
222
230
|
return {
|
|
223
231
|
hiddenSize: config.hiddenSize,
|
|
224
232
|
vocabSize: config.vocabSize,
|
|
233
|
+
normalizationType: config.normalizationType,
|
|
234
|
+
finalNormBiasTensor: config.finalNormBiasTensor,
|
|
225
235
|
rmsNormEps: config.rmsNormEps,
|
|
226
236
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
227
237
|
useTiedEmbeddings: state.useTiedEmbeddings,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readBuffer } from '../../../memory/buffer-pool.js';
|
|
2
|
-
import { matmulCPU, rmsNormCPU } from './logits/index.js';
|
|
2
|
+
import { layerNormCPU, matmulCPU, rmsNormCPU } from './logits/index.js';
|
|
3
3
|
import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer, getBufferDtype } from '../../../gpu/weight-buffer.js';
|
|
4
4
|
import { decodeReadback } from './debug-utils/index.js';
|
|
5
5
|
import { resolveExecutionSessionPlan } from './execution-plan.js';
|
|
@@ -454,14 +454,13 @@ export async function getFinalNormWeights(state) {
|
|
|
454
454
|
return weights;
|
|
455
455
|
}
|
|
456
456
|
|
|
457
|
-
export function
|
|
457
|
+
export function extractTokenEmbeddingsFromHidden(
|
|
458
458
|
hiddenStates,
|
|
459
459
|
numTokens,
|
|
460
460
|
hiddenSize,
|
|
461
|
-
embeddingMode,
|
|
462
461
|
finalNormWeights,
|
|
463
462
|
config,
|
|
464
|
-
|
|
463
|
+
finalNormBias = null
|
|
465
464
|
) {
|
|
466
465
|
const expectedLength = numTokens * hiddenSize;
|
|
467
466
|
if (hiddenStates.length !== expectedLength) {
|
|
@@ -470,16 +469,64 @@ export function extractEmbeddingFromHidden(
|
|
|
470
469
|
);
|
|
471
470
|
}
|
|
472
471
|
|
|
473
|
-
const
|
|
472
|
+
const tokenEmbeddings = new Float32Array(expectedLength);
|
|
473
|
+
let effectiveLayerNormBias = null;
|
|
474
|
+
if (config.normalizationType === 'layernorm') {
|
|
475
|
+
if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
|
|
476
|
+
throw new Error(
|
|
477
|
+
`[Pipeline] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
effectiveLayerNormBias = finalNormBias instanceof Float32Array
|
|
481
|
+
? finalNormBias
|
|
482
|
+
: new Float32Array(hiddenSize);
|
|
483
|
+
if (effectiveLayerNormBias.length !== hiddenSize) {
|
|
484
|
+
throw new Error(
|
|
485
|
+
`[Pipeline] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveLayerNormBias.length}.`
|
|
486
|
+
);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
for (let tokenIndex = 0; tokenIndex < numTokens; tokenIndex += 1) {
|
|
474
490
|
const offset = tokenIndex * hiddenSize;
|
|
475
491
|
const tokenHidden = hiddenStates.subarray(offset, offset + hiddenSize);
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
492
|
+
if (config.normalizationType === 'layernorm') {
|
|
493
|
+
tokenEmbeddings.set(layerNormCPU(
|
|
494
|
+
tokenHidden,
|
|
495
|
+
finalNormWeights,
|
|
496
|
+
effectiveLayerNormBias,
|
|
497
|
+
config.rmsNormEps
|
|
498
|
+
), offset);
|
|
499
|
+
} else {
|
|
500
|
+
tokenEmbeddings.set(rmsNormCPU(
|
|
501
|
+
tokenHidden,
|
|
502
|
+
finalNormWeights,
|
|
503
|
+
config.rmsNormEps,
|
|
504
|
+
config.rmsNormWeightOffset
|
|
505
|
+
), offset);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
return tokenEmbeddings;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
export function extractEmbeddingFromHidden(
|
|
512
|
+
hiddenStates,
|
|
513
|
+
numTokens,
|
|
514
|
+
hiddenSize,
|
|
515
|
+
embeddingMode,
|
|
516
|
+
finalNormWeights,
|
|
517
|
+
config,
|
|
518
|
+
embeddingPostprocessor = null,
|
|
519
|
+
normalizedTokenEmbeddings = null,
|
|
520
|
+
finalNormBias = null
|
|
521
|
+
) {
|
|
522
|
+
const tokenEmbeddings = normalizedTokenEmbeddings ?? extractTokenEmbeddingsFromHidden(
|
|
523
|
+
hiddenStates,
|
|
524
|
+
numTokens,
|
|
525
|
+
hiddenSize,
|
|
526
|
+
finalNormWeights,
|
|
527
|
+
config,
|
|
528
|
+
finalNormBias
|
|
529
|
+
);
|
|
483
530
|
|
|
484
531
|
const postprocessorConfig = config?.embeddingPostprocessor ?? null;
|
|
485
532
|
const resolvedEmbeddingMode = postprocessorConfig?.poolingMode ?? embeddingMode;
|
|
@@ -491,13 +538,14 @@ export function extractEmbeddingFromHidden(
|
|
|
491
538
|
|
|
492
539
|
let pooled;
|
|
493
540
|
if (resolvedEmbeddingMode === 'last') {
|
|
494
|
-
|
|
541
|
+
const offset = (numTokens - 1) * hiddenSize;
|
|
542
|
+
pooled = tokenEmbeddings.slice(offset, offset + hiddenSize);
|
|
495
543
|
} else if (resolvedEmbeddingMode === 'mean') {
|
|
496
544
|
pooled = new Float32Array(hiddenSize);
|
|
497
545
|
for (let t = 0; t < numTokens; t++) {
|
|
498
|
-
const
|
|
546
|
+
const offset = t * hiddenSize;
|
|
499
547
|
for (let i = 0; i < hiddenSize; i++) {
|
|
500
|
-
pooled[i] +=
|
|
548
|
+
pooled[i] += tokenEmbeddings[offset + i];
|
|
501
549
|
}
|
|
502
550
|
}
|
|
503
551
|
const invTokens = numTokens > 0 ? (1 / numTokens) : 1;
|
|
@@ -1222,7 +1222,9 @@ export async function advanceWithTokenAndEmbedding(state, tokenId, opts, helpers
|
|
|
1222
1222
|
embeddingMode,
|
|
1223
1223
|
finalNormWeights,
|
|
1224
1224
|
config,
|
|
1225
|
-
state.embeddingPostprocessor
|
|
1225
|
+
state.embeddingPostprocessor,
|
|
1226
|
+
null,
|
|
1227
|
+
state.weights.get('final_norm_bias') ?? null
|
|
1226
1228
|
);
|
|
1227
1229
|
} finally {
|
|
1228
1230
|
const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
|
|
@@ -63,6 +63,7 @@ import {
|
|
|
63
63
|
resolveAdvanceEmbeddingMode,
|
|
64
64
|
getFinalNormWeights,
|
|
65
65
|
extractEmbeddingFromHidden,
|
|
66
|
+
extractTokenEmbeddingsFromHidden,
|
|
66
67
|
} from './generator-runtime.js';
|
|
67
68
|
|
|
68
69
|
import { resolveSamplingConfig } from './sampling-config.js';
|
|
@@ -642,7 +643,20 @@ export class PipelineGenerator {
|
|
|
642
643
|
embeddingMode,
|
|
643
644
|
finalNormWeights,
|
|
644
645
|
config,
|
|
645
|
-
this.#state.embeddingPostprocessor
|
|
646
|
+
this.#state.embeddingPostprocessor,
|
|
647
|
+
null,
|
|
648
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
_extractTokenEmbeddingsFromHidden(hiddenStates, numTokens, hiddenSize, finalNormWeights, config) {
|
|
653
|
+
return extractTokenEmbeddingsFromHidden(
|
|
654
|
+
hiddenStates,
|
|
655
|
+
numTokens,
|
|
656
|
+
hiddenSize,
|
|
657
|
+
finalNormWeights,
|
|
658
|
+
config,
|
|
659
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
646
660
|
);
|
|
647
661
|
}
|
|
648
662
|
|
|
@@ -2112,6 +2126,34 @@ export class PipelineGenerator {
|
|
|
2112
2126
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2113
2127
|
this.#state.stats.prefillProfileSteps = [];
|
|
2114
2128
|
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
2129
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2130
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2131
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
2132
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
2133
|
+
const captureConfig = {
|
|
2134
|
+
...createDefaultCaptureConfig(),
|
|
2135
|
+
enabled: true,
|
|
2136
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2137
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2138
|
+
};
|
|
2139
|
+
validateCaptureConfig(captureConfig);
|
|
2140
|
+
this.#state.operatorDiagnostics = {
|
|
2141
|
+
enabled: diagnosticsEnabled === true,
|
|
2142
|
+
captureConfig,
|
|
2143
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
2144
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2145
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2146
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2147
|
+
}) : null,
|
|
2148
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
2149
|
+
dir: tsirFixtureCfg.dir,
|
|
2150
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
2151
|
+
? tsirFixtureCfg.layerFilter
|
|
2152
|
+
: null,
|
|
2153
|
+
records: [],
|
|
2154
|
+
} : null,
|
|
2155
|
+
};
|
|
2156
|
+
}
|
|
2115
2157
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2116
2158
|
const inputStart = performance.now();
|
|
2117
2159
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
@@ -2178,12 +2220,32 @@ export class PipelineGenerator {
|
|
|
2178
2220
|
}
|
|
2179
2221
|
|
|
2180
2222
|
let embedding;
|
|
2223
|
+
let tokenEmbeddings = null;
|
|
2224
|
+
let logits = null;
|
|
2181
2225
|
let hiddenBytes = 0;
|
|
2182
2226
|
let readbackMs = 0;
|
|
2183
2227
|
let decodeHiddenMs = 0;
|
|
2184
2228
|
let finalNormMs = 0;
|
|
2185
2229
|
let extractMs = 0;
|
|
2230
|
+
let logitsMs = 0;
|
|
2186
2231
|
try {
|
|
2232
|
+
if (options.__returnSequenceLogits === true) {
|
|
2233
|
+
const logitsStart = performance.now();
|
|
2234
|
+
logits = await computeLogits(
|
|
2235
|
+
currentHiddenBuffer,
|
|
2236
|
+
numTokens,
|
|
2237
|
+
getLogitsWeights(this.#state),
|
|
2238
|
+
getLogitsConfig(this.#state),
|
|
2239
|
+
this.#state.useGPU,
|
|
2240
|
+
this.#state.debugFlags,
|
|
2241
|
+
undefined,
|
|
2242
|
+
undefined,
|
|
2243
|
+
this.#state.runtimeConfig.shared.debug.probes,
|
|
2244
|
+
{ lastPositionOnly: false },
|
|
2245
|
+
this.#state.operatorDiagnostics
|
|
2246
|
+
);
|
|
2247
|
+
logitsMs = performance.now() - logitsStart;
|
|
2248
|
+
}
|
|
2187
2249
|
const hiddenSize = config.hiddenSize;
|
|
2188
2250
|
hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
2189
2251
|
const readbackStart = performance.now();
|
|
@@ -2199,13 +2261,25 @@ export class PipelineGenerator {
|
|
|
2199
2261
|
const finalNormWeights = await this._getFinalNormWeights();
|
|
2200
2262
|
finalNormMs = performance.now() - finalNormStart;
|
|
2201
2263
|
const extractStart = performance.now();
|
|
2202
|
-
|
|
2264
|
+
if (options.__returnTokenEmbeddings === true) {
|
|
2265
|
+
tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
|
|
2266
|
+
hiddenStates,
|
|
2267
|
+
numTokens,
|
|
2268
|
+
hiddenSize,
|
|
2269
|
+
finalNormWeights,
|
|
2270
|
+
config
|
|
2271
|
+
);
|
|
2272
|
+
}
|
|
2273
|
+
embedding = extractEmbeddingFromHidden(
|
|
2203
2274
|
hiddenStates,
|
|
2204
2275
|
numTokens,
|
|
2205
2276
|
hiddenSize,
|
|
2206
2277
|
opts.embeddingMode,
|
|
2207
2278
|
finalNormWeights,
|
|
2208
|
-
config
|
|
2279
|
+
config,
|
|
2280
|
+
this.#state.embeddingPostprocessor,
|
|
2281
|
+
tokenEmbeddings,
|
|
2282
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
2209
2283
|
);
|
|
2210
2284
|
extractMs = performance.now() - extractStart;
|
|
2211
2285
|
} finally {
|
|
@@ -2222,6 +2296,7 @@ export class PipelineGenerator {
|
|
|
2222
2296
|
decodeHiddenMs,
|
|
2223
2297
|
finalNormMs,
|
|
2224
2298
|
extractMs,
|
|
2299
|
+
logitsMs,
|
|
2225
2300
|
hiddenBytes,
|
|
2226
2301
|
tokens: numTokens,
|
|
2227
2302
|
activationDtype,
|
|
@@ -2241,6 +2316,8 @@ export class PipelineGenerator {
|
|
|
2241
2316
|
seqLen: this.#state.currentSeqLen,
|
|
2242
2317
|
tokens: inputIds,
|
|
2243
2318
|
embedding,
|
|
2319
|
+
tokenEmbeddings,
|
|
2320
|
+
logits,
|
|
2244
2321
|
embeddingMode: opts.embeddingMode,
|
|
2245
2322
|
phase,
|
|
2246
2323
|
linearAttention: null,
|
|
@@ -2257,12 +2334,22 @@ export class PipelineGenerator {
|
|
|
2257
2334
|
seqLen: this.#state.currentSeqLen,
|
|
2258
2335
|
tokens: inputIds,
|
|
2259
2336
|
embedding,
|
|
2337
|
+
tokenEmbeddings,
|
|
2338
|
+
logits,
|
|
2260
2339
|
embeddingMode: opts.embeddingMode,
|
|
2261
2340
|
phase,
|
|
2262
2341
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2263
2342
|
};
|
|
2264
2343
|
} finally {
|
|
2265
2344
|
this._closeFinitenessFallbackWindow(opts);
|
|
2345
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2346
|
+
? {
|
|
2347
|
+
enabled: true,
|
|
2348
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2349
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2350
|
+
}
|
|
2351
|
+
: null;
|
|
2352
|
+
this.#state.operatorDiagnostics = null;
|
|
2266
2353
|
}
|
|
2267
2354
|
}
|
|
2268
2355
|
|