doppler-gpu 0.4.10 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +40 -1
- package/models/catalog.json +637 -32
- package/package.json +24 -3
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +135 -13
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +69 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +73 -11
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +24 -4
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +205 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +196 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +33 -0
- package/src/inference/pipelines/text/types.d.ts +66 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +9 -2
- package/src/inference/pipelines/text.js +116 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -186,6 +186,8 @@ export interface ParsedModelConfig {
|
|
|
186
186
|
ropeLocalScaling: RopeScalingConfig | null;
|
|
187
187
|
quantization: string;
|
|
188
188
|
quantMethod: string | null;
|
|
189
|
+
normalizationType: 'rmsnorm' | 'layernorm';
|
|
190
|
+
finalNormBiasTensor: string | null;
|
|
189
191
|
rmsNormEps: number;
|
|
190
192
|
rmsNormWeightOffset: boolean;
|
|
191
193
|
postAttentionNorm: boolean;
|
|
@@ -200,6 +202,7 @@ export interface ParsedModelConfig {
|
|
|
200
202
|
embeddingVocabSize: number | null;
|
|
201
203
|
embeddingPostprocessor: ManifestEmbeddingPostprocessorSchema | null;
|
|
202
204
|
hiddenActivation: ActivationType;
|
|
205
|
+
gatedActivation: boolean;
|
|
203
206
|
ffnBranchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
|
|
204
207
|
useDoubleWideMlp: boolean;
|
|
205
208
|
swigluLimit: number | null;
|
|
@@ -216,6 +219,8 @@ export interface ParsedModelConfig {
|
|
|
216
219
|
finalLogitSoftcapping: number | null;
|
|
217
220
|
attnLogitSoftcapping: number | null;
|
|
218
221
|
queryKeyNorm: boolean;
|
|
222
|
+
queryKeyNormType: 'rmsnorm' | 'layernorm';
|
|
223
|
+
queryKeyNormAxis: 'head' | 'projection';
|
|
219
224
|
queryKeyNormLayers: number[] | null;
|
|
220
225
|
queryKeyNormWeightLayers: number[] | null;
|
|
221
226
|
valueNorm: boolean;
|
|
@@ -458,6 +458,9 @@ export function getStopTokenIds(manifest) {
|
|
|
458
458
|
if (typeof eosTokenId === 'number') return [eosTokenId];
|
|
459
459
|
const modelId = manifest?.modelId ?? 'unknown';
|
|
460
460
|
if (eosTokenId == null) {
|
|
461
|
+
if (manifest?.modelType === 'embedding') {
|
|
462
|
+
return [];
|
|
463
|
+
}
|
|
461
464
|
throw new Error(
|
|
462
465
|
`Manifest "${modelId}" is missing eos_token_id. Re-convert the model with tokenizer metadata.`
|
|
463
466
|
);
|
|
@@ -708,6 +711,18 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
708
711
|
if (inf.attention.queryKeyNorm == null) {
|
|
709
712
|
errors.push('attention.queryKeyNorm is required');
|
|
710
713
|
}
|
|
714
|
+
if (
|
|
715
|
+
inf.attention.queryKeyNormType !== undefined
|
|
716
|
+
&& !['rmsnorm', 'layernorm'].includes(inf.attention.queryKeyNormType)
|
|
717
|
+
) {
|
|
718
|
+
errors.push('attention.queryKeyNormType must be "rmsnorm" or "layernorm"');
|
|
719
|
+
}
|
|
720
|
+
if (
|
|
721
|
+
inf.attention.queryKeyNormAxis !== undefined
|
|
722
|
+
&& !['head', 'projection'].includes(inf.attention.queryKeyNormAxis)
|
|
723
|
+
) {
|
|
724
|
+
errors.push('attention.queryKeyNormAxis must be "head" or "projection"');
|
|
725
|
+
}
|
|
711
726
|
if (
|
|
712
727
|
inf.attention.queryKeyNormLayers !== undefined
|
|
713
728
|
&& inf.attention.queryKeyNormLayers !== null
|
|
@@ -756,6 +771,12 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
756
771
|
}
|
|
757
772
|
|
|
758
773
|
// Normalization fields
|
|
774
|
+
if (
|
|
775
|
+
inf.normalization.type !== undefined
|
|
776
|
+
&& !['rmsnorm', 'layernorm'].includes(inf.normalization.type)
|
|
777
|
+
) {
|
|
778
|
+
errors.push('normalization.type must be rmsnorm or layernorm');
|
|
779
|
+
}
|
|
759
780
|
if (inf.normalization.rmsNormWeightOffset == null) {
|
|
760
781
|
errors.push('normalization.rmsNormWeightOffset is required');
|
|
761
782
|
}
|
|
@@ -771,6 +792,23 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
771
792
|
if (inf.normalization.postFeedforwardNorm == null) {
|
|
772
793
|
errors.push('normalization.postFeedforwardNorm is required');
|
|
773
794
|
}
|
|
795
|
+
if (
|
|
796
|
+
inf.normalization.finalNormBiasTensor !== undefined
|
|
797
|
+
&&
|
|
798
|
+
inf.normalization.finalNormBiasTensor !== null
|
|
799
|
+
&& (
|
|
800
|
+
typeof inf.normalization.finalNormBiasTensor !== 'string'
|
|
801
|
+
|| inf.normalization.finalNormBiasTensor.trim() === ''
|
|
802
|
+
)
|
|
803
|
+
) {
|
|
804
|
+
errors.push('normalization.finalNormBiasTensor must be null or a non-empty string');
|
|
805
|
+
}
|
|
806
|
+
if (
|
|
807
|
+
(inf.normalization.type ?? 'rmsnorm') === 'rmsnorm'
|
|
808
|
+
&& (inf.normalization.finalNormBiasTensor ?? null) !== null
|
|
809
|
+
) {
|
|
810
|
+
errors.push('normalization.finalNormBiasTensor requires normalization.type="layernorm"');
|
|
811
|
+
}
|
|
774
812
|
|
|
775
813
|
// FFN fields
|
|
776
814
|
if (inf.ffn.activation == null) {
|
|
@@ -944,6 +982,54 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
944
982
|
}
|
|
945
983
|
}
|
|
946
984
|
}
|
|
985
|
+
if (
|
|
986
|
+
inf.output.lmHeadBiasTensor !== undefined
|
|
987
|
+
&& inf.output.lmHeadBiasTensor !== null
|
|
988
|
+
&& (typeof inf.output.lmHeadBiasTensor !== 'string' || inf.output.lmHeadBiasTensor.trim() === '')
|
|
989
|
+
) {
|
|
990
|
+
errors.push('output.lmHeadBiasTensor must be null or a non-empty tensor name');
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
if (inf.supportsSequence !== undefined && typeof inf.supportsSequence !== 'boolean') {
|
|
994
|
+
errors.push('supportsSequence must be boolean when provided');
|
|
995
|
+
}
|
|
996
|
+
if (inf.sequence !== undefined && inf.sequence !== null) {
|
|
997
|
+
const sequence = inf.sequence;
|
|
998
|
+
if (!sequence || typeof sequence !== 'object' || Array.isArray(sequence)) {
|
|
999
|
+
errors.push('sequence must be null or an object');
|
|
1000
|
+
} else {
|
|
1001
|
+
if (sequence.alphabet !== 'amino_acid' && sequence.alphabet !== 'nucleotide') {
|
|
1002
|
+
errors.push('sequence.alphabet must be "amino_acid" or "nucleotide"');
|
|
1003
|
+
}
|
|
1004
|
+
if (typeof sequence.tokenEmbeddings !== 'boolean') {
|
|
1005
|
+
errors.push('sequence.tokenEmbeddings must be boolean');
|
|
1006
|
+
}
|
|
1007
|
+
if (typeof sequence.logits !== 'boolean') {
|
|
1008
|
+
errors.push('sequence.logits must be boolean');
|
|
1009
|
+
}
|
|
1010
|
+
if (sequence.pooledEmbedding !== null) {
|
|
1011
|
+
const pooling = sequence.pooledEmbedding;
|
|
1012
|
+
if (!pooling || typeof pooling !== 'object' || Array.isArray(pooling)) {
|
|
1013
|
+
errors.push('sequence.pooledEmbedding must be null or an object');
|
|
1014
|
+
} else {
|
|
1015
|
+
if (pooling.mode !== 'mean' && pooling.mode !== 'last') {
|
|
1016
|
+
errors.push('sequence.pooledEmbedding.mode must be "mean" or "last"');
|
|
1017
|
+
}
|
|
1018
|
+
if (!Array.isArray(pooling.excludeTokenIds) || pooling.excludeTokenIds.some(
|
|
1019
|
+
(tokenId) => !Number.isInteger(tokenId) || tokenId < 0
|
|
1020
|
+
)) {
|
|
1021
|
+
errors.push('sequence.pooledEmbedding.excludeTokenIds must be an array of non-negative integers');
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
if (inf.supportsSequence === true && (inf.sequence == null || typeof inf.sequence !== 'object')) {
|
|
1028
|
+
errors.push('sequence is required when supportsSequence=true');
|
|
1029
|
+
}
|
|
1030
|
+
if (inf.supportsSequence !== true && inf.sequence != null) {
|
|
1031
|
+
errors.push('supportsSequence must be true when sequence is configured');
|
|
1032
|
+
}
|
|
947
1033
|
|
|
948
1034
|
// Layer pattern fields
|
|
949
1035
|
if (inf.layerPattern?.type == null) {
|
|
@@ -1843,6 +1929,8 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1843
1929
|
ropeLocalScaling,
|
|
1844
1930
|
quantization: manifest.quantization,
|
|
1845
1931
|
quantMethod: config.quantization_config?.quant_method ?? null,
|
|
1932
|
+
normalizationType: inf.normalization.type ?? 'rmsnorm',
|
|
1933
|
+
finalNormBiasTensor: inf.normalization.finalNormBiasTensor ?? null,
|
|
1846
1934
|
rmsNormEps: inf.normalization.rmsNormEps,
|
|
1847
1935
|
rmsNormWeightOffset: inf.normalization.rmsNormWeightOffset,
|
|
1848
1936
|
postAttentionNorm: inf.normalization.postAttentionNorm,
|
|
@@ -1857,6 +1945,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1857
1945
|
embeddingVocabSize: inf.output.embeddingVocabSize,
|
|
1858
1946
|
embeddingPostprocessor,
|
|
1859
1947
|
hiddenActivation,
|
|
1948
|
+
gatedActivation: inf.ffn.gatedActivation,
|
|
1860
1949
|
ffnBranchMode: inf.ffn.branchMode,
|
|
1861
1950
|
useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
|
|
1862
1951
|
swigluLimit: inf.ffn.swigluLimit,
|
|
@@ -1873,6 +1962,8 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1873
1962
|
finalLogitSoftcapping: inf.output.finalLogitSoftcapping,
|
|
1874
1963
|
attnLogitSoftcapping: inf.attention.attnLogitSoftcapping,
|
|
1875
1964
|
queryKeyNorm: inf.attention.queryKeyNorm,
|
|
1965
|
+
queryKeyNormType: inf.attention.queryKeyNormType ?? 'rmsnorm',
|
|
1966
|
+
queryKeyNormAxis: inf.attention.queryKeyNormAxis ?? 'head',
|
|
1876
1967
|
queryKeyNormLayers: inf.attention.queryKeyNormLayers ?? null,
|
|
1877
1968
|
queryKeyNormWeightLayers: inf.attention.queryKeyNormWeightLayers ?? null,
|
|
1878
1969
|
valueNorm: inf.attention.valueNorm,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
|
-
doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
|
|
4
|
+
doBiasAdd, doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
|
|
5
5
|
releaseOrTrack
|
|
6
6
|
} from '../ops.js';
|
|
7
7
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
@@ -29,7 +29,7 @@ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
|
|
|
29
29
|
import { isKernelDebugEnabled, dumpTokenVector, decodeReadback, getLogitsHealth, shouldDebugLayerOutput } from '../debug-utils.js';
|
|
30
30
|
import { applyLoRA } from '../lora-apply.js';
|
|
31
31
|
import { getLoRAModule } from '../lora.js';
|
|
32
|
-
import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
|
|
32
|
+
import { getWeightBuffer, getNormWeightBuffer, getVectorTensor } from '../weights.js';
|
|
33
33
|
import { runProbes } from '../probes.js';
|
|
34
34
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
35
35
|
import {
|
|
@@ -49,6 +49,19 @@ function resolveActivationOp(hiddenActivation) {
|
|
|
49
49
|
return selectRuleValue('inference', 'ffn', 'activationOp', { hiddenActivation });
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
async function captureTrainingActivation(context, layerIdx, tensor, numTokens, hiddenSize) {
|
|
53
|
+
const capture = context.trainingCapture;
|
|
54
|
+
if (!capture || capture.layerIdx !== layerIdx || capture.stage !== 'ffn_act') return;
|
|
55
|
+
await capture.capture({
|
|
56
|
+
layerIdx,
|
|
57
|
+
stage: 'ffn_act',
|
|
58
|
+
tensor,
|
|
59
|
+
numTokens,
|
|
60
|
+
hiddenSize,
|
|
61
|
+
recorder: context.recorder ?? null,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
52
65
|
function hasQ4KMaterialization(weight) {
|
|
53
66
|
return isWeightBuffer(weight) && !!weight.materializations?.q4k?.buffer;
|
|
54
67
|
}
|
|
@@ -527,6 +540,189 @@ async function dispatchFusedGateUp({
|
|
|
527
540
|
return fusedOutput;
|
|
528
541
|
}
|
|
529
542
|
|
|
543
|
+
async function applyDenseProjectionBias(
|
|
544
|
+
tensor,
|
|
545
|
+
biasWeight,
|
|
546
|
+
label,
|
|
547
|
+
numTokens,
|
|
548
|
+
dim,
|
|
549
|
+
context,
|
|
550
|
+
layerIdx
|
|
551
|
+
) {
|
|
552
|
+
if (!biasWeight) return tensor;
|
|
553
|
+
const { tensor: biasTensor, owned } = getVectorTensor(
|
|
554
|
+
biasWeight,
|
|
555
|
+
label,
|
|
556
|
+
dim,
|
|
557
|
+
context.weightConfig,
|
|
558
|
+
context.debugFlags
|
|
559
|
+
);
|
|
560
|
+
try {
|
|
561
|
+
return await doBiasAdd(tensor, biasTensor, numTokens, dim, {
|
|
562
|
+
label,
|
|
563
|
+
layerIdx,
|
|
564
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
565
|
+
}, context.recorder);
|
|
566
|
+
} finally {
|
|
567
|
+
if (owned) releaseOrTrack(context.recorder, biasTensor.buffer, context.decodeBuffers);
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
async function runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, context, layerWeights) {
|
|
572
|
+
const { config, recorder } = context;
|
|
573
|
+
const { hiddenSize, hiddenActivation } = config;
|
|
574
|
+
const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
|
|
575
|
+
if (layerWeights.gate || layerWeights.gateUp) {
|
|
576
|
+
throw new Error(`Layer ${layerIdx} declares an ungated FFN but contains gate weights.`);
|
|
577
|
+
}
|
|
578
|
+
if (!layerWeights.up || !layerWeights.down) {
|
|
579
|
+
throw new Error(`Layer ${layerIdx} ungated FFN requires up_proj and down_proj weights.`);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
const kernelPath = context.kernelPath ?? null;
|
|
583
|
+
const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
|
|
584
|
+
const ffnStepPrecision = context.ffnStepPrecision ?? null;
|
|
585
|
+
const upInputDtype = resolveMatmulStepDtype(
|
|
586
|
+
'ffn_up', phase, layerIdx, kernelPath, inputTensor.dtype, 'inputDtype', ffnStepPrecision
|
|
587
|
+
);
|
|
588
|
+
const upOutputDtype = resolveMatmulStepDtype(
|
|
589
|
+
'ffn_up', phase, layerIdx, kernelPath, inputTensor.dtype, 'outputDtype', ffnStepPrecision
|
|
590
|
+
);
|
|
591
|
+
const downInputDtype = resolveMatmulStepDtype(
|
|
592
|
+
'ffn_down', phase, layerIdx, kernelPath, upOutputDtype, 'inputDtype', ffnStepPrecision
|
|
593
|
+
);
|
|
594
|
+
const downOutputDtype = resolveMatmulStepDtype(
|
|
595
|
+
'ffn_down', phase, layerIdx, kernelPath, inputTensor.dtype, 'outputDtype', ffnStepPrecision
|
|
596
|
+
);
|
|
597
|
+
|
|
598
|
+
let upInput = inputTensor;
|
|
599
|
+
let upInputOwned = false;
|
|
600
|
+
if (upInputDtype && upInputDtype !== inputTensor.dtype) {
|
|
601
|
+
upInput = await coerceTensorDtype(inputTensor, upInputDtype, recorder, {
|
|
602
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
603
|
+
op: 'ffn_up_input',
|
|
604
|
+
transitionDeclaredBy: 'step_precision',
|
|
605
|
+
});
|
|
606
|
+
upInputOwned = upInput !== inputTensor;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
const upWeight = getWeightBuffer(layerWeights.up, 'ffn_up');
|
|
610
|
+
let upOutput = await doMatmul(
|
|
611
|
+
upInput,
|
|
612
|
+
upWeight,
|
|
613
|
+
numTokens,
|
|
614
|
+
intermediateSize,
|
|
615
|
+
hiddenSize,
|
|
616
|
+
{
|
|
617
|
+
transposeB: 'auto',
|
|
618
|
+
label: `L${layerIdx}.ffn_up`,
|
|
619
|
+
layerIdx,
|
|
620
|
+
kernelPath,
|
|
621
|
+
outputDtype: upOutputDtype,
|
|
622
|
+
role: 'ffn_up',
|
|
623
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
624
|
+
},
|
|
625
|
+
recorder
|
|
626
|
+
);
|
|
627
|
+
const loraUp = getLoRAModule(context.lora ?? null, layerIdx, 'up_proj');
|
|
628
|
+
if (loraUp) {
|
|
629
|
+
const combined = await applyLoRA(
|
|
630
|
+
upInput,
|
|
631
|
+
upOutput,
|
|
632
|
+
loraUp,
|
|
633
|
+
{ M: numTokens, N: intermediateSize, K: hiddenSize },
|
|
634
|
+
getWeightBuffer,
|
|
635
|
+
recorder,
|
|
636
|
+
{ kernelPath }
|
|
637
|
+
);
|
|
638
|
+
if (combined.buffer !== upOutput.buffer) {
|
|
639
|
+
releaseOrTrack(recorder, upOutput.buffer, context.decodeBuffers);
|
|
640
|
+
upOutput = combined;
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
upOutput = await applyDenseProjectionBias(
|
|
644
|
+
upOutput,
|
|
645
|
+
layerWeights.upBias,
|
|
646
|
+
`L${layerIdx}.ffn_up_bias`,
|
|
647
|
+
numTokens,
|
|
648
|
+
intermediateSize,
|
|
649
|
+
context,
|
|
650
|
+
layerIdx
|
|
651
|
+
);
|
|
652
|
+
const activated = await dispatchActivation(hiddenActivation, upOutput, {
|
|
653
|
+
size: numTokens * intermediateSize,
|
|
654
|
+
gate: null,
|
|
655
|
+
label: `L${layerIdx}.ffn_activation`,
|
|
656
|
+
layerIdx,
|
|
657
|
+
}, recorder);
|
|
658
|
+
|
|
659
|
+
let downInput = activated;
|
|
660
|
+
let downInputOwned = false;
|
|
661
|
+
if (downInputDtype && activated.dtype !== downInputDtype) {
|
|
662
|
+
downInput = await coerceTensorDtype(activated, downInputDtype, recorder, {
|
|
663
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
664
|
+
op: 'ffn_down_input',
|
|
665
|
+
transitionDeclaredBy: 'step_precision',
|
|
666
|
+
});
|
|
667
|
+
downInputOwned = downInput !== activated;
|
|
668
|
+
}
|
|
669
|
+
const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
|
|
670
|
+
let output = await doMatmul(
|
|
671
|
+
downInput,
|
|
672
|
+
downWeight,
|
|
673
|
+
numTokens,
|
|
674
|
+
hiddenSize,
|
|
675
|
+
intermediateSize,
|
|
676
|
+
{
|
|
677
|
+
transposeB: 'auto',
|
|
678
|
+
label: `L${layerIdx}.ffn_down`,
|
|
679
|
+
layerIdx,
|
|
680
|
+
kernelPath,
|
|
681
|
+
outputDtype: downOutputDtype,
|
|
682
|
+
role: 'ffn_down',
|
|
683
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
684
|
+
},
|
|
685
|
+
recorder
|
|
686
|
+
);
|
|
687
|
+
const loraDown = getLoRAModule(context.lora ?? null, layerIdx, 'down_proj');
|
|
688
|
+
if (loraDown) {
|
|
689
|
+
const combined = await applyLoRA(
|
|
690
|
+
downInput,
|
|
691
|
+
output,
|
|
692
|
+
loraDown,
|
|
693
|
+
{ M: numTokens, N: hiddenSize, K: intermediateSize },
|
|
694
|
+
getWeightBuffer,
|
|
695
|
+
recorder,
|
|
696
|
+
{ kernelPath }
|
|
697
|
+
);
|
|
698
|
+
if (combined.buffer !== output.buffer) {
|
|
699
|
+
releaseOrTrack(recorder, output.buffer, context.decodeBuffers);
|
|
700
|
+
output = combined;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
output = await applyDenseProjectionBias(
|
|
704
|
+
output,
|
|
705
|
+
layerWeights.downBias,
|
|
706
|
+
`L${layerIdx}.ffn_down_bias`,
|
|
707
|
+
numTokens,
|
|
708
|
+
hiddenSize,
|
|
709
|
+
context,
|
|
710
|
+
layerIdx
|
|
711
|
+
);
|
|
712
|
+
|
|
713
|
+
if (!isGpuBufferInstance(layerWeights.up) && !isWeightBuffer(layerWeights.up)) {
|
|
714
|
+
releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight, context.decodeBuffers);
|
|
715
|
+
}
|
|
716
|
+
if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
|
|
717
|
+
releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight, context.decodeBuffers);
|
|
718
|
+
}
|
|
719
|
+
if (upInputOwned) releaseOrTrack(recorder, upInput.buffer, context.decodeBuffers);
|
|
720
|
+
if (downInputOwned) releaseOrTrack(recorder, downInput.buffer, context.decodeBuffers);
|
|
721
|
+
releaseOrTrack(recorder, upOutput.buffer, context.decodeBuffers);
|
|
722
|
+
releaseOrTrack(recorder, activated.buffer, context.decodeBuffers);
|
|
723
|
+
return output;
|
|
724
|
+
}
|
|
725
|
+
|
|
530
726
|
|
|
531
727
|
export async function runDenseFFNGPU(
|
|
532
728
|
layerIdx,
|
|
@@ -548,6 +744,10 @@ export async function runDenseFFNGPU(
|
|
|
548
744
|
const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
|
|
549
745
|
const gateUpPathMode = resolveGateUpPathMode({ kernelPath, phase, layerIdx });
|
|
550
746
|
|
|
747
|
+
if (config.gatedActivation === false) {
|
|
748
|
+
return runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, context, layerWeights);
|
|
749
|
+
}
|
|
750
|
+
|
|
551
751
|
if (layerWeights?.gateUp && layerWeights?.down) {
|
|
552
752
|
const gateUpWeight = getWeightBuffer(layerWeights.gateUp, 'ffn_gate_up');
|
|
553
753
|
const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
|
|
@@ -643,6 +843,7 @@ export async function runDenseFFNGPU(
|
|
|
643
843
|
});
|
|
644
844
|
}
|
|
645
845
|
enqueueRecordedDenseHealth(context, layerIdx, 'ffn_act', activatedOutput, numTokens * intermediateSize);
|
|
846
|
+
await captureTrainingActivation(context, layerIdx, activatedOutput, numTokens, intermediateSize);
|
|
646
847
|
|
|
647
848
|
if (recorder) {
|
|
648
849
|
recorder.trackTemporaryBuffer(gateUpOutput.buffer);
|
|
@@ -869,6 +1070,7 @@ export async function runDenseFFNGPU(
|
|
|
869
1070
|
dtype: fusedOutput.dtype,
|
|
870
1071
|
});
|
|
871
1072
|
enqueueRecordedDenseHealth(context, layerIdx, 'ffn_fused_gate_up', fusedOutput, numTokens * intermediateSize);
|
|
1073
|
+
await captureTrainingActivation(context, layerIdx, fusedOutput, numTokens, intermediateSize);
|
|
872
1074
|
|
|
873
1075
|
let downInput = fusedOutput;
|
|
874
1076
|
if (downInputDtype && fusedOutput.dtype !== downInputDtype) {
|
|
@@ -1282,6 +1484,7 @@ export async function runDenseFFNGPU(
|
|
|
1282
1484
|
dtype: activatedOutput.dtype,
|
|
1283
1485
|
});
|
|
1284
1486
|
enqueueRecordedDenseHealth(context, layerIdx, 'ffn_act', activatedOutput, numTokens * intermediateSize);
|
|
1487
|
+
await captureTrainingActivation(context, layerIdx, activatedOutput, numTokens, intermediateSize);
|
|
1285
1488
|
|
|
1286
1489
|
if (recorder) {
|
|
1287
1490
|
recorder.trackTemporaryBuffer(gateOutput.buffer);
|
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import {
|
|
4
|
+
doCast,
|
|
5
|
+
doLayerNorm,
|
|
6
|
+
doRMSNorm,
|
|
7
|
+
doRMSNormStats,
|
|
8
|
+
doResidualAdd,
|
|
9
|
+
doResidualNextRMSNormPair,
|
|
10
|
+
releaseOrTrack,
|
|
11
|
+
} from '../ops.js';
|
|
12
|
+
import { getNormWeightBuffer, getVectorTensor } from '../weights.js';
|
|
5
13
|
import { runProbes } from '../probes.js';
|
|
6
14
|
import { isMoELayerLocal } from './types.js';
|
|
7
15
|
import { runDenseFFNGPU } from './dense.js';
|
|
@@ -15,6 +23,7 @@ import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
|
15
23
|
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
16
24
|
|
|
17
25
|
function shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context }) {
|
|
26
|
+
if (context.config.normalizationType !== 'rmsnorm') return false;
|
|
18
27
|
if (session?.usePostAttnNormFusedGateUp !== true) return false;
|
|
19
28
|
if (numTokens !== 1) return false;
|
|
20
29
|
if (postAttn?.dtype !== 'f32' || fusedResidualInput?.dtype !== 'f32') return false;
|
|
@@ -92,7 +101,44 @@ export async function processFFNStandard(
|
|
|
92
101
|
releasePostAttnNormWeight = !isGpuBufferInstance(layerWeights.postAttnNorm) && !isWeightBuffer(layerWeights.postAttnNorm);
|
|
93
102
|
const session = getRuntimeConfig().inference?.session;
|
|
94
103
|
|
|
95
|
-
if (
|
|
104
|
+
if (config.normalizationType === 'layernorm') {
|
|
105
|
+
if (!layerWeights.postAttentionNormBias) {
|
|
106
|
+
throw new Error(`Layer ${layerIdx} requires post_attention_layernorm.bias for LayerNorm execution.`);
|
|
107
|
+
}
|
|
108
|
+
let normInput = postAttn;
|
|
109
|
+
if (fusedResidualInput) {
|
|
110
|
+
normInput = await doResidualAdd(postAttn, fusedResidualInput, size, recorder, {
|
|
111
|
+
label: `L${layerIdx}.attention_residual`,
|
|
112
|
+
layerIdx,
|
|
113
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
114
|
+
});
|
|
115
|
+
prenormSumBuffer = normInput.buffer;
|
|
116
|
+
}
|
|
117
|
+
const { tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
|
|
118
|
+
layerWeights.postAttentionNormBias,
|
|
119
|
+
`L${layerIdx}.post_attn_norm_bias`,
|
|
120
|
+
hiddenSize,
|
|
121
|
+
weightConfig,
|
|
122
|
+
debugFlags
|
|
123
|
+
);
|
|
124
|
+
try {
|
|
125
|
+
normedTensor = await doLayerNorm(
|
|
126
|
+
normInput,
|
|
127
|
+
normWeightBuf,
|
|
128
|
+
normBiasTensor.buffer,
|
|
129
|
+
rmsNormEps,
|
|
130
|
+
{
|
|
131
|
+
batchSize: numTokens,
|
|
132
|
+
hiddenSize,
|
|
133
|
+
label: `L${layerIdx}.post_attn_norm`,
|
|
134
|
+
layerIdx,
|
|
135
|
+
},
|
|
136
|
+
recorder
|
|
137
|
+
);
|
|
138
|
+
} finally {
|
|
139
|
+
if (normBiasOwned) releaseOrTrack(recorder, normBiasTensor.buffer, decodeBuffers);
|
|
140
|
+
}
|
|
141
|
+
} else if (fusedResidualInput) {
|
|
96
142
|
const bytesPerElement = postAttn.dtype === 'f16' ? 2 : 4;
|
|
97
143
|
prenormSumBuffer = acquireBuffer(size * bytesPerElement, undefined, 'fused_prenorm_sum');
|
|
98
144
|
if (shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context })) {
|
|
@@ -226,6 +272,7 @@ export async function processFFNStandard(
|
|
|
226
272
|
: 1;
|
|
227
273
|
if (
|
|
228
274
|
nextInputNormFusion
|
|
275
|
+
&& config.normalizationType === 'rmsnorm'
|
|
229
276
|
&& residualInput.dtype === 'f32'
|
|
230
277
|
&& residualTensor.dtype === 'f32'
|
|
231
278
|
) {
|
|
@@ -204,13 +204,21 @@ function getWeightBufferConfig(state) {
|
|
|
204
204
|
export function getLogitsWeights(state) {
|
|
205
205
|
const finalNorm = state.weights.get('final_norm');
|
|
206
206
|
const lmHead = state.weights.get('lm_head');
|
|
207
|
+
const lmHeadBias = state.weights.get('lm_head_bias') ?? null;
|
|
208
|
+
const finalNormBias = state.weights.get('final_norm_bias') ?? null;
|
|
207
209
|
if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array || isWeightBuffer(finalNorm))) {
|
|
208
210
|
throw new Error('Final norm not found or invalid type');
|
|
209
211
|
}
|
|
210
212
|
if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
|
|
211
213
|
throw new Error('LM head not found or invalid type');
|
|
212
214
|
}
|
|
213
|
-
|
|
215
|
+
if (lmHeadBias !== null && !(lmHeadBias instanceof Float32Array)) {
|
|
216
|
+
throw new Error('LM head bias is not a Float32Array');
|
|
217
|
+
}
|
|
218
|
+
if (finalNormBias !== null && !(finalNormBias instanceof Float32Array)) {
|
|
219
|
+
throw new Error('Final norm bias is not a Float32Array');
|
|
220
|
+
}
|
|
221
|
+
return { finalNorm, finalNormBias, lmHead, lmHeadBias };
|
|
214
222
|
}
|
|
215
223
|
|
|
216
224
|
|
|
@@ -222,6 +230,8 @@ export function getLogitsConfig(state) {
|
|
|
222
230
|
return {
|
|
223
231
|
hiddenSize: config.hiddenSize,
|
|
224
232
|
vocabSize: config.vocabSize,
|
|
233
|
+
normalizationType: config.normalizationType,
|
|
234
|
+
finalNormBiasTensor: config.finalNormBiasTensor,
|
|
225
235
|
rmsNormEps: config.rmsNormEps,
|
|
226
236
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
227
237
|
useTiedEmbeddings: state.useTiedEmbeddings,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readBuffer } from '../../../memory/buffer-pool.js';
|
|
2
|
-
import { matmulCPU, rmsNormCPU } from './logits/index.js';
|
|
2
|
+
import { layerNormCPU, matmulCPU, rmsNormCPU } from './logits/index.js';
|
|
3
3
|
import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer, getBufferDtype } from '../../../gpu/weight-buffer.js';
|
|
4
4
|
import { decodeReadback } from './debug-utils/index.js';
|
|
5
5
|
import { resolveExecutionSessionPlan } from './execution-plan.js';
|
|
@@ -454,14 +454,13 @@ export async function getFinalNormWeights(state) {
|
|
|
454
454
|
return weights;
|
|
455
455
|
}
|
|
456
456
|
|
|
457
|
-
export function
|
|
457
|
+
export function extractTokenEmbeddingsFromHidden(
|
|
458
458
|
hiddenStates,
|
|
459
459
|
numTokens,
|
|
460
460
|
hiddenSize,
|
|
461
|
-
embeddingMode,
|
|
462
461
|
finalNormWeights,
|
|
463
462
|
config,
|
|
464
|
-
|
|
463
|
+
finalNormBias = null
|
|
465
464
|
) {
|
|
466
465
|
const expectedLength = numTokens * hiddenSize;
|
|
467
466
|
if (hiddenStates.length !== expectedLength) {
|
|
@@ -470,16 +469,64 @@ export function extractEmbeddingFromHidden(
|
|
|
470
469
|
);
|
|
471
470
|
}
|
|
472
471
|
|
|
473
|
-
const
|
|
472
|
+
const tokenEmbeddings = new Float32Array(expectedLength);
|
|
473
|
+
let effectiveLayerNormBias = null;
|
|
474
|
+
if (config.normalizationType === 'layernorm') {
|
|
475
|
+
if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
|
|
476
|
+
throw new Error(
|
|
477
|
+
`[Pipeline] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
effectiveLayerNormBias = finalNormBias instanceof Float32Array
|
|
481
|
+
? finalNormBias
|
|
482
|
+
: new Float32Array(hiddenSize);
|
|
483
|
+
if (effectiveLayerNormBias.length !== hiddenSize) {
|
|
484
|
+
throw new Error(
|
|
485
|
+
`[Pipeline] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveLayerNormBias.length}.`
|
|
486
|
+
);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
for (let tokenIndex = 0; tokenIndex < numTokens; tokenIndex += 1) {
|
|
474
490
|
const offset = tokenIndex * hiddenSize;
|
|
475
491
|
const tokenHidden = hiddenStates.subarray(offset, offset + hiddenSize);
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
492
|
+
if (config.normalizationType === 'layernorm') {
|
|
493
|
+
tokenEmbeddings.set(layerNormCPU(
|
|
494
|
+
tokenHidden,
|
|
495
|
+
finalNormWeights,
|
|
496
|
+
effectiveLayerNormBias,
|
|
497
|
+
config.rmsNormEps
|
|
498
|
+
), offset);
|
|
499
|
+
} else {
|
|
500
|
+
tokenEmbeddings.set(rmsNormCPU(
|
|
501
|
+
tokenHidden,
|
|
502
|
+
finalNormWeights,
|
|
503
|
+
config.rmsNormEps,
|
|
504
|
+
config.rmsNormWeightOffset
|
|
505
|
+
), offset);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
return tokenEmbeddings;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
export function extractEmbeddingFromHidden(
|
|
512
|
+
hiddenStates,
|
|
513
|
+
numTokens,
|
|
514
|
+
hiddenSize,
|
|
515
|
+
embeddingMode,
|
|
516
|
+
finalNormWeights,
|
|
517
|
+
config,
|
|
518
|
+
embeddingPostprocessor = null,
|
|
519
|
+
normalizedTokenEmbeddings = null,
|
|
520
|
+
finalNormBias = null
|
|
521
|
+
) {
|
|
522
|
+
const tokenEmbeddings = normalizedTokenEmbeddings ?? extractTokenEmbeddingsFromHidden(
|
|
523
|
+
hiddenStates,
|
|
524
|
+
numTokens,
|
|
525
|
+
hiddenSize,
|
|
526
|
+
finalNormWeights,
|
|
527
|
+
config,
|
|
528
|
+
finalNormBias
|
|
529
|
+
);
|
|
483
530
|
|
|
484
531
|
const postprocessorConfig = config?.embeddingPostprocessor ?? null;
|
|
485
532
|
const resolvedEmbeddingMode = postprocessorConfig?.poolingMode ?? embeddingMode;
|
|
@@ -491,13 +538,14 @@ export function extractEmbeddingFromHidden(
|
|
|
491
538
|
|
|
492
539
|
let pooled;
|
|
493
540
|
if (resolvedEmbeddingMode === 'last') {
|
|
494
|
-
|
|
541
|
+
const offset = (numTokens - 1) * hiddenSize;
|
|
542
|
+
pooled = tokenEmbeddings.slice(offset, offset + hiddenSize);
|
|
495
543
|
} else if (resolvedEmbeddingMode === 'mean') {
|
|
496
544
|
pooled = new Float32Array(hiddenSize);
|
|
497
545
|
for (let t = 0; t < numTokens; t++) {
|
|
498
|
-
const
|
|
546
|
+
const offset = t * hiddenSize;
|
|
499
547
|
for (let i = 0; i < hiddenSize; i++) {
|
|
500
|
-
pooled[i] +=
|
|
548
|
+
pooled[i] += tokenEmbeddings[offset + i];
|
|
501
549
|
}
|
|
502
550
|
}
|
|
503
551
|
const invTokens = numTokens > 0 ? (1 / numTokens) : 1;
|
|
@@ -1222,7 +1222,9 @@ export async function advanceWithTokenAndEmbedding(state, tokenId, opts, helpers
|
|
|
1222
1222
|
embeddingMode,
|
|
1223
1223
|
finalNormWeights,
|
|
1224
1224
|
config,
|
|
1225
|
-
state.embeddingPostprocessor
|
|
1225
|
+
state.embeddingPostprocessor,
|
|
1226
|
+
null,
|
|
1227
|
+
state.weights.get('final_norm_bias') ?? null
|
|
1226
1228
|
);
|
|
1227
1229
|
} finally {
|
|
1228
1230
|
const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import type { CommandRecorder, ProfileTimings } from '../../../gpu/command-recorder.js';
|
|
11
11
|
import type { PipelineState } from './state.js';
|
|
12
|
-
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext, WorkloadPhaseTiming } from './types.js';
|
|
12
|
+
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext, WorkloadPhaseTiming, NativeLoRAPrefillOptions, NativeLoRAPrefillResult } from './types.js';
|
|
13
13
|
import type { LogitsConfig, LogitsWeights } from './logits/index.js';
|
|
14
14
|
import type { WeightBufferConfig } from './weights.js';
|
|
15
15
|
import type { ChatMessage } from './chat-format.js';
|
|
@@ -65,6 +65,10 @@ export declare class PipelineGenerator {
|
|
|
65
65
|
resetGenerationState(): void;
|
|
66
66
|
resetToSeqLen(seqLen: number): void;
|
|
67
67
|
prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
|
|
68
|
+
prefillForLoRATraining(
|
|
69
|
+
inputIds: readonly number[] | Int32Array | Uint32Array,
|
|
70
|
+
options: NativeLoRAPrefillOptions
|
|
71
|
+
): Promise<NativeLoRAPrefillResult>;
|
|
68
72
|
computeDiffusionGemmaCanvasLogits(
|
|
69
73
|
args: DiffusionGemmaCanvasLogitsInput,
|
|
70
74
|
options?: GenerateOptions & { __internalGenerate?: boolean }
|