doppler-gpu 0.4.10 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +95 -0
  2. package/README.md +40 -1
  3. package/models/catalog.json +637 -32
  4. package/package.json +24 -3
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +135 -13
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +69 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
  18. package/src/config/kernels/kernel-ref-digests.js +2 -1
  19. package/src/config/kernels/registry.json +73 -11
  20. package/src/config/merge.js +24 -0
  21. package/src/config/required-inference-fields-contract-check.js +2 -0
  22. package/src/config/schema/adapter.schema.d.ts +6 -1
  23. package/src/config/schema/adapter.schema.js +5 -0
  24. package/src/config/schema/converter.schema.d.ts +7 -0
  25. package/src/config/schema/converter.schema.js +15 -0
  26. package/src/config/schema/inference.schema.d.ts +8 -0
  27. package/src/config/schema/manifest.schema.d.ts +32 -1
  28. package/src/config/schema/manifest.schema.js +26 -0
  29. package/src/converter/conversion-plan.js +15 -1
  30. package/src/converter/core.js +5 -4
  31. package/src/converter/source-tensor-rules.d.ts +50 -0
  32. package/src/converter/source-tensor-rules.js +180 -0
  33. package/src/converter/source-tokenizer.d.ts +45 -0
  34. package/src/converter/source-tokenizer.js +155 -0
  35. package/src/experimental/adapters/artifact-contract.d.ts +83 -0
  36. package/src/experimental/adapters/artifact-contract.js +254 -0
  37. package/src/experimental/adapters/index.d.ts +25 -0
  38. package/src/experimental/adapters/index.js +12 -0
  39. package/src/experimental/adapters/lora-loader.js +24 -4
  40. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  41. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  42. package/src/experimental/training/artifacts.d.ts +160 -0
  43. package/src/experimental/training/attention-backward.d.ts +64 -0
  44. package/src/experimental/training/attention-forward.d.ts +22 -0
  45. package/src/experimental/training/attention-forward.js +82 -0
  46. package/src/experimental/training/autograd.d.ts +76 -0
  47. package/src/experimental/training/checkpoint.d.ts +36 -0
  48. package/src/experimental/training/clip.d.ts +9 -0
  49. package/src/experimental/training/dataloader.d.ts +8 -0
  50. package/src/experimental/training/distillation/student-fixture.js +1 -0
  51. package/src/experimental/training/export.d.ts +41 -0
  52. package/src/experimental/training/lora-pipeline.d.ts +186 -0
  53. package/src/experimental/training/lora-pipeline.js +5 -0
  54. package/src/experimental/training/lora.d.ts +20 -0
  55. package/src/experimental/training/lora.js +4 -3
  56. package/src/experimental/training/loss-scaling.d.ts +21 -0
  57. package/src/experimental/training/loss.d.ts +10 -0
  58. package/src/experimental/training/native-qwen-lora.d.ts +106 -0
  59. package/src/experimental/training/native-qwen-lora.js +449 -0
  60. package/src/experimental/training/objectives/base.d.ts +58 -0
  61. package/src/experimental/training/optimizer.d.ts +22 -0
  62. package/src/experimental/training/optimizer.js +1 -0
  63. package/src/experimental/training/runner.d.ts +249 -0
  64. package/src/experimental/training/trainer.d.ts +89 -0
  65. package/src/experimental/training/workloads.d.ts +205 -0
  66. package/src/gpu/kernels/backward/adam.d.ts +1 -0
  67. package/src/gpu/kernels/backward/adam.js +6 -2
  68. package/src/gpu/kernels/backward/adam.wgsl +3 -2
  69. package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
  70. package/src/gpu/kernels/backward/utils.js +18 -4
  71. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  72. package/src/inference/pipelines/text/attention/projections.js +161 -24
  73. package/src/inference/pipelines/text/attention/record.js +67 -9
  74. package/src/inference/pipelines/text/attention/run.js +60 -7
  75. package/src/inference/pipelines/text/attention/types.js +18 -9
  76. package/src/inference/pipelines/text/config.d.ts +5 -0
  77. package/src/inference/pipelines/text/config.js +91 -0
  78. package/src/inference/pipelines/text/ffn/dense.js +205 -2
  79. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  80. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  81. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  82. package/src/inference/pipelines/text/generator-steps.js +3 -1
  83. package/src/inference/pipelines/text/generator.d.ts +5 -1
  84. package/src/inference/pipelines/text/generator.js +196 -3
  85. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  86. package/src/inference/pipelines/text/init.d.ts +2 -0
  87. package/src/inference/pipelines/text/init.js +2 -0
  88. package/src/inference/pipelines/text/layer.js +6 -0
  89. package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
  90. package/src/inference/pipelines/text/linear-attention.js +77 -6
  91. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  92. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  93. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  94. package/src/inference/pipelines/text/logits/index.js +85 -11
  95. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  96. package/src/inference/pipelines/text/lora-types.d.ts +6 -1
  97. package/src/inference/pipelines/text/lora-types.js +5 -0
  98. package/src/inference/pipelines/text/ops.js +33 -0
  99. package/src/inference/pipelines/text/types.d.ts +66 -0
  100. package/src/inference/pipelines/text/weights.d.ts +10 -0
  101. package/src/inference/pipelines/text/weights.js +33 -1
  102. package/src/inference/pipelines/text.d.ts +9 -2
  103. package/src/inference/pipelines/text.js +116 -0
  104. package/src/inference/tokenizers/behavior-flags.js +5 -1
  105. package/src/inference/tokenizers/bundled.js +99 -8
  106. package/src/loader/doppler-loader.d.ts +2 -0
  107. package/src/loader/doppler-loader.js +9 -0
  108. package/src/loader/final-weights-loader.d.ts +8 -0
  109. package/src/loader/final-weights-loader.js +42 -0
  110. package/src/loader/layer-loader.d.ts +6 -0
  111. package/src/loader/layer-loader.js +102 -3
  112. package/src/loader/loader-types.d.ts +15 -2
  113. package/src/models/diffusiongemma.js +1 -1
  114. package/src/models/embeddinggemma.js +1 -1
  115. package/src/models/gemma3.js +1 -1
  116. package/src/models/gemma4.js +1 -1
  117. package/src/models/qwen3.js +1 -1
  118. package/src/tooling/hf-registry-utils.js +1 -1
  119. package/src/tooling/node-browser-command-runner.d.ts +16 -0
  120. package/src/tooling/node-browser-command-runner.js +1 -1
  121. package/src/tooling/node-converter.js +33 -1
  122. package/src/tooling/node-webgpu.js +85 -2
  123. package/src/training-browser.d.ts +19 -0
  124. package/src/training-browser.js +13 -0
  125. package/src/training.d.ts +137 -0
  126. package/src/training.js +219 -0
  127. package/src/version.js +1 -1
  128. package/tests/kernels/browser/kernel-suite.js +9 -1
  129. package/tests/kernels/browser/test-page.js +4 -2
  130. package/tests/kernels/reference/adam.js +4 -2
  131. package/NOTICE +0 -5
@@ -186,6 +186,8 @@ export interface ParsedModelConfig {
186
186
  ropeLocalScaling: RopeScalingConfig | null;
187
187
  quantization: string;
188
188
  quantMethod: string | null;
189
+ normalizationType: 'rmsnorm' | 'layernorm';
190
+ finalNormBiasTensor: string | null;
189
191
  rmsNormEps: number;
190
192
  rmsNormWeightOffset: boolean;
191
193
  postAttentionNorm: boolean;
@@ -200,6 +202,7 @@ export interface ParsedModelConfig {
200
202
  embeddingVocabSize: number | null;
201
203
  embeddingPostprocessor: ManifestEmbeddingPostprocessorSchema | null;
202
204
  hiddenActivation: ActivationType;
205
+ gatedActivation: boolean;
203
206
  ffnBranchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
204
207
  useDoubleWideMlp: boolean;
205
208
  swigluLimit: number | null;
@@ -216,6 +219,8 @@ export interface ParsedModelConfig {
216
219
  finalLogitSoftcapping: number | null;
217
220
  attnLogitSoftcapping: number | null;
218
221
  queryKeyNorm: boolean;
222
+ queryKeyNormType: 'rmsnorm' | 'layernorm';
223
+ queryKeyNormAxis: 'head' | 'projection';
219
224
  queryKeyNormLayers: number[] | null;
220
225
  queryKeyNormWeightLayers: number[] | null;
221
226
  valueNorm: boolean;
@@ -458,6 +458,9 @@ export function getStopTokenIds(manifest) {
458
458
  if (typeof eosTokenId === 'number') return [eosTokenId];
459
459
  const modelId = manifest?.modelId ?? 'unknown';
460
460
  if (eosTokenId == null) {
461
+ if (manifest?.modelType === 'embedding') {
462
+ return [];
463
+ }
461
464
  throw new Error(
462
465
  `Manifest "${modelId}" is missing eos_token_id. Re-convert the model with tokenizer metadata.`
463
466
  );
@@ -708,6 +711,18 @@ export function validateRequiredInferenceFields(inf, modelId) {
708
711
  if (inf.attention.queryKeyNorm == null) {
709
712
  errors.push('attention.queryKeyNorm is required');
710
713
  }
714
+ if (
715
+ inf.attention.queryKeyNormType !== undefined
716
+ && !['rmsnorm', 'layernorm'].includes(inf.attention.queryKeyNormType)
717
+ ) {
718
+ errors.push('attention.queryKeyNormType must be "rmsnorm" or "layernorm"');
719
+ }
720
+ if (
721
+ inf.attention.queryKeyNormAxis !== undefined
722
+ && !['head', 'projection'].includes(inf.attention.queryKeyNormAxis)
723
+ ) {
724
+ errors.push('attention.queryKeyNormAxis must be "head" or "projection"');
725
+ }
711
726
  if (
712
727
  inf.attention.queryKeyNormLayers !== undefined
713
728
  && inf.attention.queryKeyNormLayers !== null
@@ -756,6 +771,12 @@ export function validateRequiredInferenceFields(inf, modelId) {
756
771
  }
757
772
 
758
773
  // Normalization fields
774
+ if (
775
+ inf.normalization.type !== undefined
776
+ && !['rmsnorm', 'layernorm'].includes(inf.normalization.type)
777
+ ) {
778
+ errors.push('normalization.type must be rmsnorm or layernorm');
779
+ }
759
780
  if (inf.normalization.rmsNormWeightOffset == null) {
760
781
  errors.push('normalization.rmsNormWeightOffset is required');
761
782
  }
@@ -771,6 +792,23 @@ export function validateRequiredInferenceFields(inf, modelId) {
771
792
  if (inf.normalization.postFeedforwardNorm == null) {
772
793
  errors.push('normalization.postFeedforwardNorm is required');
773
794
  }
795
+ if (
796
+ inf.normalization.finalNormBiasTensor !== undefined
797
+ &&
798
+ inf.normalization.finalNormBiasTensor !== null
799
+ && (
800
+ typeof inf.normalization.finalNormBiasTensor !== 'string'
801
+ || inf.normalization.finalNormBiasTensor.trim() === ''
802
+ )
803
+ ) {
804
+ errors.push('normalization.finalNormBiasTensor must be null or a non-empty string');
805
+ }
806
+ if (
807
+ (inf.normalization.type ?? 'rmsnorm') === 'rmsnorm'
808
+ && (inf.normalization.finalNormBiasTensor ?? null) !== null
809
+ ) {
810
+ errors.push('normalization.finalNormBiasTensor requires normalization.type="layernorm"');
811
+ }
774
812
 
775
813
  // FFN fields
776
814
  if (inf.ffn.activation == null) {
@@ -944,6 +982,54 @@ export function validateRequiredInferenceFields(inf, modelId) {
944
982
  }
945
983
  }
946
984
  }
985
+ if (
986
+ inf.output.lmHeadBiasTensor !== undefined
987
+ && inf.output.lmHeadBiasTensor !== null
988
+ && (typeof inf.output.lmHeadBiasTensor !== 'string' || inf.output.lmHeadBiasTensor.trim() === '')
989
+ ) {
990
+ errors.push('output.lmHeadBiasTensor must be null or a non-empty tensor name');
991
+ }
992
+
993
+ if (inf.supportsSequence !== undefined && typeof inf.supportsSequence !== 'boolean') {
994
+ errors.push('supportsSequence must be boolean when provided');
995
+ }
996
+ if (inf.sequence !== undefined && inf.sequence !== null) {
997
+ const sequence = inf.sequence;
998
+ if (!sequence || typeof sequence !== 'object' || Array.isArray(sequence)) {
999
+ errors.push('sequence must be null or an object');
1000
+ } else {
1001
+ if (sequence.alphabet !== 'amino_acid' && sequence.alphabet !== 'nucleotide') {
1002
+ errors.push('sequence.alphabet must be "amino_acid" or "nucleotide"');
1003
+ }
1004
+ if (typeof sequence.tokenEmbeddings !== 'boolean') {
1005
+ errors.push('sequence.tokenEmbeddings must be boolean');
1006
+ }
1007
+ if (typeof sequence.logits !== 'boolean') {
1008
+ errors.push('sequence.logits must be boolean');
1009
+ }
1010
+ if (sequence.pooledEmbedding !== null) {
1011
+ const pooling = sequence.pooledEmbedding;
1012
+ if (!pooling || typeof pooling !== 'object' || Array.isArray(pooling)) {
1013
+ errors.push('sequence.pooledEmbedding must be null or an object');
1014
+ } else {
1015
+ if (pooling.mode !== 'mean' && pooling.mode !== 'last') {
1016
+ errors.push('sequence.pooledEmbedding.mode must be "mean" or "last"');
1017
+ }
1018
+ if (!Array.isArray(pooling.excludeTokenIds) || pooling.excludeTokenIds.some(
1019
+ (tokenId) => !Number.isInteger(tokenId) || tokenId < 0
1020
+ )) {
1021
+ errors.push('sequence.pooledEmbedding.excludeTokenIds must be an array of non-negative integers');
1022
+ }
1023
+ }
1024
+ }
1025
+ }
1026
+ }
1027
+ if (inf.supportsSequence === true && (inf.sequence == null || typeof inf.sequence !== 'object')) {
1028
+ errors.push('sequence is required when supportsSequence=true');
1029
+ }
1030
+ if (inf.supportsSequence !== true && inf.sequence != null) {
1031
+ errors.push('supportsSequence must be true when sequence is configured');
1032
+ }
947
1033
 
948
1034
  // Layer pattern fields
949
1035
  if (inf.layerPattern?.type == null) {
@@ -1843,6 +1929,8 @@ function toParsedConfigFromMerged(merged, manifest) {
1843
1929
  ropeLocalScaling,
1844
1930
  quantization: manifest.quantization,
1845
1931
  quantMethod: config.quantization_config?.quant_method ?? null,
1932
+ normalizationType: inf.normalization.type ?? 'rmsnorm',
1933
+ finalNormBiasTensor: inf.normalization.finalNormBiasTensor ?? null,
1846
1934
  rmsNormEps: inf.normalization.rmsNormEps,
1847
1935
  rmsNormWeightOffset: inf.normalization.rmsNormWeightOffset,
1848
1936
  postAttentionNorm: inf.normalization.postAttentionNorm,
@@ -1857,6 +1945,7 @@ function toParsedConfigFromMerged(merged, manifest) {
1857
1945
  embeddingVocabSize: inf.output.embeddingVocabSize,
1858
1946
  embeddingPostprocessor,
1859
1947
  hiddenActivation,
1948
+ gatedActivation: inf.ffn.gatedActivation,
1860
1949
  ffnBranchMode: inf.ffn.branchMode,
1861
1950
  useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
1862
1951
  swigluLimit: inf.ffn.swigluLimit,
@@ -1873,6 +1962,8 @@ function toParsedConfigFromMerged(merged, manifest) {
1873
1962
  finalLogitSoftcapping: inf.output.finalLogitSoftcapping,
1874
1963
  attnLogitSoftcapping: inf.attention.attnLogitSoftcapping,
1875
1964
  queryKeyNorm: inf.attention.queryKeyNorm,
1965
+ queryKeyNormType: inf.attention.queryKeyNormType ?? 'rmsnorm',
1966
+ queryKeyNormAxis: inf.attention.queryKeyNormAxis ?? 'head',
1876
1967
  queryKeyNormLayers: inf.attention.queryKeyNormLayers ?? null,
1877
1968
  queryKeyNormWeightLayers: inf.attention.queryKeyNormWeightLayers ?? null,
1878
1969
  valueNorm: inf.attention.valueNorm,
@@ -1,7 +1,7 @@
1
1
 
2
2
 
3
3
  import {
4
- doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
4
+ doBiasAdd, doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
5
5
  releaseOrTrack
6
6
  } from '../ops.js';
7
7
  import { createTensor } from '../../../../gpu/tensor.js';
@@ -29,7 +29,7 @@ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
29
29
  import { isKernelDebugEnabled, dumpTokenVector, decodeReadback, getLogitsHealth, shouldDebugLayerOutput } from '../debug-utils.js';
30
30
  import { applyLoRA } from '../lora-apply.js';
31
31
  import { getLoRAModule } from '../lora.js';
32
- import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
32
+ import { getWeightBuffer, getNormWeightBuffer, getVectorTensor } from '../weights.js';
33
33
  import { runProbes } from '../probes.js';
34
34
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
35
35
  import {
@@ -49,6 +49,19 @@ function resolveActivationOp(hiddenActivation) {
49
49
  return selectRuleValue('inference', 'ffn', 'activationOp', { hiddenActivation });
50
50
  }
51
51
 
52
+ async function captureTrainingActivation(context, layerIdx, tensor, numTokens, hiddenSize) {
53
+ const capture = context.trainingCapture;
54
+ if (!capture || capture.layerIdx !== layerIdx || capture.stage !== 'ffn_act') return;
55
+ await capture.capture({
56
+ layerIdx,
57
+ stage: 'ffn_act',
58
+ tensor,
59
+ numTokens,
60
+ hiddenSize,
61
+ recorder: context.recorder ?? null,
62
+ });
63
+ }
64
+
52
65
  function hasQ4KMaterialization(weight) {
53
66
  return isWeightBuffer(weight) && !!weight.materializations?.q4k?.buffer;
54
67
  }
@@ -527,6 +540,189 @@ async function dispatchFusedGateUp({
527
540
  return fusedOutput;
528
541
  }
529
542
 
543
+ async function applyDenseProjectionBias(
544
+ tensor,
545
+ biasWeight,
546
+ label,
547
+ numTokens,
548
+ dim,
549
+ context,
550
+ layerIdx
551
+ ) {
552
+ if (!biasWeight) return tensor;
553
+ const { tensor: biasTensor, owned } = getVectorTensor(
554
+ biasWeight,
555
+ label,
556
+ dim,
557
+ context.weightConfig,
558
+ context.debugFlags
559
+ );
560
+ try {
561
+ return await doBiasAdd(tensor, biasTensor, numTokens, dim, {
562
+ label,
563
+ layerIdx,
564
+ executionPolicies: context.executionPolicies ?? null,
565
+ }, context.recorder);
566
+ } finally {
567
+ if (owned) releaseOrTrack(context.recorder, biasTensor.buffer, context.decodeBuffers);
568
+ }
569
+ }
570
+
571
+ async function runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, context, layerWeights) {
572
+ const { config, recorder } = context;
573
+ const { hiddenSize, hiddenActivation } = config;
574
+ const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
575
+ if (layerWeights.gate || layerWeights.gateUp) {
576
+ throw new Error(`Layer ${layerIdx} declares an ungated FFN but contains gate weights.`);
577
+ }
578
+ if (!layerWeights.up || !layerWeights.down) {
579
+ throw new Error(`Layer ${layerIdx} ungated FFN requires up_proj and down_proj weights.`);
580
+ }
581
+
582
+ const kernelPath = context.kernelPath ?? null;
583
+ const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
584
+ const ffnStepPrecision = context.ffnStepPrecision ?? null;
585
+ const upInputDtype = resolveMatmulStepDtype(
586
+ 'ffn_up', phase, layerIdx, kernelPath, inputTensor.dtype, 'inputDtype', ffnStepPrecision
587
+ );
588
+ const upOutputDtype = resolveMatmulStepDtype(
589
+ 'ffn_up', phase, layerIdx, kernelPath, inputTensor.dtype, 'outputDtype', ffnStepPrecision
590
+ );
591
+ const downInputDtype = resolveMatmulStepDtype(
592
+ 'ffn_down', phase, layerIdx, kernelPath, upOutputDtype, 'inputDtype', ffnStepPrecision
593
+ );
594
+ const downOutputDtype = resolveMatmulStepDtype(
595
+ 'ffn_down', phase, layerIdx, kernelPath, inputTensor.dtype, 'outputDtype', ffnStepPrecision
596
+ );
597
+
598
+ let upInput = inputTensor;
599
+ let upInputOwned = false;
600
+ if (upInputDtype && upInputDtype !== inputTensor.dtype) {
601
+ upInput = await coerceTensorDtype(inputTensor, upInputDtype, recorder, {
602
+ executionPolicies: context.executionPolicies ?? null,
603
+ op: 'ffn_up_input',
604
+ transitionDeclaredBy: 'step_precision',
605
+ });
606
+ upInputOwned = upInput !== inputTensor;
607
+ }
608
+
609
+ const upWeight = getWeightBuffer(layerWeights.up, 'ffn_up');
610
+ let upOutput = await doMatmul(
611
+ upInput,
612
+ upWeight,
613
+ numTokens,
614
+ intermediateSize,
615
+ hiddenSize,
616
+ {
617
+ transposeB: 'auto',
618
+ label: `L${layerIdx}.ffn_up`,
619
+ layerIdx,
620
+ kernelPath,
621
+ outputDtype: upOutputDtype,
622
+ role: 'ffn_up',
623
+ executionPolicies: context.executionPolicies ?? null,
624
+ },
625
+ recorder
626
+ );
627
+ const loraUp = getLoRAModule(context.lora ?? null, layerIdx, 'up_proj');
628
+ if (loraUp) {
629
+ const combined = await applyLoRA(
630
+ upInput,
631
+ upOutput,
632
+ loraUp,
633
+ { M: numTokens, N: intermediateSize, K: hiddenSize },
634
+ getWeightBuffer,
635
+ recorder,
636
+ { kernelPath }
637
+ );
638
+ if (combined.buffer !== upOutput.buffer) {
639
+ releaseOrTrack(recorder, upOutput.buffer, context.decodeBuffers);
640
+ upOutput = combined;
641
+ }
642
+ }
643
+ upOutput = await applyDenseProjectionBias(
644
+ upOutput,
645
+ layerWeights.upBias,
646
+ `L${layerIdx}.ffn_up_bias`,
647
+ numTokens,
648
+ intermediateSize,
649
+ context,
650
+ layerIdx
651
+ );
652
+ const activated = await dispatchActivation(hiddenActivation, upOutput, {
653
+ size: numTokens * intermediateSize,
654
+ gate: null,
655
+ label: `L${layerIdx}.ffn_activation`,
656
+ layerIdx,
657
+ }, recorder);
658
+
659
+ let downInput = activated;
660
+ let downInputOwned = false;
661
+ if (downInputDtype && activated.dtype !== downInputDtype) {
662
+ downInput = await coerceTensorDtype(activated, downInputDtype, recorder, {
663
+ executionPolicies: context.executionPolicies ?? null,
664
+ op: 'ffn_down_input',
665
+ transitionDeclaredBy: 'step_precision',
666
+ });
667
+ downInputOwned = downInput !== activated;
668
+ }
669
+ const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
670
+ let output = await doMatmul(
671
+ downInput,
672
+ downWeight,
673
+ numTokens,
674
+ hiddenSize,
675
+ intermediateSize,
676
+ {
677
+ transposeB: 'auto',
678
+ label: `L${layerIdx}.ffn_down`,
679
+ layerIdx,
680
+ kernelPath,
681
+ outputDtype: downOutputDtype,
682
+ role: 'ffn_down',
683
+ executionPolicies: context.executionPolicies ?? null,
684
+ },
685
+ recorder
686
+ );
687
+ const loraDown = getLoRAModule(context.lora ?? null, layerIdx, 'down_proj');
688
+ if (loraDown) {
689
+ const combined = await applyLoRA(
690
+ downInput,
691
+ output,
692
+ loraDown,
693
+ { M: numTokens, N: hiddenSize, K: intermediateSize },
694
+ getWeightBuffer,
695
+ recorder,
696
+ { kernelPath }
697
+ );
698
+ if (combined.buffer !== output.buffer) {
699
+ releaseOrTrack(recorder, output.buffer, context.decodeBuffers);
700
+ output = combined;
701
+ }
702
+ }
703
+ output = await applyDenseProjectionBias(
704
+ output,
705
+ layerWeights.downBias,
706
+ `L${layerIdx}.ffn_down_bias`,
707
+ numTokens,
708
+ hiddenSize,
709
+ context,
710
+ layerIdx
711
+ );
712
+
713
+ if (!isGpuBufferInstance(layerWeights.up) && !isWeightBuffer(layerWeights.up)) {
714
+ releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight, context.decodeBuffers);
715
+ }
716
+ if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
717
+ releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight, context.decodeBuffers);
718
+ }
719
+ if (upInputOwned) releaseOrTrack(recorder, upInput.buffer, context.decodeBuffers);
720
+ if (downInputOwned) releaseOrTrack(recorder, downInput.buffer, context.decodeBuffers);
721
+ releaseOrTrack(recorder, upOutput.buffer, context.decodeBuffers);
722
+ releaseOrTrack(recorder, activated.buffer, context.decodeBuffers);
723
+ return output;
724
+ }
725
+
530
726
 
531
727
  export async function runDenseFFNGPU(
532
728
  layerIdx,
@@ -548,6 +744,10 @@ export async function runDenseFFNGPU(
548
744
  const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
549
745
  const gateUpPathMode = resolveGateUpPathMode({ kernelPath, phase, layerIdx });
550
746
 
747
+ if (config.gatedActivation === false) {
748
+ return runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, context, layerWeights);
749
+ }
750
+
551
751
  if (layerWeights?.gateUp && layerWeights?.down) {
552
752
  const gateUpWeight = getWeightBuffer(layerWeights.gateUp, 'ffn_gate_up');
553
753
  const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
@@ -643,6 +843,7 @@ export async function runDenseFFNGPU(
643
843
  });
644
844
  }
645
845
  enqueueRecordedDenseHealth(context, layerIdx, 'ffn_act', activatedOutput, numTokens * intermediateSize);
846
+ await captureTrainingActivation(context, layerIdx, activatedOutput, numTokens, intermediateSize);
646
847
 
647
848
  if (recorder) {
648
849
  recorder.trackTemporaryBuffer(gateUpOutput.buffer);
@@ -869,6 +1070,7 @@ export async function runDenseFFNGPU(
869
1070
  dtype: fusedOutput.dtype,
870
1071
  });
871
1072
  enqueueRecordedDenseHealth(context, layerIdx, 'ffn_fused_gate_up', fusedOutput, numTokens * intermediateSize);
1073
+ await captureTrainingActivation(context, layerIdx, fusedOutput, numTokens, intermediateSize);
872
1074
 
873
1075
  let downInput = fusedOutput;
874
1076
  if (downInputDtype && fusedOutput.dtype !== downInputDtype) {
@@ -1282,6 +1484,7 @@ export async function runDenseFFNGPU(
1282
1484
  dtype: activatedOutput.dtype,
1283
1485
  });
1284
1486
  enqueueRecordedDenseHealth(context, layerIdx, 'ffn_act', activatedOutput, numTokens * intermediateSize);
1487
+ await captureTrainingActivation(context, layerIdx, activatedOutput, numTokens, intermediateSize);
1285
1488
 
1286
1489
  if (recorder) {
1287
1490
  recorder.trackTemporaryBuffer(gateOutput.buffer);
@@ -1,7 +1,15 @@
1
1
 
2
2
 
3
- import { doCast, doRMSNorm, doRMSNormStats, doResidualAdd, doResidualNextRMSNormPair, releaseOrTrack } from '../ops.js';
4
- import { getNormWeightBuffer } from '../weights.js';
3
+ import {
4
+ doCast,
5
+ doLayerNorm,
6
+ doRMSNorm,
7
+ doRMSNormStats,
8
+ doResidualAdd,
9
+ doResidualNextRMSNormPair,
10
+ releaseOrTrack,
11
+ } from '../ops.js';
12
+ import { getNormWeightBuffer, getVectorTensor } from '../weights.js';
5
13
  import { runProbes } from '../probes.js';
6
14
  import { isMoELayerLocal } from './types.js';
7
15
  import { runDenseFFNGPU } from './dense.js';
@@ -15,6 +23,7 @@ import { selectRuleValue } from '../../../../rules/rule-registry.js';
15
23
  import { getRuntimeConfig } from '../../../../config/runtime.js';
16
24
 
17
25
  function shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context }) {
26
+ if (context.config.normalizationType !== 'rmsnorm') return false;
18
27
  if (session?.usePostAttnNormFusedGateUp !== true) return false;
19
28
  if (numTokens !== 1) return false;
20
29
  if (postAttn?.dtype !== 'f32' || fusedResidualInput?.dtype !== 'f32') return false;
@@ -92,7 +101,44 @@ export async function processFFNStandard(
92
101
  releasePostAttnNormWeight = !isGpuBufferInstance(layerWeights.postAttnNorm) && !isWeightBuffer(layerWeights.postAttnNorm);
93
102
  const session = getRuntimeConfig().inference?.session;
94
103
 
95
- if (fusedResidualInput) {
104
+ if (config.normalizationType === 'layernorm') {
105
+ if (!layerWeights.postAttentionNormBias) {
106
+ throw new Error(`Layer ${layerIdx} requires post_attention_layernorm.bias for LayerNorm execution.`);
107
+ }
108
+ let normInput = postAttn;
109
+ if (fusedResidualInput) {
110
+ normInput = await doResidualAdd(postAttn, fusedResidualInput, size, recorder, {
111
+ label: `L${layerIdx}.attention_residual`,
112
+ layerIdx,
113
+ executionPolicies: context.executionPolicies ?? null,
114
+ });
115
+ prenormSumBuffer = normInput.buffer;
116
+ }
117
+ const { tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
118
+ layerWeights.postAttentionNormBias,
119
+ `L${layerIdx}.post_attn_norm_bias`,
120
+ hiddenSize,
121
+ weightConfig,
122
+ debugFlags
123
+ );
124
+ try {
125
+ normedTensor = await doLayerNorm(
126
+ normInput,
127
+ normWeightBuf,
128
+ normBiasTensor.buffer,
129
+ rmsNormEps,
130
+ {
131
+ batchSize: numTokens,
132
+ hiddenSize,
133
+ label: `L${layerIdx}.post_attn_norm`,
134
+ layerIdx,
135
+ },
136
+ recorder
137
+ );
138
+ } finally {
139
+ if (normBiasOwned) releaseOrTrack(recorder, normBiasTensor.buffer, decodeBuffers);
140
+ }
141
+ } else if (fusedResidualInput) {
96
142
  const bytesPerElement = postAttn.dtype === 'f16' ? 2 : 4;
97
143
  prenormSumBuffer = acquireBuffer(size * bytesPerElement, undefined, 'fused_prenorm_sum');
98
144
  if (shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context })) {
@@ -226,6 +272,7 @@ export async function processFFNStandard(
226
272
  : 1;
227
273
  if (
228
274
  nextInputNormFusion
275
+ && config.normalizationType === 'rmsnorm'
229
276
  && residualInput.dtype === 'f32'
230
277
  && residualTensor.dtype === 'f32'
231
278
  ) {
@@ -204,13 +204,21 @@ function getWeightBufferConfig(state) {
204
204
  export function getLogitsWeights(state) {
205
205
  const finalNorm = state.weights.get('final_norm');
206
206
  const lmHead = state.weights.get('lm_head');
207
+ const lmHeadBias = state.weights.get('lm_head_bias') ?? null;
208
+ const finalNormBias = state.weights.get('final_norm_bias') ?? null;
207
209
  if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array || isWeightBuffer(finalNorm))) {
208
210
  throw new Error('Final norm not found or invalid type');
209
211
  }
210
212
  if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
211
213
  throw new Error('LM head not found or invalid type');
212
214
  }
213
- return { finalNorm, lmHead };
215
+ if (lmHeadBias !== null && !(lmHeadBias instanceof Float32Array)) {
216
+ throw new Error('LM head bias is not a Float32Array');
217
+ }
218
+ if (finalNormBias !== null && !(finalNormBias instanceof Float32Array)) {
219
+ throw new Error('Final norm bias is not a Float32Array');
220
+ }
221
+ return { finalNorm, finalNormBias, lmHead, lmHeadBias };
214
222
  }
215
223
 
216
224
 
@@ -222,6 +230,8 @@ export function getLogitsConfig(state) {
222
230
  return {
223
231
  hiddenSize: config.hiddenSize,
224
232
  vocabSize: config.vocabSize,
233
+ normalizationType: config.normalizationType,
234
+ finalNormBiasTensor: config.finalNormBiasTensor,
225
235
  rmsNormEps: config.rmsNormEps,
226
236
  rmsNormWeightOffset: config.rmsNormWeightOffset,
227
237
  useTiedEmbeddings: state.useTiedEmbeddings,
@@ -1,5 +1,5 @@
1
1
  import { readBuffer } from '../../../memory/buffer-pool.js';
2
- import { matmulCPU, rmsNormCPU } from './logits/index.js';
2
+ import { layerNormCPU, matmulCPU, rmsNormCPU } from './logits/index.js';
3
3
  import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer, getBufferDtype } from '../../../gpu/weight-buffer.js';
4
4
  import { decodeReadback } from './debug-utils/index.js';
5
5
  import { resolveExecutionSessionPlan } from './execution-plan.js';
@@ -454,14 +454,13 @@ export async function getFinalNormWeights(state) {
454
454
  return weights;
455
455
  }
456
456
 
457
- export function extractEmbeddingFromHidden(
457
+ export function extractTokenEmbeddingsFromHidden(
458
458
  hiddenStates,
459
459
  numTokens,
460
460
  hiddenSize,
461
- embeddingMode,
462
461
  finalNormWeights,
463
462
  config,
464
- embeddingPostprocessor = null
463
+ finalNormBias = null
465
464
  ) {
466
465
  const expectedLength = numTokens * hiddenSize;
467
466
  if (hiddenStates.length !== expectedLength) {
@@ -470,16 +469,64 @@ export function extractEmbeddingFromHidden(
470
469
  );
471
470
  }
472
471
 
473
- const applyFinalNorm = (tokenIndex) => {
472
+ const tokenEmbeddings = new Float32Array(expectedLength);
473
+ let effectiveLayerNormBias = null;
474
+ if (config.normalizationType === 'layernorm') {
475
+ if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
476
+ throw new Error(
477
+ `[Pipeline] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
478
+ );
479
+ }
480
+ effectiveLayerNormBias = finalNormBias instanceof Float32Array
481
+ ? finalNormBias
482
+ : new Float32Array(hiddenSize);
483
+ if (effectiveLayerNormBias.length !== hiddenSize) {
484
+ throw new Error(
485
+ `[Pipeline] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveLayerNormBias.length}.`
486
+ );
487
+ }
488
+ }
489
+ for (let tokenIndex = 0; tokenIndex < numTokens; tokenIndex += 1) {
474
490
  const offset = tokenIndex * hiddenSize;
475
491
  const tokenHidden = hiddenStates.subarray(offset, offset + hiddenSize);
476
- return rmsNormCPU(
477
- tokenHidden,
478
- finalNormWeights,
479
- config.rmsNormEps,
480
- config.rmsNormWeightOffset
481
- );
482
- };
492
+ if (config.normalizationType === 'layernorm') {
493
+ tokenEmbeddings.set(layerNormCPU(
494
+ tokenHidden,
495
+ finalNormWeights,
496
+ effectiveLayerNormBias,
497
+ config.rmsNormEps
498
+ ), offset);
499
+ } else {
500
+ tokenEmbeddings.set(rmsNormCPU(
501
+ tokenHidden,
502
+ finalNormWeights,
503
+ config.rmsNormEps,
504
+ config.rmsNormWeightOffset
505
+ ), offset);
506
+ }
507
+ }
508
+ return tokenEmbeddings;
509
+ }
510
+
511
+ export function extractEmbeddingFromHidden(
512
+ hiddenStates,
513
+ numTokens,
514
+ hiddenSize,
515
+ embeddingMode,
516
+ finalNormWeights,
517
+ config,
518
+ embeddingPostprocessor = null,
519
+ normalizedTokenEmbeddings = null,
520
+ finalNormBias = null
521
+ ) {
522
+ const tokenEmbeddings = normalizedTokenEmbeddings ?? extractTokenEmbeddingsFromHidden(
523
+ hiddenStates,
524
+ numTokens,
525
+ hiddenSize,
526
+ finalNormWeights,
527
+ config,
528
+ finalNormBias
529
+ );
483
530
 
484
531
  const postprocessorConfig = config?.embeddingPostprocessor ?? null;
485
532
  const resolvedEmbeddingMode = postprocessorConfig?.poolingMode ?? embeddingMode;
@@ -491,13 +538,14 @@ export function extractEmbeddingFromHidden(
491
538
 
492
539
  let pooled;
493
540
  if (resolvedEmbeddingMode === 'last') {
494
- pooled = applyFinalNorm(numTokens - 1);
541
+ const offset = (numTokens - 1) * hiddenSize;
542
+ pooled = tokenEmbeddings.slice(offset, offset + hiddenSize);
495
543
  } else if (resolvedEmbeddingMode === 'mean') {
496
544
  pooled = new Float32Array(hiddenSize);
497
545
  for (let t = 0; t < numTokens; t++) {
498
- const tokenEmbedding = applyFinalNorm(t);
546
+ const offset = t * hiddenSize;
499
547
  for (let i = 0; i < hiddenSize; i++) {
500
- pooled[i] += tokenEmbedding[i];
548
+ pooled[i] += tokenEmbeddings[offset + i];
501
549
  }
502
550
  }
503
551
  const invTokens = numTokens > 0 ? (1 / numTokens) : 1;
@@ -1222,7 +1222,9 @@ export async function advanceWithTokenAndEmbedding(state, tokenId, opts, helpers
1222
1222
  embeddingMode,
1223
1223
  finalNormWeights,
1224
1224
  config,
1225
- state.embeddingPostprocessor
1225
+ state.embeddingPostprocessor,
1226
+ null,
1227
+ state.weights.get('final_norm_bias') ?? null
1226
1228
  );
1227
1229
  } finally {
1228
1230
  const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
@@ -9,7 +9,7 @@
9
9
 
10
10
  import type { CommandRecorder, ProfileTimings } from '../../../gpu/command-recorder.js';
11
11
  import type { PipelineState } from './state.js';
12
- import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext, WorkloadPhaseTiming } from './types.js';
12
+ import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext, WorkloadPhaseTiming, NativeLoRAPrefillOptions, NativeLoRAPrefillResult } from './types.js';
13
13
  import type { LogitsConfig, LogitsWeights } from './logits/index.js';
14
14
  import type { WeightBufferConfig } from './weights.js';
15
15
  import type { ChatMessage } from './chat-format.js';
@@ -65,6 +65,10 @@ export declare class PipelineGenerator {
65
65
  resetGenerationState(): void;
66
66
  resetToSeqLen(seqLen: number): void;
67
67
  prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
68
+ prefillForLoRATraining(
69
+ inputIds: readonly number[] | Int32Array | Uint32Array,
70
+ options: NativeLoRAPrefillOptions
71
+ ): Promise<NativeLoRAPrefillResult>;
68
72
  computeDiffusionGemmaCanvasLogits(
69
73
  args: DiffusionGemmaCanvasLogitsInput,
70
74
  options?: GenerateOptions & { __internalGenerate?: boolean }