doppler-gpu 0.4.10 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +27 -0
  3. package/models/catalog.json +612 -7
  4. package/package.json +13 -1
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +126 -4
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +64 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/kernels/registry.json +52 -9
  18. package/src/config/merge.js +24 -0
  19. package/src/config/required-inference-fields-contract-check.js +2 -0
  20. package/src/config/schema/converter.schema.d.ts +7 -0
  21. package/src/config/schema/converter.schema.js +15 -0
  22. package/src/config/schema/inference.schema.d.ts +8 -0
  23. package/src/config/schema/manifest.schema.d.ts +32 -1
  24. package/src/config/schema/manifest.schema.js +26 -0
  25. package/src/converter/conversion-plan.js +15 -1
  26. package/src/converter/core.js +5 -4
  27. package/src/converter/source-tensor-rules.d.ts +50 -0
  28. package/src/converter/source-tensor-rules.js +180 -0
  29. package/src/converter/source-tokenizer.d.ts +45 -0
  30. package/src/converter/source-tokenizer.js +155 -0
  31. package/src/experimental/adapters/lora-loader.js +15 -0
  32. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  33. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  34. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  35. package/src/inference/pipelines/text/attention/projections.js +161 -24
  36. package/src/inference/pipelines/text/attention/record.js +67 -9
  37. package/src/inference/pipelines/text/attention/run.js +60 -7
  38. package/src/inference/pipelines/text/attention/types.js +18 -9
  39. package/src/inference/pipelines/text/config.d.ts +5 -0
  40. package/src/inference/pipelines/text/config.js +91 -0
  41. package/src/inference/pipelines/text/ffn/dense.js +189 -2
  42. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  43. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  44. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  45. package/src/inference/pipelines/text/generator-steps.js +3 -1
  46. package/src/inference/pipelines/text/generator.js +90 -3
  47. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  48. package/src/inference/pipelines/text/init.d.ts +2 -0
  49. package/src/inference/pipelines/text/init.js +2 -0
  50. package/src/inference/pipelines/text/layer.js +6 -0
  51. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  52. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  53. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  54. package/src/inference/pipelines/text/logits/index.js +85 -11
  55. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  56. package/src/inference/pipelines/text/ops.js +32 -0
  57. package/src/inference/pipelines/text/types.d.ts +38 -0
  58. package/src/inference/pipelines/text/weights.d.ts +10 -0
  59. package/src/inference/pipelines/text/weights.js +33 -1
  60. package/src/inference/pipelines/text.d.ts +4 -2
  61. package/src/inference/pipelines/text.js +112 -0
  62. package/src/inference/tokenizers/behavior-flags.js +5 -1
  63. package/src/inference/tokenizers/bundled.js +99 -8
  64. package/src/loader/doppler-loader.d.ts +2 -0
  65. package/src/loader/doppler-loader.js +9 -0
  66. package/src/loader/final-weights-loader.d.ts +8 -0
  67. package/src/loader/final-weights-loader.js +42 -0
  68. package/src/loader/layer-loader.d.ts +6 -0
  69. package/src/loader/layer-loader.js +102 -3
  70. package/src/loader/loader-types.d.ts +15 -2
  71. package/src/tooling/node-converter.js +33 -1
  72. package/src/tooling/node-webgpu.js +85 -2
  73. package/src/version.js +1 -1
@@ -1,7 +1,7 @@
1
1
 
2
2
 
3
3
  import {
4
- doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
4
+ doBiasAdd, doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
5
5
  releaseOrTrack
6
6
  } from '../ops.js';
7
7
  import { createTensor } from '../../../../gpu/tensor.js';
@@ -29,7 +29,7 @@ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
29
29
  import { isKernelDebugEnabled, dumpTokenVector, decodeReadback, getLogitsHealth, shouldDebugLayerOutput } from '../debug-utils.js';
30
30
  import { applyLoRA } from '../lora-apply.js';
31
31
  import { getLoRAModule } from '../lora.js';
32
- import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
32
+ import { getWeightBuffer, getNormWeightBuffer, getVectorTensor } from '../weights.js';
33
33
  import { runProbes } from '../probes.js';
34
34
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
35
35
  import {
@@ -527,6 +527,189 @@ async function dispatchFusedGateUp({
527
527
  return fusedOutput;
528
528
  }
529
529
 
530
+ async function applyDenseProjectionBias(
531
+ tensor,
532
+ biasWeight,
533
+ label,
534
+ numTokens,
535
+ dim,
536
+ context,
537
+ layerIdx
538
+ ) {
539
+ if (!biasWeight) return tensor;
540
+ const { tensor: biasTensor, owned } = getVectorTensor(
541
+ biasWeight,
542
+ label,
543
+ dim,
544
+ context.weightConfig,
545
+ context.debugFlags
546
+ );
547
+ try {
548
+ return await doBiasAdd(tensor, biasTensor, numTokens, dim, {
549
+ label,
550
+ layerIdx,
551
+ executionPolicies: context.executionPolicies ?? null,
552
+ }, context.recorder);
553
+ } finally {
554
+ if (owned) releaseOrTrack(context.recorder, biasTensor.buffer, context.decodeBuffers);
555
+ }
556
+ }
557
+
558
+ async function runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, context, layerWeights) {
559
+ const { config, recorder } = context;
560
+ const { hiddenSize, hiddenActivation } = config;
561
+ const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
562
+ if (layerWeights.gate || layerWeights.gateUp) {
563
+ throw new Error(`Layer ${layerIdx} declares an ungated FFN but contains gate weights.`);
564
+ }
565
+ if (!layerWeights.up || !layerWeights.down) {
566
+ throw new Error(`Layer ${layerIdx} ungated FFN requires up_proj and down_proj weights.`);
567
+ }
568
+
569
+ const kernelPath = context.kernelPath ?? null;
570
+ const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
571
+ const ffnStepPrecision = context.ffnStepPrecision ?? null;
572
+ const upInputDtype = resolveMatmulStepDtype(
573
+ 'ffn_up', phase, layerIdx, kernelPath, inputTensor.dtype, 'inputDtype', ffnStepPrecision
574
+ );
575
+ const upOutputDtype = resolveMatmulStepDtype(
576
+ 'ffn_up', phase, layerIdx, kernelPath, inputTensor.dtype, 'outputDtype', ffnStepPrecision
577
+ );
578
+ const downInputDtype = resolveMatmulStepDtype(
579
+ 'ffn_down', phase, layerIdx, kernelPath, upOutputDtype, 'inputDtype', ffnStepPrecision
580
+ );
581
+ const downOutputDtype = resolveMatmulStepDtype(
582
+ 'ffn_down', phase, layerIdx, kernelPath, inputTensor.dtype, 'outputDtype', ffnStepPrecision
583
+ );
584
+
585
+ let upInput = inputTensor;
586
+ let upInputOwned = false;
587
+ if (upInputDtype && upInputDtype !== inputTensor.dtype) {
588
+ upInput = await coerceTensorDtype(inputTensor, upInputDtype, recorder, {
589
+ executionPolicies: context.executionPolicies ?? null,
590
+ op: 'ffn_up_input',
591
+ transitionDeclaredBy: 'step_precision',
592
+ });
593
+ upInputOwned = upInput !== inputTensor;
594
+ }
595
+
596
+ const upWeight = getWeightBuffer(layerWeights.up, 'ffn_up');
597
+ let upOutput = await doMatmul(
598
+ upInput,
599
+ upWeight,
600
+ numTokens,
601
+ intermediateSize,
602
+ hiddenSize,
603
+ {
604
+ transposeB: 'auto',
605
+ label: `L${layerIdx}.ffn_up`,
606
+ layerIdx,
607
+ kernelPath,
608
+ outputDtype: upOutputDtype,
609
+ role: 'ffn_up',
610
+ executionPolicies: context.executionPolicies ?? null,
611
+ },
612
+ recorder
613
+ );
614
+ const loraUp = getLoRAModule(context.lora ?? null, layerIdx, 'up_proj');
615
+ if (loraUp) {
616
+ const combined = await applyLoRA(
617
+ upInput,
618
+ upOutput,
619
+ loraUp,
620
+ { M: numTokens, N: intermediateSize, K: hiddenSize },
621
+ getWeightBuffer,
622
+ recorder,
623
+ { kernelPath }
624
+ );
625
+ if (combined.buffer !== upOutput.buffer) {
626
+ releaseOrTrack(recorder, upOutput.buffer, context.decodeBuffers);
627
+ upOutput = combined;
628
+ }
629
+ }
630
+ upOutput = await applyDenseProjectionBias(
631
+ upOutput,
632
+ layerWeights.upBias,
633
+ `L${layerIdx}.ffn_up_bias`,
634
+ numTokens,
635
+ intermediateSize,
636
+ context,
637
+ layerIdx
638
+ );
639
+ const activated = await dispatchActivation(hiddenActivation, upOutput, {
640
+ size: numTokens * intermediateSize,
641
+ gate: null,
642
+ label: `L${layerIdx}.ffn_activation`,
643
+ layerIdx,
644
+ }, recorder);
645
+
646
+ let downInput = activated;
647
+ let downInputOwned = false;
648
+ if (downInputDtype && activated.dtype !== downInputDtype) {
649
+ downInput = await coerceTensorDtype(activated, downInputDtype, recorder, {
650
+ executionPolicies: context.executionPolicies ?? null,
651
+ op: 'ffn_down_input',
652
+ transitionDeclaredBy: 'step_precision',
653
+ });
654
+ downInputOwned = downInput !== activated;
655
+ }
656
+ const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
657
+ let output = await doMatmul(
658
+ downInput,
659
+ downWeight,
660
+ numTokens,
661
+ hiddenSize,
662
+ intermediateSize,
663
+ {
664
+ transposeB: 'auto',
665
+ label: `L${layerIdx}.ffn_down`,
666
+ layerIdx,
667
+ kernelPath,
668
+ outputDtype: downOutputDtype,
669
+ role: 'ffn_down',
670
+ executionPolicies: context.executionPolicies ?? null,
671
+ },
672
+ recorder
673
+ );
674
+ const loraDown = getLoRAModule(context.lora ?? null, layerIdx, 'down_proj');
675
+ if (loraDown) {
676
+ const combined = await applyLoRA(
677
+ downInput,
678
+ output,
679
+ loraDown,
680
+ { M: numTokens, N: hiddenSize, K: intermediateSize },
681
+ getWeightBuffer,
682
+ recorder,
683
+ { kernelPath }
684
+ );
685
+ if (combined.buffer !== output.buffer) {
686
+ releaseOrTrack(recorder, output.buffer, context.decodeBuffers);
687
+ output = combined;
688
+ }
689
+ }
690
+ output = await applyDenseProjectionBias(
691
+ output,
692
+ layerWeights.downBias,
693
+ `L${layerIdx}.ffn_down_bias`,
694
+ numTokens,
695
+ hiddenSize,
696
+ context,
697
+ layerIdx
698
+ );
699
+
700
+ if (!isGpuBufferInstance(layerWeights.up) && !isWeightBuffer(layerWeights.up)) {
701
+ releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight, context.decodeBuffers);
702
+ }
703
+ if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
704
+ releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight, context.decodeBuffers);
705
+ }
706
+ if (upInputOwned) releaseOrTrack(recorder, upInput.buffer, context.decodeBuffers);
707
+ if (downInputOwned) releaseOrTrack(recorder, downInput.buffer, context.decodeBuffers);
708
+ releaseOrTrack(recorder, upOutput.buffer, context.decodeBuffers);
709
+ releaseOrTrack(recorder, activated.buffer, context.decodeBuffers);
710
+ return output;
711
+ }
712
+
530
713
 
531
714
  export async function runDenseFFNGPU(
532
715
  layerIdx,
@@ -548,6 +731,10 @@ export async function runDenseFFNGPU(
548
731
  const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
549
732
  const gateUpPathMode = resolveGateUpPathMode({ kernelPath, phase, layerIdx });
550
733
 
734
+ if (config.gatedActivation === false) {
735
+ return runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, context, layerWeights);
736
+ }
737
+
551
738
  if (layerWeights?.gateUp && layerWeights?.down) {
552
739
  const gateUpWeight = getWeightBuffer(layerWeights.gateUp, 'ffn_gate_up');
553
740
  const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
@@ -1,7 +1,15 @@
1
1
 
2
2
 
3
- import { doCast, doRMSNorm, doRMSNormStats, doResidualAdd, doResidualNextRMSNormPair, releaseOrTrack } from '../ops.js';
4
- import { getNormWeightBuffer } from '../weights.js';
3
+ import {
4
+ doCast,
5
+ doLayerNorm,
6
+ doRMSNorm,
7
+ doRMSNormStats,
8
+ doResidualAdd,
9
+ doResidualNextRMSNormPair,
10
+ releaseOrTrack,
11
+ } from '../ops.js';
12
+ import { getNormWeightBuffer, getVectorTensor } from '../weights.js';
5
13
  import { runProbes } from '../probes.js';
6
14
  import { isMoELayerLocal } from './types.js';
7
15
  import { runDenseFFNGPU } from './dense.js';
@@ -15,6 +23,7 @@ import { selectRuleValue } from '../../../../rules/rule-registry.js';
15
23
  import { getRuntimeConfig } from '../../../../config/runtime.js';
16
24
 
17
25
  function shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context }) {
26
+ if (context.config.normalizationType !== 'rmsnorm') return false;
18
27
  if (session?.usePostAttnNormFusedGateUp !== true) return false;
19
28
  if (numTokens !== 1) return false;
20
29
  if (postAttn?.dtype !== 'f32' || fusedResidualInput?.dtype !== 'f32') return false;
@@ -92,7 +101,44 @@ export async function processFFNStandard(
92
101
  releasePostAttnNormWeight = !isGpuBufferInstance(layerWeights.postAttnNorm) && !isWeightBuffer(layerWeights.postAttnNorm);
93
102
  const session = getRuntimeConfig().inference?.session;
94
103
 
95
- if (fusedResidualInput) {
104
+ if (config.normalizationType === 'layernorm') {
105
+ if (!layerWeights.postAttentionNormBias) {
106
+ throw new Error(`Layer ${layerIdx} requires post_attention_layernorm.bias for LayerNorm execution.`);
107
+ }
108
+ let normInput = postAttn;
109
+ if (fusedResidualInput) {
110
+ normInput = await doResidualAdd(postAttn, fusedResidualInput, size, recorder, {
111
+ label: `L${layerIdx}.attention_residual`,
112
+ layerIdx,
113
+ executionPolicies: context.executionPolicies ?? null,
114
+ });
115
+ prenormSumBuffer = normInput.buffer;
116
+ }
117
+ const { tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
118
+ layerWeights.postAttentionNormBias,
119
+ `L${layerIdx}.post_attn_norm_bias`,
120
+ hiddenSize,
121
+ weightConfig,
122
+ debugFlags
123
+ );
124
+ try {
125
+ normedTensor = await doLayerNorm(
126
+ normInput,
127
+ normWeightBuf,
128
+ normBiasTensor.buffer,
129
+ rmsNormEps,
130
+ {
131
+ batchSize: numTokens,
132
+ hiddenSize,
133
+ label: `L${layerIdx}.post_attn_norm`,
134
+ layerIdx,
135
+ },
136
+ recorder
137
+ );
138
+ } finally {
139
+ if (normBiasOwned) releaseOrTrack(recorder, normBiasTensor.buffer, decodeBuffers);
140
+ }
141
+ } else if (fusedResidualInput) {
96
142
  const bytesPerElement = postAttn.dtype === 'f16' ? 2 : 4;
97
143
  prenormSumBuffer = acquireBuffer(size * bytesPerElement, undefined, 'fused_prenorm_sum');
98
144
  if (shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context })) {
@@ -226,6 +272,7 @@ export async function processFFNStandard(
226
272
  : 1;
227
273
  if (
228
274
  nextInputNormFusion
275
+ && config.normalizationType === 'rmsnorm'
229
276
  && residualInput.dtype === 'f32'
230
277
  && residualTensor.dtype === 'f32'
231
278
  ) {
@@ -204,13 +204,21 @@ function getWeightBufferConfig(state) {
204
204
  export function getLogitsWeights(state) {
205
205
  const finalNorm = state.weights.get('final_norm');
206
206
  const lmHead = state.weights.get('lm_head');
207
+ const lmHeadBias = state.weights.get('lm_head_bias') ?? null;
208
+ const finalNormBias = state.weights.get('final_norm_bias') ?? null;
207
209
  if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array || isWeightBuffer(finalNorm))) {
208
210
  throw new Error('Final norm not found or invalid type');
209
211
  }
210
212
  if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
211
213
  throw new Error('LM head not found or invalid type');
212
214
  }
213
- return { finalNorm, lmHead };
215
+ if (lmHeadBias !== null && !(lmHeadBias instanceof Float32Array)) {
216
+ throw new Error('LM head bias is not a Float32Array');
217
+ }
218
+ if (finalNormBias !== null && !(finalNormBias instanceof Float32Array)) {
219
+ throw new Error('Final norm bias is not a Float32Array');
220
+ }
221
+ return { finalNorm, finalNormBias, lmHead, lmHeadBias };
214
222
  }
215
223
 
216
224
 
@@ -222,6 +230,8 @@ export function getLogitsConfig(state) {
222
230
  return {
223
231
  hiddenSize: config.hiddenSize,
224
232
  vocabSize: config.vocabSize,
233
+ normalizationType: config.normalizationType,
234
+ finalNormBiasTensor: config.finalNormBiasTensor,
225
235
  rmsNormEps: config.rmsNormEps,
226
236
  rmsNormWeightOffset: config.rmsNormWeightOffset,
227
237
  useTiedEmbeddings: state.useTiedEmbeddings,
@@ -1,5 +1,5 @@
1
1
  import { readBuffer } from '../../../memory/buffer-pool.js';
2
- import { matmulCPU, rmsNormCPU } from './logits/index.js';
2
+ import { layerNormCPU, matmulCPU, rmsNormCPU } from './logits/index.js';
3
3
  import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer, getBufferDtype } from '../../../gpu/weight-buffer.js';
4
4
  import { decodeReadback } from './debug-utils/index.js';
5
5
  import { resolveExecutionSessionPlan } from './execution-plan.js';
@@ -454,14 +454,13 @@ export async function getFinalNormWeights(state) {
454
454
  return weights;
455
455
  }
456
456
 
457
- export function extractEmbeddingFromHidden(
457
+ export function extractTokenEmbeddingsFromHidden(
458
458
  hiddenStates,
459
459
  numTokens,
460
460
  hiddenSize,
461
- embeddingMode,
462
461
  finalNormWeights,
463
462
  config,
464
- embeddingPostprocessor = null
463
+ finalNormBias = null
465
464
  ) {
466
465
  const expectedLength = numTokens * hiddenSize;
467
466
  if (hiddenStates.length !== expectedLength) {
@@ -470,16 +469,64 @@ export function extractEmbeddingFromHidden(
470
469
  );
471
470
  }
472
471
 
473
- const applyFinalNorm = (tokenIndex) => {
472
+ const tokenEmbeddings = new Float32Array(expectedLength);
473
+ let effectiveLayerNormBias = null;
474
+ if (config.normalizationType === 'layernorm') {
475
+ if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
476
+ throw new Error(
477
+ `[Pipeline] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
478
+ );
479
+ }
480
+ effectiveLayerNormBias = finalNormBias instanceof Float32Array
481
+ ? finalNormBias
482
+ : new Float32Array(hiddenSize);
483
+ if (effectiveLayerNormBias.length !== hiddenSize) {
484
+ throw new Error(
485
+ `[Pipeline] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveLayerNormBias.length}.`
486
+ );
487
+ }
488
+ }
489
+ for (let tokenIndex = 0; tokenIndex < numTokens; tokenIndex += 1) {
474
490
  const offset = tokenIndex * hiddenSize;
475
491
  const tokenHidden = hiddenStates.subarray(offset, offset + hiddenSize);
476
- return rmsNormCPU(
477
- tokenHidden,
478
- finalNormWeights,
479
- config.rmsNormEps,
480
- config.rmsNormWeightOffset
481
- );
482
- };
492
+ if (config.normalizationType === 'layernorm') {
493
+ tokenEmbeddings.set(layerNormCPU(
494
+ tokenHidden,
495
+ finalNormWeights,
496
+ effectiveLayerNormBias,
497
+ config.rmsNormEps
498
+ ), offset);
499
+ } else {
500
+ tokenEmbeddings.set(rmsNormCPU(
501
+ tokenHidden,
502
+ finalNormWeights,
503
+ config.rmsNormEps,
504
+ config.rmsNormWeightOffset
505
+ ), offset);
506
+ }
507
+ }
508
+ return tokenEmbeddings;
509
+ }
510
+
511
+ export function extractEmbeddingFromHidden(
512
+ hiddenStates,
513
+ numTokens,
514
+ hiddenSize,
515
+ embeddingMode,
516
+ finalNormWeights,
517
+ config,
518
+ embeddingPostprocessor = null,
519
+ normalizedTokenEmbeddings = null,
520
+ finalNormBias = null
521
+ ) {
522
+ const tokenEmbeddings = normalizedTokenEmbeddings ?? extractTokenEmbeddingsFromHidden(
523
+ hiddenStates,
524
+ numTokens,
525
+ hiddenSize,
526
+ finalNormWeights,
527
+ config,
528
+ finalNormBias
529
+ );
483
530
 
484
531
  const postprocessorConfig = config?.embeddingPostprocessor ?? null;
485
532
  const resolvedEmbeddingMode = postprocessorConfig?.poolingMode ?? embeddingMode;
@@ -491,13 +538,14 @@ export function extractEmbeddingFromHidden(
491
538
 
492
539
  let pooled;
493
540
  if (resolvedEmbeddingMode === 'last') {
494
- pooled = applyFinalNorm(numTokens - 1);
541
+ const offset = (numTokens - 1) * hiddenSize;
542
+ pooled = tokenEmbeddings.slice(offset, offset + hiddenSize);
495
543
  } else if (resolvedEmbeddingMode === 'mean') {
496
544
  pooled = new Float32Array(hiddenSize);
497
545
  for (let t = 0; t < numTokens; t++) {
498
- const tokenEmbedding = applyFinalNorm(t);
546
+ const offset = t * hiddenSize;
499
547
  for (let i = 0; i < hiddenSize; i++) {
500
- pooled[i] += tokenEmbedding[i];
548
+ pooled[i] += tokenEmbeddings[offset + i];
501
549
  }
502
550
  }
503
551
  const invTokens = numTokens > 0 ? (1 / numTokens) : 1;
@@ -1222,7 +1222,9 @@ export async function advanceWithTokenAndEmbedding(state, tokenId, opts, helpers
1222
1222
  embeddingMode,
1223
1223
  finalNormWeights,
1224
1224
  config,
1225
- state.embeddingPostprocessor
1225
+ state.embeddingPostprocessor,
1226
+ null,
1227
+ state.weights.get('final_norm_bias') ?? null
1226
1228
  );
1227
1229
  } finally {
1228
1230
  const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
@@ -63,6 +63,7 @@ import {
63
63
  resolveAdvanceEmbeddingMode,
64
64
  getFinalNormWeights,
65
65
  extractEmbeddingFromHidden,
66
+ extractTokenEmbeddingsFromHidden,
66
67
  } from './generator-runtime.js';
67
68
 
68
69
  import { resolveSamplingConfig } from './sampling-config.js';
@@ -642,7 +643,20 @@ export class PipelineGenerator {
642
643
  embeddingMode,
643
644
  finalNormWeights,
644
645
  config,
645
- this.#state.embeddingPostprocessor
646
+ this.#state.embeddingPostprocessor,
647
+ null,
648
+ this.#state.weights.get('final_norm_bias') ?? null
649
+ );
650
+ }
651
+
652
+ _extractTokenEmbeddingsFromHidden(hiddenStates, numTokens, hiddenSize, finalNormWeights, config) {
653
+ return extractTokenEmbeddingsFromHidden(
654
+ hiddenStates,
655
+ numTokens,
656
+ hiddenSize,
657
+ finalNormWeights,
658
+ config,
659
+ this.#state.weights.get('final_norm_bias') ?? null
646
660
  );
647
661
  }
648
662
 
@@ -2112,6 +2126,34 @@ export class PipelineGenerator {
2112
2126
  this.#state.stats.gpuTimePrefillMs = undefined;
2113
2127
  this.#state.stats.prefillProfileSteps = [];
2114
2128
  const opts = resolvePrefillEmbeddingOptions(this.#state, options);
2129
+ const diagnosticsEnabled = options?.diagnostics?.enabled === true
2130
+ || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
2131
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
2132
+ if (diagnosticsEnabled || tsirFixtureCfg) {
2133
+ const captureConfig = {
2134
+ ...createDefaultCaptureConfig(),
2135
+ enabled: true,
2136
+ defaultLevel: CAPTURE_LEVELS.SLICE,
2137
+ ...(options?.diagnostics?.captureConfig ?? {}),
2138
+ };
2139
+ validateCaptureConfig(captureConfig);
2140
+ this.#state.operatorDiagnostics = {
2141
+ enabled: diagnosticsEnabled === true,
2142
+ captureConfig,
2143
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
2144
+ modelHash: this.#state.manifest?.modelId ?? null,
2145
+ runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
2146
+ executionPlanHash: opts.executionPlan?.id ?? null,
2147
+ }) : null,
2148
+ tsirFixture: tsirFixtureCfg ? {
2149
+ dir: tsirFixtureCfg.dir,
2150
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
2151
+ ? tsirFixtureCfg.layerFilter
2152
+ : null,
2153
+ records: [],
2154
+ } : null,
2155
+ };
2156
+ }
2115
2157
  const prefillStartSeqLen = this.#state.currentSeqLen;
2116
2158
  const inputStart = performance.now();
2117
2159
  const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
@@ -2178,12 +2220,32 @@ export class PipelineGenerator {
2178
2220
  }
2179
2221
 
2180
2222
  let embedding;
2223
+ let tokenEmbeddings = null;
2224
+ let logits = null;
2181
2225
  let hiddenBytes = 0;
2182
2226
  let readbackMs = 0;
2183
2227
  let decodeHiddenMs = 0;
2184
2228
  let finalNormMs = 0;
2185
2229
  let extractMs = 0;
2230
+ let logitsMs = 0;
2186
2231
  try {
2232
+ if (options.__returnSequenceLogits === true) {
2233
+ const logitsStart = performance.now();
2234
+ logits = await computeLogits(
2235
+ currentHiddenBuffer,
2236
+ numTokens,
2237
+ getLogitsWeights(this.#state),
2238
+ getLogitsConfig(this.#state),
2239
+ this.#state.useGPU,
2240
+ this.#state.debugFlags,
2241
+ undefined,
2242
+ undefined,
2243
+ this.#state.runtimeConfig.shared.debug.probes,
2244
+ { lastPositionOnly: false },
2245
+ this.#state.operatorDiagnostics
2246
+ );
2247
+ logitsMs = performance.now() - logitsStart;
2248
+ }
2187
2249
  const hiddenSize = config.hiddenSize;
2188
2250
  hiddenBytes = numTokens * hiddenSize * activationBytes;
2189
2251
  const readbackStart = performance.now();
@@ -2199,13 +2261,25 @@ export class PipelineGenerator {
2199
2261
  const finalNormWeights = await this._getFinalNormWeights();
2200
2262
  finalNormMs = performance.now() - finalNormStart;
2201
2263
  const extractStart = performance.now();
2202
- embedding = this._extractEmbeddingFromHidden(
2264
+ if (options.__returnTokenEmbeddings === true) {
2265
+ tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
2266
+ hiddenStates,
2267
+ numTokens,
2268
+ hiddenSize,
2269
+ finalNormWeights,
2270
+ config
2271
+ );
2272
+ }
2273
+ embedding = extractEmbeddingFromHidden(
2203
2274
  hiddenStates,
2204
2275
  numTokens,
2205
2276
  hiddenSize,
2206
2277
  opts.embeddingMode,
2207
2278
  finalNormWeights,
2208
- config
2279
+ config,
2280
+ this.#state.embeddingPostprocessor,
2281
+ tokenEmbeddings,
2282
+ this.#state.weights.get('final_norm_bias') ?? null
2209
2283
  );
2210
2284
  extractMs = performance.now() - extractStart;
2211
2285
  } finally {
@@ -2222,6 +2296,7 @@ export class PipelineGenerator {
2222
2296
  decodeHiddenMs,
2223
2297
  finalNormMs,
2224
2298
  extractMs,
2299
+ logitsMs,
2225
2300
  hiddenBytes,
2226
2301
  tokens: numTokens,
2227
2302
  activationDtype,
@@ -2241,6 +2316,8 @@ export class PipelineGenerator {
2241
2316
  seqLen: this.#state.currentSeqLen,
2242
2317
  tokens: inputIds,
2243
2318
  embedding,
2319
+ tokenEmbeddings,
2320
+ logits,
2244
2321
  embeddingMode: opts.embeddingMode,
2245
2322
  phase,
2246
2323
  linearAttention: null,
@@ -2257,12 +2334,22 @@ export class PipelineGenerator {
2257
2334
  seqLen: this.#state.currentSeqLen,
2258
2335
  tokens: inputIds,
2259
2336
  embedding,
2337
+ tokenEmbeddings,
2338
+ logits,
2260
2339
  embeddingMode: opts.embeddingMode,
2261
2340
  phase,
2262
2341
  linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2263
2342
  };
2264
2343
  } finally {
2265
2344
  this._closeFinitenessFallbackWindow(opts);
2345
+ this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2346
+ ? {
2347
+ enabled: true,
2348
+ timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2349
+ recordCount: this.#state.operatorDiagnostics.emitter.length,
2350
+ }
2351
+ : null;
2352
+ this.#state.operatorDiagnostics = null;
2266
2353
  }
2267
2354
  }
2268
2355