doppler-gpu 0.4.8 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +62 -12
  3. package/models/catalog.json +21 -7
  4. package/package.json +25 -1
  5. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  6. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  7. package/src/config/kernels/backward-registry.json +13 -2
  8. package/src/config/kernels/kernel-ref-digests.js +8 -4
  9. package/src/config/kernels/registry.json +1035 -2677
  10. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  11. package/src/config/schema/training-metrics.schema.js +4 -0
  12. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  13. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  14. package/src/experimental/training/attention-backward.js +123 -55
  15. package/src/experimental/training/autograd.js +101 -5
  16. package/src/experimental/training/clip.js +6 -1
  17. package/src/experimental/training/datasets/text-pairs.js +80 -8
  18. package/src/experimental/training/distillation/student-fixture.js +470 -114
  19. package/src/experimental/training/lora-pipeline.js +202 -27
  20. package/src/experimental/training/lora.js +10 -4
  21. package/src/experimental/training/optimizer.js +9 -1
  22. package/src/experimental/training/runner.js +18 -4
  23. package/src/experimental/training/workloads.js +11 -0
  24. package/src/gpu/device.js +100 -32
  25. package/src/gpu/kernels/backward/adam.js +6 -4
  26. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  27. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  28. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  29. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  30. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  31. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  32. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  33. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  34. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  35. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  36. package/src/gpu/kernels/backward/index.d.ts +3 -0
  37. package/src/gpu/kernels/backward/index.js +3 -0
  38. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  39. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  40. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  41. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  42. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  43. package/src/gpu/kernels/backward/utils.js +9 -2
  44. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  45. package/src/gpu/kernels/index.d.ts +5 -0
  46. package/src/gpu/kernels/index.js +5 -0
  47. package/src/gpu/kernels/matmul-selection.js +5 -0
  48. package/src/gpu/kernels/silu.d.ts +1 -0
  49. package/src/gpu/kernels/silu.js +19 -4
  50. package/src/inference/pipelines/text/embed.js +1 -1
  51. package/src/inference/pipelines/text/execution-v1.js +8 -2
  52. package/src/inference/pipelines/text/ffn/dense.js +2 -0
  53. package/src/inference/pipelines/text/generator.js +29 -0
  54. package/src/inference/pipelines/text/lora-apply.js +4 -4
  55. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  56. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  57. package/src/inference/tokenizers/bundled.js +105 -32
  58. package/src/rules/kernels/matmul.rules.json +1 -0
  59. package/src/tooling/node-converter.js +20 -5
  60. package/src/tooling/runtime-optimization.d.ts +109 -0
  61. package/src/tooling/runtime-optimization.js +872 -0
  62. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  63. package/src/tooling-experimental-exports.shared.js +18 -0
  64. package/src/tooling-exports.shared.d.ts +19 -0
  65. package/src/tooling-exports.shared.js +11 -0
  66. package/src/version.js +1 -1
@@ -8,6 +8,8 @@ import {
8
8
  runResidualAdd,
9
9
  runRMSNorm,
10
10
  runRoPE,
11
+ runScale,
12
+ runSiLU,
11
13
  runSiLURowSplit,
12
14
  } from '../../../gpu/kernels/index.js';
13
15
  import { createTensor } from '../../../gpu/tensor.js';
@@ -17,6 +19,8 @@ import { OpType } from '../autograd.js';
17
19
  import { LoraAdapter } from '../lora.js';
18
20
  import { normalizeOptionalString } from './suite-data.js';
19
21
  import { LORA_MODULE_ALIASES } from '../../../inference/pipelines/text/lora.js';
22
+ import { resolveEmbeddingScale } from '../../../inference/pipelines/text/embed.js';
23
+ import { isSlidingLayerType } from '../../../inference/pipelines/text/attention/dispatch-params.js';
20
24
 
21
25
  const DISTILL_ADAPTER_TOP_K = 64;
22
26
  const DISTILL_STUDENT_GRAPH_PROJECTION = 'projection_head';
@@ -26,6 +30,8 @@ const TRANSFORMER_LORA_TARGET_MODULES = Object.freeze([
26
30
  'k_proj',
27
31
  'v_proj',
28
32
  'o_proj',
33
+ 'gate_proj',
34
+ 'up_proj',
29
35
  'gate_up_proj',
30
36
  'down_proj',
31
37
  ]);
@@ -56,6 +62,22 @@ function releaseTensor(tensor) {
56
62
  releaseBuffer(tensor.buffer);
57
63
  }
58
64
 
65
+ function tensorElementCount(shape) {
66
+ return shape.reduce((product, value) => product * value, 1);
67
+ }
68
+
69
+ async function recordTensorView(tape, input, shape, label) {
70
+ if (tensorElementCount(input.shape) !== tensorElementCount(shape)) {
71
+ throw new Error(`${label} cannot change tensor element count.`);
72
+ }
73
+ return tape.record(
74
+ OpType.RESHAPE,
75
+ (value) => createTensor(value.buffer, value.dtype, [...shape], label),
76
+ [input],
77
+ { shape: [...shape] }
78
+ );
79
+ }
80
+
59
81
  function toFloat32Array(values, label = 'values') {
60
82
  if (values instanceof Float32Array) return values;
61
83
  if (ArrayBuffer.isView(values)) {
@@ -140,11 +162,14 @@ function resolveTransformerLoraShape(moduleName, dims) {
140
162
  return { inDim: dims.hiddenSize, outDim: dims.numKVHeads * dims.headDim };
141
163
  }
142
164
  if (moduleName === 'o_proj') {
143
- return { inDim: dims.hiddenSize, outDim: dims.hiddenSize };
165
+ return { inDim: dims.attentionSize, outDim: dims.hiddenSize };
144
166
  }
145
167
  if (moduleName === 'gate_up_proj') {
146
168
  return { inDim: dims.hiddenSize, outDim: dims.intermediateSize * 2 };
147
169
  }
170
+ if (moduleName === 'gate_proj' || moduleName === 'up_proj') {
171
+ return { inDim: dims.hiddenSize, outDim: dims.intermediateSize };
172
+ }
148
173
  if (moduleName === 'down_proj') {
149
174
  return { inDim: dims.intermediateSize, outDim: dims.hiddenSize };
150
175
  }
@@ -184,7 +209,13 @@ function resolveTensorDtype(value, fallback = 'f32') {
184
209
  return normalized === 'f16' ? 'f16' : (normalized === 'f32' ? 'f32' : fallback);
185
210
  }
186
211
 
187
- async function ensureTrainableTensor(value, shape, label, ownedTrainables = null) {
212
+ async function ensureTrainableTensor(
213
+ value,
214
+ shape,
215
+ label,
216
+ ownedTrainables = null,
217
+ options = {}
218
+ ) {
188
219
  if (!value) {
189
220
  throw new Error(`Distill full-graph student missing required weight "${label}".`);
190
221
  }
@@ -194,6 +225,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
194
225
  }
195
226
  return tensor;
196
227
  };
228
+ const preserveF16 = options.preserveF16 === true;
197
229
  if (isWeightBuffer(value)) {
198
230
  if (value.dtype === 'f32') {
199
231
  return value;
@@ -201,6 +233,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
201
233
  if (value.dtype === 'f16') {
202
234
  const sourceShape = Array.isArray(value.shape) && value.shape.length > 0 ? value.shape : [...shape];
203
235
  const source = createTensor(value.buffer, 'f16', sourceShape, `${label}_source_f16`);
236
+ if (preserveF16) return source;
204
237
  const promoted = await castF16ToF32(source);
205
238
  return registerOwned(createTensor(promoted.buffer, 'f32', sourceShape, `${label}_trainable_f32`));
206
239
  }
@@ -212,6 +245,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
212
245
  const dtype = rawDtype === 'f16' ? 'f16' : 'f32';
213
246
  const tensor = createTensor(value, dtype, sourceShape, label);
214
247
  if (dtype === 'f16') {
248
+ if (preserveF16) return tensor;
215
249
  const promoted = await castF16ToF32(tensor);
216
250
  return registerOwned(createTensor(promoted.buffer, 'f32', sourceShape, `${label}_trainable_f32`));
217
251
  }
@@ -241,6 +275,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
241
275
  throw new Error(`Distill full-graph student weight "${label}" has non-typed f16 CPU data.`);
242
276
  }
243
277
  const source = makeTensorFromF16Bits(raw, sourceShape, `${label}_cpu_f16`);
278
+ if (preserveF16) return registerOwned(source);
244
279
  const promoted = await castF16ToF32(source);
245
280
  releaseTensor(source);
246
281
  return registerOwned(createTensor(promoted.buffer, 'f32', sourceShape, `${label}_trainable_f32`));
@@ -256,6 +291,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
256
291
  label
257
292
  );
258
293
  if (tensor.dtype === 'f16') {
294
+ if (preserveF16) return tensor;
259
295
  const promoted = await castF16ToF32(tensor);
260
296
  return registerOwned(createTensor(promoted.buffer, 'f32', resolvedShape, `${label}_trainable_f32`));
261
297
  }
@@ -312,40 +348,6 @@ function resolveLayerFfnIntermediateSize(layerIdx, weights, fallback) {
312
348
  throw new Error(`Distill full-graph student cannot resolve FFN size for layer ${layerIdx}.`);
313
349
  }
314
350
 
315
- async function fuseGateUpTensors(gateTensor, upTensor, intermediateSize, hiddenSize, label, ownedTrainables = null) {
316
- const device = getDevice();
317
- if (!device) {
318
- throw new Error('Distill full-graph student requires active GPU device.');
319
- }
320
- if (gateTensor?.dtype !== 'f32' || upTensor?.dtype !== 'f32') {
321
- throw new Error(`Distill fused gate_up expects f32 tensors for "${label}".`);
322
- }
323
- const expectedRows = intermediateSize;
324
- const expectedCols = hiddenSize;
325
- const gateRows = Number.isFinite(gateTensor?.shape?.[0]) ? gateTensor.shape[0] : 0;
326
- const gateCols = Number.isFinite(gateTensor?.shape?.[1]) ? gateTensor.shape[1] : 0;
327
- const upRows = Number.isFinite(upTensor?.shape?.[0]) ? upTensor.shape[0] : 0;
328
- const upCols = Number.isFinite(upTensor?.shape?.[1]) ? upTensor.shape[1] : 0;
329
- if (gateRows !== expectedRows || gateCols !== expectedCols || upRows !== expectedRows || upCols !== expectedCols) {
330
- throw new Error(
331
- `Distill gate/up shape mismatch for "${label}": gate=[${gateRows},${gateCols}] up=[${upRows},${upCols}] ` +
332
- `expected=[${expectedRows},${expectedCols}]`
333
- );
334
- }
335
- const rowBytes = expectedCols * 4;
336
- const blockBytes = expectedRows * rowBytes;
337
- const fusedBuffer = acquireBuffer(blockBytes * 2, undefined, `${label}_fused`);
338
- const encoder = device.createCommandEncoder();
339
- encoder.copyBufferToBuffer(gateTensor.buffer, 0, fusedBuffer, 0, blockBytes);
340
- encoder.copyBufferToBuffer(upTensor.buffer, 0, fusedBuffer, blockBytes, blockBytes);
341
- device.queue.submit([encoder.finish()]);
342
- const fused = createTensor(fusedBuffer, 'f32', [expectedRows * 2, expectedCols], `${label}_fused`);
343
- if (ownedTrainables instanceof Set) {
344
- ownedTrainables.add(fused);
345
- }
346
- return fused;
347
- }
348
-
349
351
  function resolvePhasePrompts(batch, phase) {
350
352
  const distill = batch?.distill || {};
351
353
  const prompts = phase === 'positive'
@@ -522,8 +524,10 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
522
524
  const swigluLimit = Number.isFinite(modelConfig.swigluLimit) ? modelConfig.swigluLimit : 0;
523
525
  const useEmbeddingTranspose = modelConfig.embeddingTranspose === true;
524
526
  const tieWordEmbeddings = modelConfig.useTiedEmbeddings === true;
527
+ const embeddingScale = resolveEmbeddingScale(modelConfig, hiddenSize);
525
528
  const loraConfig = normalizeTransformerLoraConfig(options.loraAdapter || null);
526
529
  const freezeBaseGrad = Boolean(loraConfig);
530
+ const frozenWeightOptions = { preserveF16: freezeBaseGrad };
527
531
  const stopBaseWeight = freezeBaseGrad ? { stopGradInputs: [1] } : {};
528
532
  const stopRopeWeights = freezeBaseGrad ? { stopGradInputs: [1, 2] } : {};
529
533
 
@@ -542,7 +546,8 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
542
546
  studentPipeline.weights.get('embed'),
543
547
  [vocabSize, hiddenSize],
544
548
  'embed',
545
- ownedTrainables
549
+ ownedTrainables,
550
+ frozenWeightOptions
546
551
  );
547
552
  const lmHeadWeight = tieWordEmbeddings
548
553
  ? embeddingWeight
@@ -550,7 +555,8 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
550
555
  studentPipeline.weights.get('lm_head'),
551
556
  [vocabSize, hiddenSize],
552
557
  'lm_head',
553
- ownedTrainables
558
+ ownedTrainables,
559
+ frozenWeightOptions
554
560
  );
555
561
  const finalNormWeight = await ensureNormTensor(
556
562
  studentPipeline.weights.get('final_norm'),
@@ -573,6 +579,30 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
573
579
  'rope_sin',
574
580
  ownedTrainables
575
581
  );
582
+ const hasLocalAttention = Array.isArray(modelConfig.layerTypes)
583
+ && modelConfig.layerTypes.some((layerType) => isSlidingLayerType(layerType));
584
+ let ropeLocalCos = null;
585
+ let ropeLocalSin = null;
586
+ if (hasLocalAttention) {
587
+ if (!(studentPipeline.ropeLocalCos instanceof GPUBuffer)
588
+ || !(studentPipeline.ropeLocalSin instanceof GPUBuffer)) {
589
+ throw new Error(
590
+ 'Distill full-graph student requires local RoPE tables for sliding-attention layers.'
591
+ );
592
+ }
593
+ ropeLocalCos = await ensureTrainableTensor(
594
+ createTensor(studentPipeline.ropeLocalCos, 'f32', [ropeRows, ropeDim], 'rope_local_cos'),
595
+ [ropeRows, ropeDim],
596
+ 'rope_local_cos',
597
+ ownedTrainables
598
+ );
599
+ ropeLocalSin = await ensureTrainableTensor(
600
+ createTensor(studentPipeline.ropeLocalSin, 'f32', [ropeRows, ropeDim], 'rope_local_sin'),
601
+ [ropeRows, ropeDim],
602
+ 'rope_local_sin',
603
+ ownedTrainables
604
+ );
605
+ }
576
606
 
577
607
  const layerParams = [];
578
608
  const loraParams = [];
@@ -583,6 +613,7 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
583
613
  numHeads,
584
614
  numKVHeads,
585
615
  headDim,
616
+ attentionSize: numHeads * headDim,
586
617
  };
587
618
  for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
588
619
  const layerWeights = studentPipeline.weights.get(`layer_${layerIdx}`);
@@ -592,12 +623,15 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
592
623
  const layerIntermediateSize = resolveLayerFfnIntermediateSize(layerIdx, layerWeights, intermediateSize);
593
624
  const gateUpWeight = layerWeights.gateUp || layerWeights.ffnGateUp || null;
594
625
  let layerGateUp = null;
626
+ let layerGate = null;
627
+ let layerUp = null;
595
628
  if (hasTensorPayload(gateUpWeight)) {
596
629
  layerGateUp = await ensureTrainableTensor(
597
630
  gateUpWeight,
598
631
  [layerIntermediateSize * 2, hiddenSize],
599
632
  `layer_${layerIdx}.ffn_gate_up`,
600
- ownedTrainables
633
+ ownedTrainables,
634
+ frozenWeightOptions
601
635
  );
602
636
  } else {
603
637
  const gateWeight = layerWeights.gate || layerWeights.ffnGate || null;
@@ -607,25 +641,19 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
607
641
  `Distill full-graph student missing gate/up projections on layer ${layerIdx}.`
608
642
  );
609
643
  }
610
- const gateTensor = await ensureTrainableTensor(
644
+ layerGate = await ensureTrainableTensor(
611
645
  gateWeight,
612
646
  [layerIntermediateSize, hiddenSize],
613
647
  `layer_${layerIdx}.ffn_gate`,
614
- ownedTrainables
648
+ ownedTrainables,
649
+ frozenWeightOptions
615
650
  );
616
- const upTensor = await ensureTrainableTensor(
651
+ layerUp = await ensureTrainableTensor(
617
652
  upWeight,
618
653
  [layerIntermediateSize, hiddenSize],
619
654
  `layer_${layerIdx}.ffn_up`,
620
- ownedTrainables
621
- );
622
- layerGateUp = await fuseGateUpTensors(
623
- gateTensor,
624
- upTensor,
625
- layerIntermediateSize,
626
- hiddenSize,
627
- `layer_${layerIdx}.ffn_gate_up`,
628
- ownedTrainables
655
+ ownedTrainables,
656
+ frozenWeightOptions
629
657
  );
630
658
  }
631
659
  const layer = {
@@ -635,29 +663,38 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
635
663
  `layer_${layerIdx}.input_norm`,
636
664
  ownedTrainables
637
665
  ),
666
+ queryKeyNorm: modelConfig.queryKeyNorm === true
667
+ && (!Array.isArray(modelConfig.queryKeyNormLayers)
668
+ || modelConfig.queryKeyNormLayers.includes(layerIdx)),
669
+ qNorm: null,
670
+ kNorm: null,
638
671
  qProj: await ensureTrainableTensor(
639
672
  layerWeights.qProj,
640
673
  [numHeads * headDim, hiddenSize],
641
674
  `layer_${layerIdx}.q_proj`,
642
- ownedTrainables
675
+ ownedTrainables,
676
+ frozenWeightOptions
643
677
  ),
644
678
  kProj: await ensureTrainableTensor(
645
679
  layerWeights.kProj,
646
680
  [numKVHeads * headDim, hiddenSize],
647
681
  `layer_${layerIdx}.k_proj`,
648
- ownedTrainables
682
+ ownedTrainables,
683
+ frozenWeightOptions
649
684
  ),
650
685
  vProj: await ensureTrainableTensor(
651
686
  layerWeights.vProj,
652
687
  [numKVHeads * headDim, hiddenSize],
653
688
  `layer_${layerIdx}.v_proj`,
654
- ownedTrainables
689
+ ownedTrainables,
690
+ frozenWeightOptions
655
691
  ),
656
692
  oProj: await ensureTrainableTensor(
657
693
  layerWeights.oProj,
658
- [hiddenSize, hiddenSize],
694
+ [hiddenSize, numHeads * headDim],
659
695
  `layer_${layerIdx}.o_proj`,
660
- ownedTrainables
696
+ ownedTrainables,
697
+ frozenWeightOptions
661
698
  ),
662
699
  postAttentionNorm: layerWeights.postAttentionNorm
663
700
  ? await ensureNormTensor(
@@ -667,12 +704,31 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
667
704
  ownedTrainables
668
705
  )
669
706
  : null,
707
+ preFeedforwardNorm: layerWeights.preFeedforwardNorm
708
+ ? await ensureNormTensor(
709
+ layerWeights.preFeedforwardNorm,
710
+ hiddenSize,
711
+ `layer_${layerIdx}.pre_feedforward_norm`,
712
+ ownedTrainables
713
+ )
714
+ : null,
715
+ postFeedforwardNorm: layerWeights.postFeedforwardNorm
716
+ ? await ensureNormTensor(
717
+ layerWeights.postFeedforwardNorm,
718
+ hiddenSize,
719
+ `layer_${layerIdx}.post_feedforward_norm`,
720
+ ownedTrainables
721
+ )
722
+ : null,
670
723
  gateUp: layerGateUp,
724
+ gate: layerGate,
725
+ up: layerUp,
671
726
  down: await ensureTrainableTensor(
672
727
  layerWeights.down || layerWeights.ffnDown,
673
728
  [hiddenSize, layerIntermediateSize],
674
729
  `layer_${layerIdx}.ffn_down`,
675
- ownedTrainables
730
+ ownedTrainables,
731
+ frozenWeightOptions
676
732
  ),
677
733
  intermediateSize: layerIntermediateSize,
678
734
  lora: createTransformerLoraAdapters(loraConfig, {
@@ -680,14 +736,71 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
680
736
  intermediateSize: layerIntermediateSize,
681
737
  }),
682
738
  };
739
+ if (layer.gateUp && (layer.lora.gate_proj || layer.lora.up_proj)) {
740
+ throw new Error(
741
+ `Layer ${layerIdx} has fused gate/up weights but separate gate_proj or up_proj LoRA targets.`
742
+ );
743
+ }
744
+ if (!layer.gateUp && layer.lora.gate_up_proj) {
745
+ throw new Error(
746
+ `Layer ${layerIdx} has separate gate/up weights but a fused gate_up_proj LoRA target.`
747
+ );
748
+ }
749
+ if (layer.queryKeyNorm) {
750
+ const weightedLayers = modelConfig.queryKeyNormWeightLayers;
751
+ const expectsWeightedNorm = !Array.isArray(weightedLayers) || weightedLayers.includes(layerIdx);
752
+ if (!expectsWeightedNorm) {
753
+ throw new Error(
754
+ `Distill full-graph student does not support unit-weight Q/K norm on layer ${layerIdx}.`
755
+ );
756
+ }
757
+ layer.qNorm = await ensureNormTensor(
758
+ layerWeights.qNorm,
759
+ headDim,
760
+ `layer_${layerIdx}.q_norm`,
761
+ ownedTrainables
762
+ );
763
+ layer.kNorm = await ensureNormTensor(
764
+ layerWeights.kNorm,
765
+ headDim,
766
+ `layer_${layerIdx}.k_norm`,
767
+ ownedTrainables
768
+ );
769
+ }
770
+ if (modelConfig.postAttentionNorm === true && !layer.postAttentionNorm) {
771
+ throw new Error(`Distill full-graph student missing post-attention norm on layer ${layerIdx}.`);
772
+ }
773
+ if (modelConfig.preFeedforwardNorm === true && !layer.preFeedforwardNorm) {
774
+ throw new Error(`Distill full-graph student missing pre-feedforward norm on layer ${layerIdx}.`);
775
+ }
776
+ if (modelConfig.postFeedforwardNorm === true && !layer.postFeedforwardNorm) {
777
+ throw new Error(`Distill full-graph student missing post-feedforward norm on layer ${layerIdx}.`);
778
+ }
683
779
  layers.push(layer);
684
- layerParams.push(layer.inputNorm, layer.qProj, layer.kProj, layer.vProj, layer.oProj, layer.gateUp, layer.down);
780
+ layerParams.push(
781
+ layer.inputNorm,
782
+ layer.qProj,
783
+ layer.kProj,
784
+ layer.vProj,
785
+ layer.oProj,
786
+ ...(layer.gateUp ? [layer.gateUp] : [layer.gate, layer.up]),
787
+ layer.down
788
+ );
685
789
  for (const adapter of Object.values(layer.lora)) {
686
790
  loraParams.push(adapter.A, adapter.B);
687
791
  }
688
792
  if (layer.postAttentionNorm) {
689
793
  layerParams.push(layer.postAttentionNorm);
690
794
  }
795
+ if (layer.preFeedforwardNorm) {
796
+ layerParams.push(layer.preFeedforwardNorm);
797
+ }
798
+ if (layer.postFeedforwardNorm) {
799
+ layerParams.push(layer.postFeedforwardNorm);
800
+ }
801
+ if (layer.qNorm) {
802
+ layerParams.push(layer.qNorm, layer.kNorm);
803
+ }
691
804
  }
692
805
 
693
806
  const encoderParams = [embeddingWeight, ...layerParams];
@@ -714,6 +827,17 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
714
827
  }
715
828
 
716
829
  async function runTransformerTokenTensor(tokenTensor, seqLen, tape, forwardOptions = {}) {
830
+ const captureStage = async (stage, tensor, layerIdx = null) => {
831
+ if (typeof forwardOptions.captureStage === 'function') {
832
+ await forwardOptions.captureStage({
833
+ stage,
834
+ layerIdx,
835
+ seqLen,
836
+ tensor,
837
+ });
838
+ }
839
+ return tensor;
840
+ };
717
841
  let hidden = await tape.record(
718
842
  OpType.EMBED,
719
843
  (indices, embeddings) => runGather(
@@ -738,6 +862,15 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
738
862
  ...stopBaseWeight,
739
863
  }
740
864
  );
865
+ if (embeddingScale !== 1) {
866
+ hidden = await tape.record(
867
+ OpType.SCALE,
868
+ (x) => runScale(x, embeddingScale, { count: seqLen * hiddenSize }),
869
+ [hidden],
870
+ { count: seqLen * hiddenSize, scale: embeddingScale }
871
+ );
872
+ }
873
+ await captureStage('embed.out', hidden);
741
874
 
742
875
  for (let layerIdx = 0; layerIdx < layers.length; layerIdx += 1) {
743
876
  const layer = layers[layerIdx];
@@ -750,8 +883,15 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
750
883
  rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
751
884
  }),
752
885
  [hidden, layer.inputNorm],
753
- { numTokens: seqLen, hiddenSize, eps: rmsNormEps, ...stopBaseWeight }
886
+ {
887
+ numTokens: seqLen,
888
+ hiddenSize,
889
+ eps: rmsNormEps,
890
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
891
+ ...stopBaseWeight,
892
+ }
754
893
  );
894
+ await captureStage('attn.post_input_norm', normed, layerIdx);
755
895
 
756
896
  let q2d = await tape.record(
757
897
  OpType.MATMUL,
@@ -807,23 +947,110 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
807
947
  { size: seqLen * numKVHeads * headDim }
808
948
  );
809
949
  }
950
+ await captureStage('attn.q_proj', q2d, layerIdx);
951
+ await captureStage('attn.k_proj', k2d, layerIdx);
952
+ await captureStage('attn.v_proj', v2d, layerIdx);
953
+
954
+ if (layer.queryKeyNorm) {
955
+ const qNormInput = await recordTensorView(
956
+ tape,
957
+ q2d,
958
+ [seqLen * numHeads, headDim],
959
+ `layer_${layerIdx}_q_norm_input`
960
+ );
961
+ const qNormed = await tape.record(
962
+ OpType.RMSNORM,
963
+ (x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
964
+ batchSize: seqLen * numHeads,
965
+ hiddenSize: headDim,
966
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
967
+ }),
968
+ [qNormInput, layer.qNorm],
969
+ {
970
+ numTokens: seqLen * numHeads,
971
+ hiddenSize: headDim,
972
+ eps: rmsNormEps,
973
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
974
+ ...stopBaseWeight,
975
+ }
976
+ );
977
+ q2d = await recordTensorView(
978
+ tape,
979
+ qNormed,
980
+ [seqLen, numHeads * headDim],
981
+ `layer_${layerIdx}_q_normed`
982
+ );
983
+ const kNormInput = await recordTensorView(
984
+ tape,
985
+ k2d,
986
+ [seqLen * numKVHeads, headDim],
987
+ `layer_${layerIdx}_k_norm_input`
988
+ );
989
+ const kNormed = await tape.record(
990
+ OpType.RMSNORM,
991
+ (x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
992
+ batchSize: seqLen * numKVHeads,
993
+ hiddenSize: headDim,
994
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
995
+ }),
996
+ [kNormInput, layer.kNorm],
997
+ {
998
+ numTokens: seqLen * numKVHeads,
999
+ hiddenSize: headDim,
1000
+ eps: rmsNormEps,
1001
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
1002
+ ...stopBaseWeight,
1003
+ }
1004
+ );
1005
+ k2d = await recordTensorView(
1006
+ tape,
1007
+ kNormed,
1008
+ [seqLen, numKVHeads * headDim],
1009
+ `layer_${layerIdx}_k_normed`
1010
+ );
1011
+ await captureStage('attn.q_norm', q2d, layerIdx);
1012
+ await captureStage('attn.k_norm', k2d, layerIdx);
1013
+ }
810
1014
 
811
- const q3d = createTensor(q2d.buffer, q2d.dtype, [seqLen, numHeads, headDim], `layer_${layerIdx}_q`);
812
- const k3d = createTensor(k2d.buffer, k2d.dtype, [seqLen, numKVHeads, headDim], `layer_${layerIdx}_k`);
813
- const v3d = createTensor(v2d.buffer, v2d.dtype, [seqLen, numKVHeads, headDim], `layer_${layerIdx}_v`);
1015
+ const q3d = await recordTensorView(
1016
+ tape,
1017
+ q2d,
1018
+ [seqLen, numHeads, headDim],
1019
+ `layer_${layerIdx}_q`
1020
+ );
1021
+ const k3d = await recordTensorView(
1022
+ tape,
1023
+ k2d,
1024
+ [seqLen, numKVHeads, headDim],
1025
+ `layer_${layerIdx}_k`
1026
+ );
1027
+ const v3d = await recordTensorView(
1028
+ tape,
1029
+ v2d,
1030
+ [seqLen, numKVHeads, headDim],
1031
+ `layer_${layerIdx}_v`
1032
+ );
1033
+ const useLocalRope = isSlidingLayerType(modelConfig.layerTypes?.[layerIdx]);
1034
+ const layerRopeCos = useLocalRope ? ropeLocalCos : ropeCos;
1035
+ const layerRopeSin = useLocalRope ? ropeLocalSin : ropeSin;
1036
+ if (!layerRopeCos || !layerRopeSin) {
1037
+ throw new Error(`Distill full-graph student missing RoPE tensors on layer ${layerIdx}.`);
1038
+ }
814
1039
 
815
1040
  const qRope = await tape.record(
816
1041
  OpType.ROPE,
817
1042
  (q, cos, sin) => runRoPE(q, cos, sin, seqLen, { numHeads, headDim, startPos: 0 }),
818
- [q3d, ropeCos, ropeSin],
1043
+ [q3d, layerRopeCos, layerRopeSin],
819
1044
  { seqLen, numHeads, headDim, startPos: 0, ...stopRopeWeights }
820
1045
  );
821
1046
  const kRope = await tape.record(
822
1047
  OpType.ROPE,
823
1048
  (k, cos, sin) => runRoPE(k, cos, sin, seqLen, { numHeads: numKVHeads, headDim, startPos: 0 }),
824
- [k3d, ropeCos, ropeSin],
1049
+ [k3d, layerRopeCos, layerRopeSin],
825
1050
  { seqLen, numHeads: numKVHeads, headDim, startPos: 0, ...stopRopeWeights }
826
1051
  );
1052
+ await captureStage('attn.q_rope', qRope, layerIdx);
1053
+ await captureStage('attn.k_rope', kRope, layerIdx);
827
1054
 
828
1055
  const attention = await tape.record(
829
1056
  OpType.ATTENTION,
@@ -836,23 +1063,38 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
836
1063
  scale: 1 / Math.sqrt(headDim),
837
1064
  }),
838
1065
  [qRope, kRope, v3d],
839
- { seqLen, numHeads, headDim, scale: 1 / Math.sqrt(headDim), causal: true, recomputeForward: true }
1066
+ {
1067
+ seqLen,
1068
+ numHeads,
1069
+ numKVHeads,
1070
+ headDim,
1071
+ scale: 1 / Math.sqrt(headDim),
1072
+ causal: true,
1073
+ recomputeForward: true,
1074
+ }
840
1075
  );
841
- const attention2d = createTensor(
842
- attention.buffer,
843
- attention.dtype,
844
- [seqLen, hiddenSize],
1076
+ await captureStage('attn.core_out', attention, layerIdx);
1077
+ const attention2d = await recordTensorView(
1078
+ tape,
1079
+ attention,
1080
+ [seqLen, numHeads * headDim],
845
1081
  `layer_${layerIdx}_attn_2d`
846
1082
  );
847
1083
 
848
1084
  let attentionOutput = await tape.record(
849
1085
  OpType.MATMUL,
850
- (x, w) => runMatmul(x, w, seqLen, hiddenSize, hiddenSize, {
1086
+ (x, w) => runMatmul(x, w, seqLen, hiddenSize, numHeads * headDim, {
851
1087
  transposeB: 'auto',
852
1088
  outputDtype: 'f32',
853
1089
  }),
854
1090
  [attention2d, layer.oProj],
855
- { M: seqLen, N: hiddenSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
1091
+ {
1092
+ M: seqLen,
1093
+ N: hiddenSize,
1094
+ K: numHeads * headDim,
1095
+ transposeB: 'auto',
1096
+ ...stopBaseWeight,
1097
+ }
856
1098
  );
857
1099
  if (layer.lora.o_proj) {
858
1100
  const delta = await layer.lora.o_proj.forward(attention2d, tape);
@@ -863,14 +1105,34 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
863
1105
  { size: seqLen * hiddenSize }
864
1106
  );
865
1107
  }
1108
+ await captureStage('attn.out', attentionOutput, layerIdx);
1109
+ const normalizedAttentionOutput = modelConfig.postAttentionNorm === true
1110
+ ? await tape.record(
1111
+ OpType.RMSNORM,
1112
+ (x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
1113
+ batchSize: seqLen,
1114
+ hiddenSize,
1115
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
1116
+ }),
1117
+ [attentionOutput, layer.postAttentionNorm],
1118
+ {
1119
+ numTokens: seqLen,
1120
+ hiddenSize,
1121
+ eps: rmsNormEps,
1122
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
1123
+ ...stopBaseWeight,
1124
+ }
1125
+ )
1126
+ : attentionOutput;
866
1127
  const postAttention = await tape.record(
867
1128
  OpType.RESIDUAL_ADD,
868
1129
  (a, b) => runResidualAdd(a, b, seqLen * hiddenSize),
869
- [attentionOutput, hidden],
1130
+ [normalizedAttentionOutput, hidden],
870
1131
  { size: seqLen * hiddenSize }
871
1132
  );
1133
+ await captureStage('attn.post_attn', postAttention, layerIdx);
872
1134
 
873
- const ffnInput = layer.postAttentionNorm
1135
+ const ffnInput = modelConfig.preFeedforwardNorm === true
874
1136
  ? await tape.record(
875
1137
  OpType.RMSNORM,
876
1138
  (x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
@@ -878,44 +1140,106 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
878
1140
  hiddenSize,
879
1141
  rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
880
1142
  }),
881
- [postAttention, layer.postAttentionNorm],
882
- { numTokens: seqLen, hiddenSize, eps: rmsNormEps, ...stopBaseWeight }
1143
+ [postAttention, layer.preFeedforwardNorm],
1144
+ {
1145
+ numTokens: seqLen,
1146
+ hiddenSize,
1147
+ eps: rmsNormEps,
1148
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
1149
+ ...stopBaseWeight,
1150
+ }
883
1151
  )
884
1152
  : postAttention;
885
- let gateUp = await tape.record(
886
- OpType.MATMUL,
887
- (x, w) => runMatmul(x, w, seqLen, layerIntermediateSize * 2, hiddenSize, {
888
- transposeB: 'auto',
889
- outputDtype: 'f32',
890
- }),
891
- [ffnInput, layer.gateUp],
892
- { M: seqLen, N: layerIntermediateSize * 2, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
893
- );
894
- if (layer.lora.gate_up_proj) {
895
- const delta = await layer.lora.gate_up_proj.forward(ffnInput, tape);
896
- gateUp = await tape.record(
897
- OpType.RESIDUAL_ADD,
898
- (a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize * 2),
899
- [gateUp, delta],
900
- { size: seqLen * layerIntermediateSize * 2 }
1153
+ await captureStage('ffn.in', ffnInput, layerIdx);
1154
+ let activated = null;
1155
+ if (layer.gateUp) {
1156
+ let gateUp = await tape.record(
1157
+ OpType.MATMUL,
1158
+ (x, w) => runMatmul(x, w, seqLen, layerIntermediateSize * 2, hiddenSize, {
1159
+ transposeB: 'auto',
1160
+ outputDtype: 'f32',
1161
+ }),
1162
+ [ffnInput, layer.gateUp],
1163
+ { M: seqLen, N: layerIntermediateSize * 2, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
901
1164
  );
902
- }
903
- const activated = await tape.record(
904
- OpType.SILU_ROWSPLIT,
905
- (x) => runSiLURowSplit(x, {
906
- numTokens: seqLen,
907
- dim: layerIntermediateSize,
908
- activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
909
- swigluLimit: hiddenActivation === 'gelu' ? null : swigluLimit,
910
- }),
911
- [gateUp],
912
- {
913
- numTokens: seqLen,
914
- dim: layerIntermediateSize,
915
- activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
916
- swigluLimit: hiddenActivation === 'gelu' ? 0 : swigluLimit,
1165
+ if (layer.lora.gate_up_proj) {
1166
+ const delta = await layer.lora.gate_up_proj.forward(ffnInput, tape);
1167
+ gateUp = await tape.record(
1168
+ OpType.RESIDUAL_ADD,
1169
+ (a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize * 2),
1170
+ [gateUp, delta],
1171
+ { size: seqLen * layerIntermediateSize * 2 }
1172
+ );
917
1173
  }
918
- );
1174
+ activated = await tape.record(
1175
+ OpType.SILU_ROWSPLIT,
1176
+ (x) => runSiLURowSplit(x, {
1177
+ numTokens: seqLen,
1178
+ dim: layerIntermediateSize,
1179
+ activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
1180
+ swigluLimit: hiddenActivation === 'gelu' ? null : swigluLimit,
1181
+ }),
1182
+ [gateUp],
1183
+ {
1184
+ numTokens: seqLen,
1185
+ dim: layerIntermediateSize,
1186
+ activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
1187
+ swigluLimit: hiddenActivation === 'gelu' ? 0 : swigluLimit,
1188
+ }
1189
+ );
1190
+ } else {
1191
+ if (hiddenActivation === 'gelu') {
1192
+ throw new Error('Split gate/up training currently requires SiLU activation.');
1193
+ }
1194
+ let gate = await tape.record(
1195
+ OpType.MATMUL,
1196
+ (x, w) => runMatmul(x, w, seqLen, layerIntermediateSize, hiddenSize, {
1197
+ transposeB: 'auto',
1198
+ outputDtype: 'f32',
1199
+ }),
1200
+ [ffnInput, layer.gate],
1201
+ { M: seqLen, N: layerIntermediateSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
1202
+ );
1203
+ let up = await tape.record(
1204
+ OpType.MATMUL,
1205
+ (x, w) => runMatmul(x, w, seqLen, layerIntermediateSize, hiddenSize, {
1206
+ transposeB: 'auto',
1207
+ outputDtype: 'f32',
1208
+ }),
1209
+ [ffnInput, layer.up],
1210
+ { M: seqLen, N: layerIntermediateSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
1211
+ );
1212
+ if (layer.lora.gate_proj) {
1213
+ const delta = await layer.lora.gate_proj.forward(ffnInput, tape);
1214
+ gate = await tape.record(
1215
+ OpType.RESIDUAL_ADD,
1216
+ (a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize),
1217
+ [gate, delta],
1218
+ { size: seqLen * layerIntermediateSize }
1219
+ );
1220
+ }
1221
+ if (layer.lora.up_proj) {
1222
+ const delta = await layer.lora.up_proj.forward(ffnInput, tape);
1223
+ up = await tape.record(
1224
+ OpType.RESIDUAL_ADD,
1225
+ (a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize),
1226
+ [up, delta],
1227
+ { size: seqLen * layerIntermediateSize }
1228
+ );
1229
+ }
1230
+ activated = await tape.record(
1231
+ OpType.SILU_GATED,
1232
+ (gateInput, upInput) => runSiLU(upInput, {
1233
+ size: seqLen * layerIntermediateSize,
1234
+ gate: gateInput,
1235
+ inputActivation: 'identity',
1236
+ swigluLimit,
1237
+ }),
1238
+ [gate, up],
1239
+ { count: seqLen * layerIntermediateSize, swigluLimit }
1240
+ );
1241
+ }
1242
+ await captureStage('ffn.act', activated, layerIdx);
919
1243
  let ffnOutput = await tape.record(
920
1244
  OpType.MATMUL,
921
1245
  (x, w) => runMatmul(x, w, seqLen, hiddenSize, layerIntermediateSize, {
@@ -934,14 +1258,35 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
934
1258
  { size: seqLen * hiddenSize }
935
1259
  );
936
1260
  }
1261
+ await captureStage('ffn.out', ffnOutput, layerIdx);
1262
+ const normalizedFfnOutput = modelConfig.postFeedforwardNorm === true
1263
+ ? await tape.record(
1264
+ OpType.RMSNORM,
1265
+ (x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
1266
+ batchSize: seqLen,
1267
+ hiddenSize,
1268
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
1269
+ }),
1270
+ [ffnOutput, layer.postFeedforwardNorm],
1271
+ {
1272
+ numTokens: seqLen,
1273
+ hiddenSize,
1274
+ eps: rmsNormEps,
1275
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
1276
+ ...stopBaseWeight,
1277
+ }
1278
+ )
1279
+ : ffnOutput;
937
1280
  hidden = await tape.record(
938
1281
  OpType.RESIDUAL_ADD,
939
1282
  (a, b) => runResidualAdd(a, b, seqLen * hiddenSize),
940
- [ffnOutput, postAttention],
1283
+ [normalizedFfnOutput, postAttention],
941
1284
  { size: seqLen * hiddenSize }
942
1285
  );
1286
+ await captureStage('layer.out', hidden, layerIdx);
943
1287
  }
944
1288
 
1289
+ await captureStage('final_norm.pre', hidden);
945
1290
  const finalHidden = await tape.record(
946
1291
  OpType.RMSNORM,
947
1292
  (x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
@@ -950,10 +1295,17 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
950
1295
  rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
951
1296
  }),
952
1297
  [hidden, finalNormWeight],
953
- { numTokens: seqLen, hiddenSize, eps: rmsNormEps, ...stopBaseWeight }
1298
+ {
1299
+ numTokens: seqLen,
1300
+ hiddenSize,
1301
+ eps: rmsNormEps,
1302
+ rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
1303
+ ...stopBaseWeight,
1304
+ }
954
1305
  );
1306
+ await captureStage('final_norm.out', finalHidden);
955
1307
  if (forwardOptions.logitsMode === 'all') {
956
- return tape.record(
1308
+ const logits = await tape.record(
957
1309
  OpType.MATMUL,
958
1310
  (x, w) => runMatmul(x, w, seqLen, vocabSize, hiddenSize, {
959
1311
  transposeB: 'auto',
@@ -962,6 +1314,8 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
962
1314
  [finalHidden, lmHeadWeight],
963
1315
  { M: seqLen, N: vocabSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
964
1316
  );
1317
+ await captureStage('logits.out', logits);
1318
+ return logits;
965
1319
  }
966
1320
  const lastHidden = await tape.record(
967
1321
  OpType.ROW_SLICE,
@@ -969,7 +1323,7 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
969
1323
  [finalHidden],
970
1324
  { rows: seqLen, cols: hiddenSize, rowIndex: seqLen - 1 }
971
1325
  );
972
- return tape.record(
1326
+ const logits = await tape.record(
973
1327
  OpType.MATMUL,
974
1328
  (x, w) => runMatmul(x, w, 1, vocabSize, hiddenSize, {
975
1329
  transposeB: 'auto',
@@ -978,11 +1332,13 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
978
1332
  [lastHidden, lmHeadWeight],
979
1333
  { M: 1, N: vocabSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
980
1334
  );
1335
+ await captureStage('logits.out', logits);
1336
+ return logits;
981
1337
  }
982
1338
 
983
- async function runTransformerPrompt(prompt, tape) {
1339
+ async function runTransformerPrompt(prompt, tape, forwardOptions = {}) {
984
1340
  const { tokenTensor, seqLen } = await buildPromptTokens(prompt);
985
- return runTransformerTokenTensor(tokenTensor, seqLen, tape);
1341
+ return runTransformerTokenTensor(tokenTensor, seqLen, tape, forwardOptions);
986
1342
  }
987
1343
 
988
1344
  function collectLoraTensorEntries() {
@@ -1037,7 +1393,7 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
1037
1393
  `Distill full-graph student currently requires batchSize=1, got ${prompts.length}.`
1038
1394
  );
1039
1395
  }
1040
- const logits = await runTransformerPrompt(prompts[0], tape);
1396
+ const logits = await runTransformerPrompt(prompts[0], tape, forwardOptions);
1041
1397
  return { logits };
1042
1398
  },
1043
1399
  cleanupDistillStep() {