doppler-gpu 0.4.8 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/models/catalog.json +21 -7
- package/package.json +8 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +8 -4
- package/src/config/kernels/registry.json +1035 -2677
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/workloads.js +11 -0
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/index.d.ts +5 -0
- package/src/gpu/kernels/index.js +5 -0
- package/src/gpu/kernels/matmul-selection.js +5 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-v1.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +2 -0
- package/src/inference/pipelines/text/generator.js +29 -0
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/tokenizers/bundled.js +105 -32
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/version.js +1 -1
|
@@ -8,6 +8,8 @@ import {
|
|
|
8
8
|
runResidualAdd,
|
|
9
9
|
runRMSNorm,
|
|
10
10
|
runRoPE,
|
|
11
|
+
runScale,
|
|
12
|
+
runSiLU,
|
|
11
13
|
runSiLURowSplit,
|
|
12
14
|
} from '../../../gpu/kernels/index.js';
|
|
13
15
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
@@ -17,6 +19,8 @@ import { OpType } from '../autograd.js';
|
|
|
17
19
|
import { LoraAdapter } from '../lora.js';
|
|
18
20
|
import { normalizeOptionalString } from './suite-data.js';
|
|
19
21
|
import { LORA_MODULE_ALIASES } from '../../../inference/pipelines/text/lora.js';
|
|
22
|
+
import { resolveEmbeddingScale } from '../../../inference/pipelines/text/embed.js';
|
|
23
|
+
import { isSlidingLayerType } from '../../../inference/pipelines/text/attention/dispatch-params.js';
|
|
20
24
|
|
|
21
25
|
const DISTILL_ADAPTER_TOP_K = 64;
|
|
22
26
|
const DISTILL_STUDENT_GRAPH_PROJECTION = 'projection_head';
|
|
@@ -26,6 +30,8 @@ const TRANSFORMER_LORA_TARGET_MODULES = Object.freeze([
|
|
|
26
30
|
'k_proj',
|
|
27
31
|
'v_proj',
|
|
28
32
|
'o_proj',
|
|
33
|
+
'gate_proj',
|
|
34
|
+
'up_proj',
|
|
29
35
|
'gate_up_proj',
|
|
30
36
|
'down_proj',
|
|
31
37
|
]);
|
|
@@ -56,6 +62,22 @@ function releaseTensor(tensor) {
|
|
|
56
62
|
releaseBuffer(tensor.buffer);
|
|
57
63
|
}
|
|
58
64
|
|
|
65
|
+
function tensorElementCount(shape) {
|
|
66
|
+
return shape.reduce((product, value) => product * value, 1);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function recordTensorView(tape, input, shape, label) {
|
|
70
|
+
if (tensorElementCount(input.shape) !== tensorElementCount(shape)) {
|
|
71
|
+
throw new Error(`${label} cannot change tensor element count.`);
|
|
72
|
+
}
|
|
73
|
+
return tape.record(
|
|
74
|
+
OpType.RESHAPE,
|
|
75
|
+
(value) => createTensor(value.buffer, value.dtype, [...shape], label),
|
|
76
|
+
[input],
|
|
77
|
+
{ shape: [...shape] }
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
59
81
|
function toFloat32Array(values, label = 'values') {
|
|
60
82
|
if (values instanceof Float32Array) return values;
|
|
61
83
|
if (ArrayBuffer.isView(values)) {
|
|
@@ -140,11 +162,14 @@ function resolveTransformerLoraShape(moduleName, dims) {
|
|
|
140
162
|
return { inDim: dims.hiddenSize, outDim: dims.numKVHeads * dims.headDim };
|
|
141
163
|
}
|
|
142
164
|
if (moduleName === 'o_proj') {
|
|
143
|
-
return { inDim: dims.
|
|
165
|
+
return { inDim: dims.attentionSize, outDim: dims.hiddenSize };
|
|
144
166
|
}
|
|
145
167
|
if (moduleName === 'gate_up_proj') {
|
|
146
168
|
return { inDim: dims.hiddenSize, outDim: dims.intermediateSize * 2 };
|
|
147
169
|
}
|
|
170
|
+
if (moduleName === 'gate_proj' || moduleName === 'up_proj') {
|
|
171
|
+
return { inDim: dims.hiddenSize, outDim: dims.intermediateSize };
|
|
172
|
+
}
|
|
148
173
|
if (moduleName === 'down_proj') {
|
|
149
174
|
return { inDim: dims.intermediateSize, outDim: dims.hiddenSize };
|
|
150
175
|
}
|
|
@@ -184,7 +209,13 @@ function resolveTensorDtype(value, fallback = 'f32') {
|
|
|
184
209
|
return normalized === 'f16' ? 'f16' : (normalized === 'f32' ? 'f32' : fallback);
|
|
185
210
|
}
|
|
186
211
|
|
|
187
|
-
async function ensureTrainableTensor(
|
|
212
|
+
async function ensureTrainableTensor(
|
|
213
|
+
value,
|
|
214
|
+
shape,
|
|
215
|
+
label,
|
|
216
|
+
ownedTrainables = null,
|
|
217
|
+
options = {}
|
|
218
|
+
) {
|
|
188
219
|
if (!value) {
|
|
189
220
|
throw new Error(`Distill full-graph student missing required weight "${label}".`);
|
|
190
221
|
}
|
|
@@ -194,6 +225,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
|
|
|
194
225
|
}
|
|
195
226
|
return tensor;
|
|
196
227
|
};
|
|
228
|
+
const preserveF16 = options.preserveF16 === true;
|
|
197
229
|
if (isWeightBuffer(value)) {
|
|
198
230
|
if (value.dtype === 'f32') {
|
|
199
231
|
return value;
|
|
@@ -201,6 +233,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
|
|
|
201
233
|
if (value.dtype === 'f16') {
|
|
202
234
|
const sourceShape = Array.isArray(value.shape) && value.shape.length > 0 ? value.shape : [...shape];
|
|
203
235
|
const source = createTensor(value.buffer, 'f16', sourceShape, `${label}_source_f16`);
|
|
236
|
+
if (preserveF16) return source;
|
|
204
237
|
const promoted = await castF16ToF32(source);
|
|
205
238
|
return registerOwned(createTensor(promoted.buffer, 'f32', sourceShape, `${label}_trainable_f32`));
|
|
206
239
|
}
|
|
@@ -212,6 +245,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
|
|
|
212
245
|
const dtype = rawDtype === 'f16' ? 'f16' : 'f32';
|
|
213
246
|
const tensor = createTensor(value, dtype, sourceShape, label);
|
|
214
247
|
if (dtype === 'f16') {
|
|
248
|
+
if (preserveF16) return tensor;
|
|
215
249
|
const promoted = await castF16ToF32(tensor);
|
|
216
250
|
return registerOwned(createTensor(promoted.buffer, 'f32', sourceShape, `${label}_trainable_f32`));
|
|
217
251
|
}
|
|
@@ -241,6 +275,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
|
|
|
241
275
|
throw new Error(`Distill full-graph student weight "${label}" has non-typed f16 CPU data.`);
|
|
242
276
|
}
|
|
243
277
|
const source = makeTensorFromF16Bits(raw, sourceShape, `${label}_cpu_f16`);
|
|
278
|
+
if (preserveF16) return registerOwned(source);
|
|
244
279
|
const promoted = await castF16ToF32(source);
|
|
245
280
|
releaseTensor(source);
|
|
246
281
|
return registerOwned(createTensor(promoted.buffer, 'f32', sourceShape, `${label}_trainable_f32`));
|
|
@@ -256,6 +291,7 @@ async function ensureTrainableTensor(value, shape, label, ownedTrainables = null
|
|
|
256
291
|
label
|
|
257
292
|
);
|
|
258
293
|
if (tensor.dtype === 'f16') {
|
|
294
|
+
if (preserveF16) return tensor;
|
|
259
295
|
const promoted = await castF16ToF32(tensor);
|
|
260
296
|
return registerOwned(createTensor(promoted.buffer, 'f32', resolvedShape, `${label}_trainable_f32`));
|
|
261
297
|
}
|
|
@@ -312,40 +348,6 @@ function resolveLayerFfnIntermediateSize(layerIdx, weights, fallback) {
|
|
|
312
348
|
throw new Error(`Distill full-graph student cannot resolve FFN size for layer ${layerIdx}.`);
|
|
313
349
|
}
|
|
314
350
|
|
|
315
|
-
async function fuseGateUpTensors(gateTensor, upTensor, intermediateSize, hiddenSize, label, ownedTrainables = null) {
|
|
316
|
-
const device = getDevice();
|
|
317
|
-
if (!device) {
|
|
318
|
-
throw new Error('Distill full-graph student requires active GPU device.');
|
|
319
|
-
}
|
|
320
|
-
if (gateTensor?.dtype !== 'f32' || upTensor?.dtype !== 'f32') {
|
|
321
|
-
throw new Error(`Distill fused gate_up expects f32 tensors for "${label}".`);
|
|
322
|
-
}
|
|
323
|
-
const expectedRows = intermediateSize;
|
|
324
|
-
const expectedCols = hiddenSize;
|
|
325
|
-
const gateRows = Number.isFinite(gateTensor?.shape?.[0]) ? gateTensor.shape[0] : 0;
|
|
326
|
-
const gateCols = Number.isFinite(gateTensor?.shape?.[1]) ? gateTensor.shape[1] : 0;
|
|
327
|
-
const upRows = Number.isFinite(upTensor?.shape?.[0]) ? upTensor.shape[0] : 0;
|
|
328
|
-
const upCols = Number.isFinite(upTensor?.shape?.[1]) ? upTensor.shape[1] : 0;
|
|
329
|
-
if (gateRows !== expectedRows || gateCols !== expectedCols || upRows !== expectedRows || upCols !== expectedCols) {
|
|
330
|
-
throw new Error(
|
|
331
|
-
`Distill gate/up shape mismatch for "${label}": gate=[${gateRows},${gateCols}] up=[${upRows},${upCols}] ` +
|
|
332
|
-
`expected=[${expectedRows},${expectedCols}]`
|
|
333
|
-
);
|
|
334
|
-
}
|
|
335
|
-
const rowBytes = expectedCols * 4;
|
|
336
|
-
const blockBytes = expectedRows * rowBytes;
|
|
337
|
-
const fusedBuffer = acquireBuffer(blockBytes * 2, undefined, `${label}_fused`);
|
|
338
|
-
const encoder = device.createCommandEncoder();
|
|
339
|
-
encoder.copyBufferToBuffer(gateTensor.buffer, 0, fusedBuffer, 0, blockBytes);
|
|
340
|
-
encoder.copyBufferToBuffer(upTensor.buffer, 0, fusedBuffer, blockBytes, blockBytes);
|
|
341
|
-
device.queue.submit([encoder.finish()]);
|
|
342
|
-
const fused = createTensor(fusedBuffer, 'f32', [expectedRows * 2, expectedCols], `${label}_fused`);
|
|
343
|
-
if (ownedTrainables instanceof Set) {
|
|
344
|
-
ownedTrainables.add(fused);
|
|
345
|
-
}
|
|
346
|
-
return fused;
|
|
347
|
-
}
|
|
348
|
-
|
|
349
351
|
function resolvePhasePrompts(batch, phase) {
|
|
350
352
|
const distill = batch?.distill || {};
|
|
351
353
|
const prompts = phase === 'positive'
|
|
@@ -522,8 +524,10 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
522
524
|
const swigluLimit = Number.isFinite(modelConfig.swigluLimit) ? modelConfig.swigluLimit : 0;
|
|
523
525
|
const useEmbeddingTranspose = modelConfig.embeddingTranspose === true;
|
|
524
526
|
const tieWordEmbeddings = modelConfig.useTiedEmbeddings === true;
|
|
527
|
+
const embeddingScale = resolveEmbeddingScale(modelConfig, hiddenSize);
|
|
525
528
|
const loraConfig = normalizeTransformerLoraConfig(options.loraAdapter || null);
|
|
526
529
|
const freezeBaseGrad = Boolean(loraConfig);
|
|
530
|
+
const frozenWeightOptions = { preserveF16: freezeBaseGrad };
|
|
527
531
|
const stopBaseWeight = freezeBaseGrad ? { stopGradInputs: [1] } : {};
|
|
528
532
|
const stopRopeWeights = freezeBaseGrad ? { stopGradInputs: [1, 2] } : {};
|
|
529
533
|
|
|
@@ -542,7 +546,8 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
542
546
|
studentPipeline.weights.get('embed'),
|
|
543
547
|
[vocabSize, hiddenSize],
|
|
544
548
|
'embed',
|
|
545
|
-
ownedTrainables
|
|
549
|
+
ownedTrainables,
|
|
550
|
+
frozenWeightOptions
|
|
546
551
|
);
|
|
547
552
|
const lmHeadWeight = tieWordEmbeddings
|
|
548
553
|
? embeddingWeight
|
|
@@ -550,7 +555,8 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
550
555
|
studentPipeline.weights.get('lm_head'),
|
|
551
556
|
[vocabSize, hiddenSize],
|
|
552
557
|
'lm_head',
|
|
553
|
-
ownedTrainables
|
|
558
|
+
ownedTrainables,
|
|
559
|
+
frozenWeightOptions
|
|
554
560
|
);
|
|
555
561
|
const finalNormWeight = await ensureNormTensor(
|
|
556
562
|
studentPipeline.weights.get('final_norm'),
|
|
@@ -573,6 +579,30 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
573
579
|
'rope_sin',
|
|
574
580
|
ownedTrainables
|
|
575
581
|
);
|
|
582
|
+
const hasLocalAttention = Array.isArray(modelConfig.layerTypes)
|
|
583
|
+
&& modelConfig.layerTypes.some((layerType) => isSlidingLayerType(layerType));
|
|
584
|
+
let ropeLocalCos = null;
|
|
585
|
+
let ropeLocalSin = null;
|
|
586
|
+
if (hasLocalAttention) {
|
|
587
|
+
if (!(studentPipeline.ropeLocalCos instanceof GPUBuffer)
|
|
588
|
+
|| !(studentPipeline.ropeLocalSin instanceof GPUBuffer)) {
|
|
589
|
+
throw new Error(
|
|
590
|
+
'Distill full-graph student requires local RoPE tables for sliding-attention layers.'
|
|
591
|
+
);
|
|
592
|
+
}
|
|
593
|
+
ropeLocalCos = await ensureTrainableTensor(
|
|
594
|
+
createTensor(studentPipeline.ropeLocalCos, 'f32', [ropeRows, ropeDim], 'rope_local_cos'),
|
|
595
|
+
[ropeRows, ropeDim],
|
|
596
|
+
'rope_local_cos',
|
|
597
|
+
ownedTrainables
|
|
598
|
+
);
|
|
599
|
+
ropeLocalSin = await ensureTrainableTensor(
|
|
600
|
+
createTensor(studentPipeline.ropeLocalSin, 'f32', [ropeRows, ropeDim], 'rope_local_sin'),
|
|
601
|
+
[ropeRows, ropeDim],
|
|
602
|
+
'rope_local_sin',
|
|
603
|
+
ownedTrainables
|
|
604
|
+
);
|
|
605
|
+
}
|
|
576
606
|
|
|
577
607
|
const layerParams = [];
|
|
578
608
|
const loraParams = [];
|
|
@@ -583,6 +613,7 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
583
613
|
numHeads,
|
|
584
614
|
numKVHeads,
|
|
585
615
|
headDim,
|
|
616
|
+
attentionSize: numHeads * headDim,
|
|
586
617
|
};
|
|
587
618
|
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
588
619
|
const layerWeights = studentPipeline.weights.get(`layer_${layerIdx}`);
|
|
@@ -592,12 +623,15 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
592
623
|
const layerIntermediateSize = resolveLayerFfnIntermediateSize(layerIdx, layerWeights, intermediateSize);
|
|
593
624
|
const gateUpWeight = layerWeights.gateUp || layerWeights.ffnGateUp || null;
|
|
594
625
|
let layerGateUp = null;
|
|
626
|
+
let layerGate = null;
|
|
627
|
+
let layerUp = null;
|
|
595
628
|
if (hasTensorPayload(gateUpWeight)) {
|
|
596
629
|
layerGateUp = await ensureTrainableTensor(
|
|
597
630
|
gateUpWeight,
|
|
598
631
|
[layerIntermediateSize * 2, hiddenSize],
|
|
599
632
|
`layer_${layerIdx}.ffn_gate_up`,
|
|
600
|
-
ownedTrainables
|
|
633
|
+
ownedTrainables,
|
|
634
|
+
frozenWeightOptions
|
|
601
635
|
);
|
|
602
636
|
} else {
|
|
603
637
|
const gateWeight = layerWeights.gate || layerWeights.ffnGate || null;
|
|
@@ -607,25 +641,19 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
607
641
|
`Distill full-graph student missing gate/up projections on layer ${layerIdx}.`
|
|
608
642
|
);
|
|
609
643
|
}
|
|
610
|
-
|
|
644
|
+
layerGate = await ensureTrainableTensor(
|
|
611
645
|
gateWeight,
|
|
612
646
|
[layerIntermediateSize, hiddenSize],
|
|
613
647
|
`layer_${layerIdx}.ffn_gate`,
|
|
614
|
-
ownedTrainables
|
|
648
|
+
ownedTrainables,
|
|
649
|
+
frozenWeightOptions
|
|
615
650
|
);
|
|
616
|
-
|
|
651
|
+
layerUp = await ensureTrainableTensor(
|
|
617
652
|
upWeight,
|
|
618
653
|
[layerIntermediateSize, hiddenSize],
|
|
619
654
|
`layer_${layerIdx}.ffn_up`,
|
|
620
|
-
ownedTrainables
|
|
621
|
-
|
|
622
|
-
layerGateUp = await fuseGateUpTensors(
|
|
623
|
-
gateTensor,
|
|
624
|
-
upTensor,
|
|
625
|
-
layerIntermediateSize,
|
|
626
|
-
hiddenSize,
|
|
627
|
-
`layer_${layerIdx}.ffn_gate_up`,
|
|
628
|
-
ownedTrainables
|
|
655
|
+
ownedTrainables,
|
|
656
|
+
frozenWeightOptions
|
|
629
657
|
);
|
|
630
658
|
}
|
|
631
659
|
const layer = {
|
|
@@ -635,29 +663,38 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
635
663
|
`layer_${layerIdx}.input_norm`,
|
|
636
664
|
ownedTrainables
|
|
637
665
|
),
|
|
666
|
+
queryKeyNorm: modelConfig.queryKeyNorm === true
|
|
667
|
+
&& (!Array.isArray(modelConfig.queryKeyNormLayers)
|
|
668
|
+
|| modelConfig.queryKeyNormLayers.includes(layerIdx)),
|
|
669
|
+
qNorm: null,
|
|
670
|
+
kNorm: null,
|
|
638
671
|
qProj: await ensureTrainableTensor(
|
|
639
672
|
layerWeights.qProj,
|
|
640
673
|
[numHeads * headDim, hiddenSize],
|
|
641
674
|
`layer_${layerIdx}.q_proj`,
|
|
642
|
-
ownedTrainables
|
|
675
|
+
ownedTrainables,
|
|
676
|
+
frozenWeightOptions
|
|
643
677
|
),
|
|
644
678
|
kProj: await ensureTrainableTensor(
|
|
645
679
|
layerWeights.kProj,
|
|
646
680
|
[numKVHeads * headDim, hiddenSize],
|
|
647
681
|
`layer_${layerIdx}.k_proj`,
|
|
648
|
-
ownedTrainables
|
|
682
|
+
ownedTrainables,
|
|
683
|
+
frozenWeightOptions
|
|
649
684
|
),
|
|
650
685
|
vProj: await ensureTrainableTensor(
|
|
651
686
|
layerWeights.vProj,
|
|
652
687
|
[numKVHeads * headDim, hiddenSize],
|
|
653
688
|
`layer_${layerIdx}.v_proj`,
|
|
654
|
-
ownedTrainables
|
|
689
|
+
ownedTrainables,
|
|
690
|
+
frozenWeightOptions
|
|
655
691
|
),
|
|
656
692
|
oProj: await ensureTrainableTensor(
|
|
657
693
|
layerWeights.oProj,
|
|
658
|
-
[hiddenSize,
|
|
694
|
+
[hiddenSize, numHeads * headDim],
|
|
659
695
|
`layer_${layerIdx}.o_proj`,
|
|
660
|
-
ownedTrainables
|
|
696
|
+
ownedTrainables,
|
|
697
|
+
frozenWeightOptions
|
|
661
698
|
),
|
|
662
699
|
postAttentionNorm: layerWeights.postAttentionNorm
|
|
663
700
|
? await ensureNormTensor(
|
|
@@ -667,12 +704,31 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
667
704
|
ownedTrainables
|
|
668
705
|
)
|
|
669
706
|
: null,
|
|
707
|
+
preFeedforwardNorm: layerWeights.preFeedforwardNorm
|
|
708
|
+
? await ensureNormTensor(
|
|
709
|
+
layerWeights.preFeedforwardNorm,
|
|
710
|
+
hiddenSize,
|
|
711
|
+
`layer_${layerIdx}.pre_feedforward_norm`,
|
|
712
|
+
ownedTrainables
|
|
713
|
+
)
|
|
714
|
+
: null,
|
|
715
|
+
postFeedforwardNorm: layerWeights.postFeedforwardNorm
|
|
716
|
+
? await ensureNormTensor(
|
|
717
|
+
layerWeights.postFeedforwardNorm,
|
|
718
|
+
hiddenSize,
|
|
719
|
+
`layer_${layerIdx}.post_feedforward_norm`,
|
|
720
|
+
ownedTrainables
|
|
721
|
+
)
|
|
722
|
+
: null,
|
|
670
723
|
gateUp: layerGateUp,
|
|
724
|
+
gate: layerGate,
|
|
725
|
+
up: layerUp,
|
|
671
726
|
down: await ensureTrainableTensor(
|
|
672
727
|
layerWeights.down || layerWeights.ffnDown,
|
|
673
728
|
[hiddenSize, layerIntermediateSize],
|
|
674
729
|
`layer_${layerIdx}.ffn_down`,
|
|
675
|
-
ownedTrainables
|
|
730
|
+
ownedTrainables,
|
|
731
|
+
frozenWeightOptions
|
|
676
732
|
),
|
|
677
733
|
intermediateSize: layerIntermediateSize,
|
|
678
734
|
lora: createTransformerLoraAdapters(loraConfig, {
|
|
@@ -680,14 +736,71 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
680
736
|
intermediateSize: layerIntermediateSize,
|
|
681
737
|
}),
|
|
682
738
|
};
|
|
739
|
+
if (layer.gateUp && (layer.lora.gate_proj || layer.lora.up_proj)) {
|
|
740
|
+
throw new Error(
|
|
741
|
+
`Layer ${layerIdx} has fused gate/up weights but separate gate_proj or up_proj LoRA targets.`
|
|
742
|
+
);
|
|
743
|
+
}
|
|
744
|
+
if (!layer.gateUp && layer.lora.gate_up_proj) {
|
|
745
|
+
throw new Error(
|
|
746
|
+
`Layer ${layerIdx} has separate gate/up weights but a fused gate_up_proj LoRA target.`
|
|
747
|
+
);
|
|
748
|
+
}
|
|
749
|
+
if (layer.queryKeyNorm) {
|
|
750
|
+
const weightedLayers = modelConfig.queryKeyNormWeightLayers;
|
|
751
|
+
const expectsWeightedNorm = !Array.isArray(weightedLayers) || weightedLayers.includes(layerIdx);
|
|
752
|
+
if (!expectsWeightedNorm) {
|
|
753
|
+
throw new Error(
|
|
754
|
+
`Distill full-graph student does not support unit-weight Q/K norm on layer ${layerIdx}.`
|
|
755
|
+
);
|
|
756
|
+
}
|
|
757
|
+
layer.qNorm = await ensureNormTensor(
|
|
758
|
+
layerWeights.qNorm,
|
|
759
|
+
headDim,
|
|
760
|
+
`layer_${layerIdx}.q_norm`,
|
|
761
|
+
ownedTrainables
|
|
762
|
+
);
|
|
763
|
+
layer.kNorm = await ensureNormTensor(
|
|
764
|
+
layerWeights.kNorm,
|
|
765
|
+
headDim,
|
|
766
|
+
`layer_${layerIdx}.k_norm`,
|
|
767
|
+
ownedTrainables
|
|
768
|
+
);
|
|
769
|
+
}
|
|
770
|
+
if (modelConfig.postAttentionNorm === true && !layer.postAttentionNorm) {
|
|
771
|
+
throw new Error(`Distill full-graph student missing post-attention norm on layer ${layerIdx}.`);
|
|
772
|
+
}
|
|
773
|
+
if (modelConfig.preFeedforwardNorm === true && !layer.preFeedforwardNorm) {
|
|
774
|
+
throw new Error(`Distill full-graph student missing pre-feedforward norm on layer ${layerIdx}.`);
|
|
775
|
+
}
|
|
776
|
+
if (modelConfig.postFeedforwardNorm === true && !layer.postFeedforwardNorm) {
|
|
777
|
+
throw new Error(`Distill full-graph student missing post-feedforward norm on layer ${layerIdx}.`);
|
|
778
|
+
}
|
|
683
779
|
layers.push(layer);
|
|
684
|
-
layerParams.push(
|
|
780
|
+
layerParams.push(
|
|
781
|
+
layer.inputNorm,
|
|
782
|
+
layer.qProj,
|
|
783
|
+
layer.kProj,
|
|
784
|
+
layer.vProj,
|
|
785
|
+
layer.oProj,
|
|
786
|
+
...(layer.gateUp ? [layer.gateUp] : [layer.gate, layer.up]),
|
|
787
|
+
layer.down
|
|
788
|
+
);
|
|
685
789
|
for (const adapter of Object.values(layer.lora)) {
|
|
686
790
|
loraParams.push(adapter.A, adapter.B);
|
|
687
791
|
}
|
|
688
792
|
if (layer.postAttentionNorm) {
|
|
689
793
|
layerParams.push(layer.postAttentionNorm);
|
|
690
794
|
}
|
|
795
|
+
if (layer.preFeedforwardNorm) {
|
|
796
|
+
layerParams.push(layer.preFeedforwardNorm);
|
|
797
|
+
}
|
|
798
|
+
if (layer.postFeedforwardNorm) {
|
|
799
|
+
layerParams.push(layer.postFeedforwardNorm);
|
|
800
|
+
}
|
|
801
|
+
if (layer.qNorm) {
|
|
802
|
+
layerParams.push(layer.qNorm, layer.kNorm);
|
|
803
|
+
}
|
|
691
804
|
}
|
|
692
805
|
|
|
693
806
|
const encoderParams = [embeddingWeight, ...layerParams];
|
|
@@ -714,6 +827,17 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
714
827
|
}
|
|
715
828
|
|
|
716
829
|
async function runTransformerTokenTensor(tokenTensor, seqLen, tape, forwardOptions = {}) {
|
|
830
|
+
const captureStage = async (stage, tensor, layerIdx = null) => {
|
|
831
|
+
if (typeof forwardOptions.captureStage === 'function') {
|
|
832
|
+
await forwardOptions.captureStage({
|
|
833
|
+
stage,
|
|
834
|
+
layerIdx,
|
|
835
|
+
seqLen,
|
|
836
|
+
tensor,
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
return tensor;
|
|
840
|
+
};
|
|
717
841
|
let hidden = await tape.record(
|
|
718
842
|
OpType.EMBED,
|
|
719
843
|
(indices, embeddings) => runGather(
|
|
@@ -738,6 +862,15 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
738
862
|
...stopBaseWeight,
|
|
739
863
|
}
|
|
740
864
|
);
|
|
865
|
+
if (embeddingScale !== 1) {
|
|
866
|
+
hidden = await tape.record(
|
|
867
|
+
OpType.SCALE,
|
|
868
|
+
(x) => runScale(x, embeddingScale, { count: seqLen * hiddenSize }),
|
|
869
|
+
[hidden],
|
|
870
|
+
{ count: seqLen * hiddenSize, scale: embeddingScale }
|
|
871
|
+
);
|
|
872
|
+
}
|
|
873
|
+
await captureStage('embed.out', hidden);
|
|
741
874
|
|
|
742
875
|
for (let layerIdx = 0; layerIdx < layers.length; layerIdx += 1) {
|
|
743
876
|
const layer = layers[layerIdx];
|
|
@@ -750,8 +883,15 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
750
883
|
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
751
884
|
}),
|
|
752
885
|
[hidden, layer.inputNorm],
|
|
753
|
-
{
|
|
886
|
+
{
|
|
887
|
+
numTokens: seqLen,
|
|
888
|
+
hiddenSize,
|
|
889
|
+
eps: rmsNormEps,
|
|
890
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
891
|
+
...stopBaseWeight,
|
|
892
|
+
}
|
|
754
893
|
);
|
|
894
|
+
await captureStage('attn.post_input_norm', normed, layerIdx);
|
|
755
895
|
|
|
756
896
|
let q2d = await tape.record(
|
|
757
897
|
OpType.MATMUL,
|
|
@@ -807,23 +947,110 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
807
947
|
{ size: seqLen * numKVHeads * headDim }
|
|
808
948
|
);
|
|
809
949
|
}
|
|
950
|
+
await captureStage('attn.q_proj', q2d, layerIdx);
|
|
951
|
+
await captureStage('attn.k_proj', k2d, layerIdx);
|
|
952
|
+
await captureStage('attn.v_proj', v2d, layerIdx);
|
|
953
|
+
|
|
954
|
+
if (layer.queryKeyNorm) {
|
|
955
|
+
const qNormInput = await recordTensorView(
|
|
956
|
+
tape,
|
|
957
|
+
q2d,
|
|
958
|
+
[seqLen * numHeads, headDim],
|
|
959
|
+
`layer_${layerIdx}_q_norm_input`
|
|
960
|
+
);
|
|
961
|
+
const qNormed = await tape.record(
|
|
962
|
+
OpType.RMSNORM,
|
|
963
|
+
(x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
|
|
964
|
+
batchSize: seqLen * numHeads,
|
|
965
|
+
hiddenSize: headDim,
|
|
966
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
967
|
+
}),
|
|
968
|
+
[qNormInput, layer.qNorm],
|
|
969
|
+
{
|
|
970
|
+
numTokens: seqLen * numHeads,
|
|
971
|
+
hiddenSize: headDim,
|
|
972
|
+
eps: rmsNormEps,
|
|
973
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
974
|
+
...stopBaseWeight,
|
|
975
|
+
}
|
|
976
|
+
);
|
|
977
|
+
q2d = await recordTensorView(
|
|
978
|
+
tape,
|
|
979
|
+
qNormed,
|
|
980
|
+
[seqLen, numHeads * headDim],
|
|
981
|
+
`layer_${layerIdx}_q_normed`
|
|
982
|
+
);
|
|
983
|
+
const kNormInput = await recordTensorView(
|
|
984
|
+
tape,
|
|
985
|
+
k2d,
|
|
986
|
+
[seqLen * numKVHeads, headDim],
|
|
987
|
+
`layer_${layerIdx}_k_norm_input`
|
|
988
|
+
);
|
|
989
|
+
const kNormed = await tape.record(
|
|
990
|
+
OpType.RMSNORM,
|
|
991
|
+
(x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
|
|
992
|
+
batchSize: seqLen * numKVHeads,
|
|
993
|
+
hiddenSize: headDim,
|
|
994
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
995
|
+
}),
|
|
996
|
+
[kNormInput, layer.kNorm],
|
|
997
|
+
{
|
|
998
|
+
numTokens: seqLen * numKVHeads,
|
|
999
|
+
hiddenSize: headDim,
|
|
1000
|
+
eps: rmsNormEps,
|
|
1001
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
1002
|
+
...stopBaseWeight,
|
|
1003
|
+
}
|
|
1004
|
+
);
|
|
1005
|
+
k2d = await recordTensorView(
|
|
1006
|
+
tape,
|
|
1007
|
+
kNormed,
|
|
1008
|
+
[seqLen, numKVHeads * headDim],
|
|
1009
|
+
`layer_${layerIdx}_k_normed`
|
|
1010
|
+
);
|
|
1011
|
+
await captureStage('attn.q_norm', q2d, layerIdx);
|
|
1012
|
+
await captureStage('attn.k_norm', k2d, layerIdx);
|
|
1013
|
+
}
|
|
810
1014
|
|
|
811
|
-
const q3d =
|
|
812
|
-
|
|
813
|
-
|
|
1015
|
+
const q3d = await recordTensorView(
|
|
1016
|
+
tape,
|
|
1017
|
+
q2d,
|
|
1018
|
+
[seqLen, numHeads, headDim],
|
|
1019
|
+
`layer_${layerIdx}_q`
|
|
1020
|
+
);
|
|
1021
|
+
const k3d = await recordTensorView(
|
|
1022
|
+
tape,
|
|
1023
|
+
k2d,
|
|
1024
|
+
[seqLen, numKVHeads, headDim],
|
|
1025
|
+
`layer_${layerIdx}_k`
|
|
1026
|
+
);
|
|
1027
|
+
const v3d = await recordTensorView(
|
|
1028
|
+
tape,
|
|
1029
|
+
v2d,
|
|
1030
|
+
[seqLen, numKVHeads, headDim],
|
|
1031
|
+
`layer_${layerIdx}_v`
|
|
1032
|
+
);
|
|
1033
|
+
const useLocalRope = isSlidingLayerType(modelConfig.layerTypes?.[layerIdx]);
|
|
1034
|
+
const layerRopeCos = useLocalRope ? ropeLocalCos : ropeCos;
|
|
1035
|
+
const layerRopeSin = useLocalRope ? ropeLocalSin : ropeSin;
|
|
1036
|
+
if (!layerRopeCos || !layerRopeSin) {
|
|
1037
|
+
throw new Error(`Distill full-graph student missing RoPE tensors on layer ${layerIdx}.`);
|
|
1038
|
+
}
|
|
814
1039
|
|
|
815
1040
|
const qRope = await tape.record(
|
|
816
1041
|
OpType.ROPE,
|
|
817
1042
|
(q, cos, sin) => runRoPE(q, cos, sin, seqLen, { numHeads, headDim, startPos: 0 }),
|
|
818
|
-
[q3d,
|
|
1043
|
+
[q3d, layerRopeCos, layerRopeSin],
|
|
819
1044
|
{ seqLen, numHeads, headDim, startPos: 0, ...stopRopeWeights }
|
|
820
1045
|
);
|
|
821
1046
|
const kRope = await tape.record(
|
|
822
1047
|
OpType.ROPE,
|
|
823
1048
|
(k, cos, sin) => runRoPE(k, cos, sin, seqLen, { numHeads: numKVHeads, headDim, startPos: 0 }),
|
|
824
|
-
[k3d,
|
|
1049
|
+
[k3d, layerRopeCos, layerRopeSin],
|
|
825
1050
|
{ seqLen, numHeads: numKVHeads, headDim, startPos: 0, ...stopRopeWeights }
|
|
826
1051
|
);
|
|
1052
|
+
await captureStage('attn.q_rope', qRope, layerIdx);
|
|
1053
|
+
await captureStage('attn.k_rope', kRope, layerIdx);
|
|
827
1054
|
|
|
828
1055
|
const attention = await tape.record(
|
|
829
1056
|
OpType.ATTENTION,
|
|
@@ -836,23 +1063,38 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
836
1063
|
scale: 1 / Math.sqrt(headDim),
|
|
837
1064
|
}),
|
|
838
1065
|
[qRope, kRope, v3d],
|
|
839
|
-
{
|
|
1066
|
+
{
|
|
1067
|
+
seqLen,
|
|
1068
|
+
numHeads,
|
|
1069
|
+
numKVHeads,
|
|
1070
|
+
headDim,
|
|
1071
|
+
scale: 1 / Math.sqrt(headDim),
|
|
1072
|
+
causal: true,
|
|
1073
|
+
recomputeForward: true,
|
|
1074
|
+
}
|
|
840
1075
|
);
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
1076
|
+
await captureStage('attn.core_out', attention, layerIdx);
|
|
1077
|
+
const attention2d = await recordTensorView(
|
|
1078
|
+
tape,
|
|
1079
|
+
attention,
|
|
1080
|
+
[seqLen, numHeads * headDim],
|
|
845
1081
|
`layer_${layerIdx}_attn_2d`
|
|
846
1082
|
);
|
|
847
1083
|
|
|
848
1084
|
let attentionOutput = await tape.record(
|
|
849
1085
|
OpType.MATMUL,
|
|
850
|
-
(x, w) => runMatmul(x, w, seqLen, hiddenSize,
|
|
1086
|
+
(x, w) => runMatmul(x, w, seqLen, hiddenSize, numHeads * headDim, {
|
|
851
1087
|
transposeB: 'auto',
|
|
852
1088
|
outputDtype: 'f32',
|
|
853
1089
|
}),
|
|
854
1090
|
[attention2d, layer.oProj],
|
|
855
|
-
{
|
|
1091
|
+
{
|
|
1092
|
+
M: seqLen,
|
|
1093
|
+
N: hiddenSize,
|
|
1094
|
+
K: numHeads * headDim,
|
|
1095
|
+
transposeB: 'auto',
|
|
1096
|
+
...stopBaseWeight,
|
|
1097
|
+
}
|
|
856
1098
|
);
|
|
857
1099
|
if (layer.lora.o_proj) {
|
|
858
1100
|
const delta = await layer.lora.o_proj.forward(attention2d, tape);
|
|
@@ -863,14 +1105,34 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
863
1105
|
{ size: seqLen * hiddenSize }
|
|
864
1106
|
);
|
|
865
1107
|
}
|
|
1108
|
+
await captureStage('attn.out', attentionOutput, layerIdx);
|
|
1109
|
+
const normalizedAttentionOutput = modelConfig.postAttentionNorm === true
|
|
1110
|
+
? await tape.record(
|
|
1111
|
+
OpType.RMSNORM,
|
|
1112
|
+
(x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
|
|
1113
|
+
batchSize: seqLen,
|
|
1114
|
+
hiddenSize,
|
|
1115
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
1116
|
+
}),
|
|
1117
|
+
[attentionOutput, layer.postAttentionNorm],
|
|
1118
|
+
{
|
|
1119
|
+
numTokens: seqLen,
|
|
1120
|
+
hiddenSize,
|
|
1121
|
+
eps: rmsNormEps,
|
|
1122
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
1123
|
+
...stopBaseWeight,
|
|
1124
|
+
}
|
|
1125
|
+
)
|
|
1126
|
+
: attentionOutput;
|
|
866
1127
|
const postAttention = await tape.record(
|
|
867
1128
|
OpType.RESIDUAL_ADD,
|
|
868
1129
|
(a, b) => runResidualAdd(a, b, seqLen * hiddenSize),
|
|
869
|
-
[
|
|
1130
|
+
[normalizedAttentionOutput, hidden],
|
|
870
1131
|
{ size: seqLen * hiddenSize }
|
|
871
1132
|
);
|
|
1133
|
+
await captureStage('attn.post_attn', postAttention, layerIdx);
|
|
872
1134
|
|
|
873
|
-
const ffnInput =
|
|
1135
|
+
const ffnInput = modelConfig.preFeedforwardNorm === true
|
|
874
1136
|
? await tape.record(
|
|
875
1137
|
OpType.RMSNORM,
|
|
876
1138
|
(x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
|
|
@@ -878,44 +1140,106 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
878
1140
|
hiddenSize,
|
|
879
1141
|
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
880
1142
|
}),
|
|
881
|
-
[postAttention, layer.
|
|
882
|
-
{
|
|
1143
|
+
[postAttention, layer.preFeedforwardNorm],
|
|
1144
|
+
{
|
|
1145
|
+
numTokens: seqLen,
|
|
1146
|
+
hiddenSize,
|
|
1147
|
+
eps: rmsNormEps,
|
|
1148
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
1149
|
+
...stopBaseWeight,
|
|
1150
|
+
}
|
|
883
1151
|
)
|
|
884
1152
|
: postAttention;
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
gateUp = await tape.record(
|
|
897
|
-
OpType.RESIDUAL_ADD,
|
|
898
|
-
(a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize * 2),
|
|
899
|
-
[gateUp, delta],
|
|
900
|
-
{ size: seqLen * layerIntermediateSize * 2 }
|
|
1153
|
+
await captureStage('ffn.in', ffnInput, layerIdx);
|
|
1154
|
+
let activated = null;
|
|
1155
|
+
if (layer.gateUp) {
|
|
1156
|
+
let gateUp = await tape.record(
|
|
1157
|
+
OpType.MATMUL,
|
|
1158
|
+
(x, w) => runMatmul(x, w, seqLen, layerIntermediateSize * 2, hiddenSize, {
|
|
1159
|
+
transposeB: 'auto',
|
|
1160
|
+
outputDtype: 'f32',
|
|
1161
|
+
}),
|
|
1162
|
+
[ffnInput, layer.gateUp],
|
|
1163
|
+
{ M: seqLen, N: layerIntermediateSize * 2, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
|
|
901
1164
|
);
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
}),
|
|
911
|
-
[gateUp],
|
|
912
|
-
{
|
|
913
|
-
numTokens: seqLen,
|
|
914
|
-
dim: layerIntermediateSize,
|
|
915
|
-
activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
|
|
916
|
-
swigluLimit: hiddenActivation === 'gelu' ? 0 : swigluLimit,
|
|
1165
|
+
if (layer.lora.gate_up_proj) {
|
|
1166
|
+
const delta = await layer.lora.gate_up_proj.forward(ffnInput, tape);
|
|
1167
|
+
gateUp = await tape.record(
|
|
1168
|
+
OpType.RESIDUAL_ADD,
|
|
1169
|
+
(a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize * 2),
|
|
1170
|
+
[gateUp, delta],
|
|
1171
|
+
{ size: seqLen * layerIntermediateSize * 2 }
|
|
1172
|
+
);
|
|
917
1173
|
}
|
|
918
|
-
|
|
1174
|
+
activated = await tape.record(
|
|
1175
|
+
OpType.SILU_ROWSPLIT,
|
|
1176
|
+
(x) => runSiLURowSplit(x, {
|
|
1177
|
+
numTokens: seqLen,
|
|
1178
|
+
dim: layerIntermediateSize,
|
|
1179
|
+
activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
|
|
1180
|
+
swigluLimit: hiddenActivation === 'gelu' ? null : swigluLimit,
|
|
1181
|
+
}),
|
|
1182
|
+
[gateUp],
|
|
1183
|
+
{
|
|
1184
|
+
numTokens: seqLen,
|
|
1185
|
+
dim: layerIntermediateSize,
|
|
1186
|
+
activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
|
|
1187
|
+
swigluLimit: hiddenActivation === 'gelu' ? 0 : swigluLimit,
|
|
1188
|
+
}
|
|
1189
|
+
);
|
|
1190
|
+
} else {
|
|
1191
|
+
if (hiddenActivation === 'gelu') {
|
|
1192
|
+
throw new Error('Split gate/up training currently requires SiLU activation.');
|
|
1193
|
+
}
|
|
1194
|
+
let gate = await tape.record(
|
|
1195
|
+
OpType.MATMUL,
|
|
1196
|
+
(x, w) => runMatmul(x, w, seqLen, layerIntermediateSize, hiddenSize, {
|
|
1197
|
+
transposeB: 'auto',
|
|
1198
|
+
outputDtype: 'f32',
|
|
1199
|
+
}),
|
|
1200
|
+
[ffnInput, layer.gate],
|
|
1201
|
+
{ M: seqLen, N: layerIntermediateSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
|
|
1202
|
+
);
|
|
1203
|
+
let up = await tape.record(
|
|
1204
|
+
OpType.MATMUL,
|
|
1205
|
+
(x, w) => runMatmul(x, w, seqLen, layerIntermediateSize, hiddenSize, {
|
|
1206
|
+
transposeB: 'auto',
|
|
1207
|
+
outputDtype: 'f32',
|
|
1208
|
+
}),
|
|
1209
|
+
[ffnInput, layer.up],
|
|
1210
|
+
{ M: seqLen, N: layerIntermediateSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
|
|
1211
|
+
);
|
|
1212
|
+
if (layer.lora.gate_proj) {
|
|
1213
|
+
const delta = await layer.lora.gate_proj.forward(ffnInput, tape);
|
|
1214
|
+
gate = await tape.record(
|
|
1215
|
+
OpType.RESIDUAL_ADD,
|
|
1216
|
+
(a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize),
|
|
1217
|
+
[gate, delta],
|
|
1218
|
+
{ size: seqLen * layerIntermediateSize }
|
|
1219
|
+
);
|
|
1220
|
+
}
|
|
1221
|
+
if (layer.lora.up_proj) {
|
|
1222
|
+
const delta = await layer.lora.up_proj.forward(ffnInput, tape);
|
|
1223
|
+
up = await tape.record(
|
|
1224
|
+
OpType.RESIDUAL_ADD,
|
|
1225
|
+
(a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize),
|
|
1226
|
+
[up, delta],
|
|
1227
|
+
{ size: seqLen * layerIntermediateSize }
|
|
1228
|
+
);
|
|
1229
|
+
}
|
|
1230
|
+
activated = await tape.record(
|
|
1231
|
+
OpType.SILU_GATED,
|
|
1232
|
+
(gateInput, upInput) => runSiLU(upInput, {
|
|
1233
|
+
size: seqLen * layerIntermediateSize,
|
|
1234
|
+
gate: gateInput,
|
|
1235
|
+
inputActivation: 'identity',
|
|
1236
|
+
swigluLimit,
|
|
1237
|
+
}),
|
|
1238
|
+
[gate, up],
|
|
1239
|
+
{ count: seqLen * layerIntermediateSize, swigluLimit }
|
|
1240
|
+
);
|
|
1241
|
+
}
|
|
1242
|
+
await captureStage('ffn.act', activated, layerIdx);
|
|
919
1243
|
let ffnOutput = await tape.record(
|
|
920
1244
|
OpType.MATMUL,
|
|
921
1245
|
(x, w) => runMatmul(x, w, seqLen, hiddenSize, layerIntermediateSize, {
|
|
@@ -934,14 +1258,35 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
934
1258
|
{ size: seqLen * hiddenSize }
|
|
935
1259
|
);
|
|
936
1260
|
}
|
|
1261
|
+
await captureStage('ffn.out', ffnOutput, layerIdx);
|
|
1262
|
+
const normalizedFfnOutput = modelConfig.postFeedforwardNorm === true
|
|
1263
|
+
? await tape.record(
|
|
1264
|
+
OpType.RMSNORM,
|
|
1265
|
+
(x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
|
|
1266
|
+
batchSize: seqLen,
|
|
1267
|
+
hiddenSize,
|
|
1268
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
1269
|
+
}),
|
|
1270
|
+
[ffnOutput, layer.postFeedforwardNorm],
|
|
1271
|
+
{
|
|
1272
|
+
numTokens: seqLen,
|
|
1273
|
+
hiddenSize,
|
|
1274
|
+
eps: rmsNormEps,
|
|
1275
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
1276
|
+
...stopBaseWeight,
|
|
1277
|
+
}
|
|
1278
|
+
)
|
|
1279
|
+
: ffnOutput;
|
|
937
1280
|
hidden = await tape.record(
|
|
938
1281
|
OpType.RESIDUAL_ADD,
|
|
939
1282
|
(a, b) => runResidualAdd(a, b, seqLen * hiddenSize),
|
|
940
|
-
[
|
|
1283
|
+
[normalizedFfnOutput, postAttention],
|
|
941
1284
|
{ size: seqLen * hiddenSize }
|
|
942
1285
|
);
|
|
1286
|
+
await captureStage('layer.out', hidden, layerIdx);
|
|
943
1287
|
}
|
|
944
1288
|
|
|
1289
|
+
await captureStage('final_norm.pre', hidden);
|
|
945
1290
|
const finalHidden = await tape.record(
|
|
946
1291
|
OpType.RMSNORM,
|
|
947
1292
|
(x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
|
|
@@ -950,10 +1295,17 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
950
1295
|
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
951
1296
|
}),
|
|
952
1297
|
[hidden, finalNormWeight],
|
|
953
|
-
{
|
|
1298
|
+
{
|
|
1299
|
+
numTokens: seqLen,
|
|
1300
|
+
hiddenSize,
|
|
1301
|
+
eps: rmsNormEps,
|
|
1302
|
+
rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
|
|
1303
|
+
...stopBaseWeight,
|
|
1304
|
+
}
|
|
954
1305
|
);
|
|
1306
|
+
await captureStage('final_norm.out', finalHidden);
|
|
955
1307
|
if (forwardOptions.logitsMode === 'all') {
|
|
956
|
-
|
|
1308
|
+
const logits = await tape.record(
|
|
957
1309
|
OpType.MATMUL,
|
|
958
1310
|
(x, w) => runMatmul(x, w, seqLen, vocabSize, hiddenSize, {
|
|
959
1311
|
transposeB: 'auto',
|
|
@@ -962,6 +1314,8 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
962
1314
|
[finalHidden, lmHeadWeight],
|
|
963
1315
|
{ M: seqLen, N: vocabSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
|
|
964
1316
|
);
|
|
1317
|
+
await captureStage('logits.out', logits);
|
|
1318
|
+
return logits;
|
|
965
1319
|
}
|
|
966
1320
|
const lastHidden = await tape.record(
|
|
967
1321
|
OpType.ROW_SLICE,
|
|
@@ -969,7 +1323,7 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
969
1323
|
[finalHidden],
|
|
970
1324
|
{ rows: seqLen, cols: hiddenSize, rowIndex: seqLen - 1 }
|
|
971
1325
|
);
|
|
972
|
-
|
|
1326
|
+
const logits = await tape.record(
|
|
973
1327
|
OpType.MATMUL,
|
|
974
1328
|
(x, w) => runMatmul(x, w, 1, vocabSize, hiddenSize, {
|
|
975
1329
|
transposeB: 'auto',
|
|
@@ -978,11 +1332,13 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
978
1332
|
[lastHidden, lmHeadWeight],
|
|
979
1333
|
{ M: 1, N: vocabSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
|
|
980
1334
|
);
|
|
1335
|
+
await captureStage('logits.out', logits);
|
|
1336
|
+
return logits;
|
|
981
1337
|
}
|
|
982
1338
|
|
|
983
|
-
async function runTransformerPrompt(prompt, tape) {
|
|
1339
|
+
async function runTransformerPrompt(prompt, tape, forwardOptions = {}) {
|
|
984
1340
|
const { tokenTensor, seqLen } = await buildPromptTokens(prompt);
|
|
985
|
-
return runTransformerTokenTensor(tokenTensor, seqLen, tape);
|
|
1341
|
+
return runTransformerTokenTensor(tokenTensor, seqLen, tape, forwardOptions);
|
|
986
1342
|
}
|
|
987
1343
|
|
|
988
1344
|
function collectLoraTensorEntries() {
|
|
@@ -1037,7 +1393,7 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
|
|
|
1037
1393
|
`Distill full-graph student currently requires batchSize=1, got ${prompts.length}.`
|
|
1038
1394
|
);
|
|
1039
1395
|
}
|
|
1040
|
-
const logits = await runTransformerPrompt(prompts[0], tape);
|
|
1396
|
+
const logits = await runTransformerPrompt(prompts[0], tape, forwardOptions);
|
|
1041
1397
|
return { logits };
|
|
1042
1398
|
},
|
|
1043
1399
|
cleanupDistillStep() {
|