doppler-gpu 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +62 -12
- package/models/catalog.json +21 -7
- package/package.json +25 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +8 -4
- package/src/config/kernels/registry.json +1035 -2677
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/workloads.js +11 -0
- package/src/gpu/device.js +100 -32
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/index.d.ts +5 -0
- package/src/gpu/kernels/index.js +5 -0
- package/src/gpu/kernels/matmul-selection.js +5 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-v1.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +2 -0
- package/src/inference/pipelines/text/generator.js +29 -0
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/tokenizers/bundled.js +105 -32
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling/runtime-optimization.d.ts +109 -0
- package/src/tooling/runtime-optimization.js +872 -0
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/tooling-exports.shared.d.ts +19 -0
- package/src/tooling-exports.shared.js +11 -0
- package/src/version.js +1 -1
|
@@ -10,13 +10,45 @@ function toFloat32(buffer, dtype) {
|
|
|
10
10
|
return new Float32Array(buffer);
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
function
|
|
14
|
-
const
|
|
13
|
+
function resolveAttentionGeometry(options) {
|
|
14
|
+
const seqLen = Math.floor(Number(options.seqLen));
|
|
15
|
+
const numHeads = Math.floor(Number(options.numHeads));
|
|
16
|
+
const numKVHeads = Math.floor(Number(options.numKVHeads ?? numHeads));
|
|
17
|
+
const headDim = Math.floor(Number(options.headDim));
|
|
18
|
+
if (
|
|
19
|
+
seqLen < 1
|
|
20
|
+
|| numHeads < 1
|
|
21
|
+
|| numKVHeads < 1
|
|
22
|
+
|| headDim < 1
|
|
23
|
+
|| numHeads % numKVHeads !== 0
|
|
24
|
+
) {
|
|
25
|
+
throw new Error('attention backward requires valid GQA geometry.');
|
|
26
|
+
}
|
|
27
|
+
return {
|
|
28
|
+
seqLen,
|
|
29
|
+
numHeads,
|
|
30
|
+
numKVHeads,
|
|
31
|
+
headDim,
|
|
32
|
+
headsPerKV: numHeads / numKVHeads,
|
|
33
|
+
scale: Number.isFinite(options.scale) ? options.scale : 1,
|
|
34
|
+
causal: options.causal === true,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function computeAttentionSoftmaxData(qData, kData, options) {
|
|
39
|
+
const {
|
|
40
|
+
seqLen,
|
|
41
|
+
numHeads,
|
|
42
|
+
numKVHeads,
|
|
43
|
+
headDim,
|
|
44
|
+
headsPerKV,
|
|
45
|
+
scale,
|
|
46
|
+
causal,
|
|
47
|
+
} = resolveAttentionGeometry(options);
|
|
15
48
|
const sData = new Float32Array(numHeads * seqLen * seqLen);
|
|
16
49
|
|
|
17
50
|
for (let h = 0; h < numHeads; h += 1) {
|
|
18
|
-
const
|
|
19
|
-
const kOffset = h * seqLen * headDim;
|
|
51
|
+
const kvHead = Math.floor(h / headsPerKV);
|
|
20
52
|
const sOffset = h * seqLen * seqLen;
|
|
21
53
|
|
|
22
54
|
for (let i = 0; i < seqLen; i += 1) {
|
|
@@ -27,7 +59,9 @@ function computeSoftmax(qData, kData, options) {
|
|
|
27
59
|
}
|
|
28
60
|
let sum = 0.0;
|
|
29
61
|
for (let d = 0; d < headDim; d += 1) {
|
|
30
|
-
|
|
62
|
+
const qIndex = (i * numHeads + h) * headDim + d;
|
|
63
|
+
const kIndex = (j * numKVHeads + kvHead) * headDim + d;
|
|
64
|
+
sum += qData[qIndex] * kData[kIndex];
|
|
31
65
|
}
|
|
32
66
|
const scaled = sum * scale;
|
|
33
67
|
if (scaled > rowMax) {
|
|
@@ -42,7 +76,9 @@ function computeSoftmax(qData, kData, options) {
|
|
|
42
76
|
}
|
|
43
77
|
let sum = 0.0;
|
|
44
78
|
for (let d = 0; d < headDim; d += 1) {
|
|
45
|
-
|
|
79
|
+
const qIndex = (i * numHeads + h) * headDim + d;
|
|
80
|
+
const kIndex = (j * numKVHeads + kvHead) * headDim + d;
|
|
81
|
+
sum += qData[qIndex] * kData[kIndex];
|
|
46
82
|
}
|
|
47
83
|
const expVal = Math.exp(sum * scale - rowMax);
|
|
48
84
|
sData[sOffset + i * seqLen + j] = expVal;
|
|
@@ -66,7 +102,7 @@ export async function buildAttentionSoftmaxCache(q, k, options) {
|
|
|
66
102
|
const [qBuf, kBuf] = await Promise.all([readBuffer(q.buffer), readBuffer(k.buffer)]);
|
|
67
103
|
const qData = toFloat32(qBuf, q.dtype);
|
|
68
104
|
const kData = toFloat32(kBuf, k.dtype);
|
|
69
|
-
const sData =
|
|
105
|
+
const sData = computeAttentionSoftmaxData(qData, kData, options);
|
|
70
106
|
const { seqLen, numHeads } = options;
|
|
71
107
|
return createUploadedTensor(sData, 'f32', [numHeads, seqLen, seqLen], 'attn_softmax_cache');
|
|
72
108
|
}
|
|
@@ -79,8 +115,6 @@ export async function attentionBackwardCpu(
|
|
|
79
115
|
gradOutput,
|
|
80
116
|
options
|
|
81
117
|
) {
|
|
82
|
-
const { seqLen, numHeads, headDim, scale = 1.0, causal = false } = options;
|
|
83
|
-
|
|
84
118
|
const buffers = [
|
|
85
119
|
readBuffer(q.buffer),
|
|
86
120
|
readBuffer(k.buffer),
|
|
@@ -103,20 +137,76 @@ export async function attentionBackwardCpu(
|
|
|
103
137
|
const vData = toFloat32(vBuf, v.dtype);
|
|
104
138
|
const sData = softmax
|
|
105
139
|
? toFloat32(sBuf, softmax.dtype)
|
|
106
|
-
:
|
|
140
|
+
: computeAttentionSoftmaxData(qData, kData, options);
|
|
107
141
|
const dData = toFloat32(dBuf, gradOutput.dtype);
|
|
108
142
|
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
143
|
+
const { dQ, dK, dV, geometry } = computeAttentionBackwardData(
|
|
144
|
+
qData,
|
|
145
|
+
kData,
|
|
146
|
+
vData,
|
|
147
|
+
sData,
|
|
148
|
+
dData,
|
|
149
|
+
options
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
let qBufOut = null;
|
|
153
|
+
let kBufOut = null;
|
|
154
|
+
let vBufOut = null;
|
|
155
|
+
try {
|
|
156
|
+
qBufOut = acquireBuffer(tensorBytes(q.shape, 'f32'), undefined, 'attn_backward_q');
|
|
157
|
+
kBufOut = acquireBuffer(tensorBytes(k.shape, 'f32'), undefined, 'attn_backward_k');
|
|
158
|
+
vBufOut = acquireBuffer(tensorBytes(v.shape, 'f32'), undefined, 'attn_backward_v');
|
|
159
|
+
|
|
160
|
+
uploadData(qBufOut, dQ);
|
|
161
|
+
uploadData(kBufOut, dK);
|
|
162
|
+
uploadData(vBufOut, dV);
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
gradQ: createTensor(qBufOut, 'f32', [...q.shape], 'attn_grad_q'),
|
|
166
|
+
gradK: createTensor(kBufOut, 'f32', [...k.shape], 'attn_grad_k'),
|
|
167
|
+
gradV: createTensor(vBufOut, 'f32', [...v.shape], 'attn_grad_v'),
|
|
168
|
+
geometry,
|
|
169
|
+
};
|
|
170
|
+
} catch (error) {
|
|
171
|
+
if (qBufOut) {
|
|
172
|
+
releaseBuffer(qBufOut);
|
|
173
|
+
}
|
|
174
|
+
if (kBufOut) {
|
|
175
|
+
releaseBuffer(kBufOut);
|
|
176
|
+
}
|
|
177
|
+
if (vBufOut) {
|
|
178
|
+
releaseBuffer(vBufOut);
|
|
179
|
+
}
|
|
180
|
+
throw error;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export function computeAttentionBackwardData(
|
|
185
|
+
qData,
|
|
186
|
+
kData,
|
|
187
|
+
vData,
|
|
188
|
+
sData,
|
|
189
|
+
dData,
|
|
190
|
+
options
|
|
191
|
+
) {
|
|
192
|
+
const geometry = resolveAttentionGeometry(options);
|
|
193
|
+
const {
|
|
194
|
+
seqLen,
|
|
195
|
+
numHeads,
|
|
196
|
+
numKVHeads,
|
|
197
|
+
headDim,
|
|
198
|
+
headsPerKV,
|
|
199
|
+
scale,
|
|
200
|
+
causal,
|
|
201
|
+
} = geometry;
|
|
202
|
+
|
|
203
|
+
const dQ = new Float32Array(seqLen * numHeads * headDim);
|
|
204
|
+
const dK = new Float32Array(seqLen * numKVHeads * headDim);
|
|
205
|
+
const dV = new Float32Array(seqLen * numKVHeads * headDim);
|
|
113
206
|
|
|
114
207
|
for (let h = 0; h < numHeads; h += 1) {
|
|
115
|
-
const
|
|
116
|
-
const kOffset = h * seqLen * headDim;
|
|
117
|
-
const vOffset = h * seqLen * headDim;
|
|
208
|
+
const kvHead = Math.floor(h / headsPerKV);
|
|
118
209
|
const sOffset = h * seqLen * seqLen;
|
|
119
|
-
const dOffset = h * seqLen * headDim;
|
|
120
210
|
|
|
121
211
|
// dV = S^T @ dO
|
|
122
212
|
for (let j = 0; j < seqLen; j += 1) {
|
|
@@ -127,10 +217,11 @@ export async function attentionBackwardCpu(
|
|
|
127
217
|
continue;
|
|
128
218
|
}
|
|
129
219
|
const s = sData[sOffset + i * seqLen + j];
|
|
130
|
-
const dO = dData[
|
|
220
|
+
const dO = dData[(i * numHeads + h) * headDim + d];
|
|
131
221
|
sum += s * dO;
|
|
132
222
|
}
|
|
133
|
-
|
|
223
|
+
const vIndex = (j * numKVHeads + kvHead) * headDim + d;
|
|
224
|
+
dV[vIndex] += sum;
|
|
134
225
|
}
|
|
135
226
|
}
|
|
136
227
|
|
|
@@ -143,8 +234,9 @@ export async function attentionBackwardCpu(
|
|
|
143
234
|
}
|
|
144
235
|
let sum = 0.0;
|
|
145
236
|
for (let d = 0; d < headDim; d += 1) {
|
|
146
|
-
const dO = dData[
|
|
147
|
-
const
|
|
237
|
+
const dO = dData[(i * numHeads + h) * headDim + d];
|
|
238
|
+
const vIndex = (j * numKVHeads + kvHead) * headDim + d;
|
|
239
|
+
const vVal = vData[vIndex];
|
|
148
240
|
sum += dO * vVal;
|
|
149
241
|
}
|
|
150
242
|
dS[i * seqLen + j] = sum;
|
|
@@ -179,9 +271,11 @@ export async function attentionBackwardCpu(
|
|
|
179
271
|
if (causal && j > i) {
|
|
180
272
|
continue;
|
|
181
273
|
}
|
|
182
|
-
|
|
274
|
+
const kIndex = (j * numKVHeads + kvHead) * headDim + d;
|
|
275
|
+
sum += dQK[i * seqLen + j] * kData[kIndex];
|
|
183
276
|
}
|
|
184
|
-
|
|
277
|
+
const qIndex = (i * numHeads + h) * headDim + d;
|
|
278
|
+
dQ[qIndex] = sum * scale;
|
|
185
279
|
}
|
|
186
280
|
}
|
|
187
281
|
|
|
@@ -193,40 +287,14 @@ export async function attentionBackwardCpu(
|
|
|
193
287
|
if (causal && j > i) {
|
|
194
288
|
continue;
|
|
195
289
|
}
|
|
196
|
-
|
|
290
|
+
const qIndex = (i * numHeads + h) * headDim + d;
|
|
291
|
+
sum += dQK[i * seqLen + j] * qData[qIndex];
|
|
197
292
|
}
|
|
198
|
-
|
|
293
|
+
const kIndex = (j * numKVHeads + kvHead) * headDim + d;
|
|
294
|
+
dK[kIndex] += sum * scale;
|
|
199
295
|
}
|
|
200
296
|
}
|
|
201
297
|
}
|
|
202
298
|
|
|
203
|
-
|
|
204
|
-
let kBufOut = null;
|
|
205
|
-
let vBufOut = null;
|
|
206
|
-
try {
|
|
207
|
-
qBufOut = acquireBuffer(tensorBytes([seqLen, numHeads, headDim], 'f32'), undefined, 'attn_backward_q');
|
|
208
|
-
kBufOut = acquireBuffer(tensorBytes([seqLen, numHeads, headDim], 'f32'), undefined, 'attn_backward_k');
|
|
209
|
-
vBufOut = acquireBuffer(tensorBytes([seqLen, numHeads, headDim], 'f32'), undefined, 'attn_backward_v');
|
|
210
|
-
|
|
211
|
-
uploadData(qBufOut, dQ);
|
|
212
|
-
uploadData(kBufOut, dK);
|
|
213
|
-
uploadData(vBufOut, dV);
|
|
214
|
-
|
|
215
|
-
return {
|
|
216
|
-
gradQ: createTensor(qBufOut, 'f32', [seqLen, numHeads, headDim], 'attn_grad_q'),
|
|
217
|
-
gradK: createTensor(kBufOut, 'f32', [seqLen, numHeads, headDim], 'attn_grad_k'),
|
|
218
|
-
gradV: createTensor(vBufOut, 'f32', [seqLen, numHeads, headDim], 'attn_grad_v'),
|
|
219
|
-
};
|
|
220
|
-
} catch (error) {
|
|
221
|
-
if (qBufOut) {
|
|
222
|
-
releaseBuffer(qBufOut);
|
|
223
|
-
}
|
|
224
|
-
if (kBufOut) {
|
|
225
|
-
releaseBuffer(kBufOut);
|
|
226
|
-
}
|
|
227
|
-
if (vBufOut) {
|
|
228
|
-
releaseBuffer(vBufOut);
|
|
229
|
-
}
|
|
230
|
-
throw error;
|
|
231
|
-
}
|
|
299
|
+
return { dQ, dK, dV, geometry };
|
|
232
300
|
}
|
|
@@ -13,6 +13,7 @@ export const OpType = {
|
|
|
13
13
|
MATMUL: 'matmul',
|
|
14
14
|
RMSNORM: 'rmsnorm',
|
|
15
15
|
RESIDUAL_ADD: 'residual_add',
|
|
16
|
+
RESHAPE: 'reshape',
|
|
16
17
|
ROW_SLICE: 'row_slice',
|
|
17
18
|
LAYERNORM: 'layernorm',
|
|
18
19
|
ATTENTION: 'attention',
|
|
@@ -20,6 +21,7 @@ export const OpType = {
|
|
|
20
21
|
ROPE: 'rope',
|
|
21
22
|
SILU: 'silu',
|
|
22
23
|
SILU_ROWSPLIT: 'silu_rowsplit',
|
|
24
|
+
SILU_GATED: 'silu_gated',
|
|
23
25
|
GELU: 'gelu',
|
|
24
26
|
SCALE: 'scale',
|
|
25
27
|
CROSS_ENTROPY: 'cross_entropy',
|
|
@@ -30,6 +32,38 @@ export const OpType = {
|
|
|
30
32
|
CONV2D: 'conv2d',
|
|
31
33
|
};
|
|
32
34
|
|
|
35
|
+
export function resolveMatmulBackwardOptions(options = {}) {
|
|
36
|
+
const stopped = new Set(
|
|
37
|
+
(Array.isArray(options.stopGradInputs) ? options.stopGradInputs : [])
|
|
38
|
+
.map((value) => Math.floor(Number(value)))
|
|
39
|
+
.filter((value) => Number.isInteger(value))
|
|
40
|
+
);
|
|
41
|
+
return {
|
|
42
|
+
...options,
|
|
43
|
+
computeGradInput: options.computeGradInput !== false && !stopped.has(0),
|
|
44
|
+
computeGradWeight: options.computeGradWeight !== false && !stopped.has(1),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function computeSiluGatedBackwardValues(gate, up, gradOutput, swigluLimit = 0) {
|
|
49
|
+
if (gate.length !== up.length || gate.length !== gradOutput.length) {
|
|
50
|
+
throw new Error('gated SiLU backward requires equal-length gate, up, and gradient arrays.');
|
|
51
|
+
}
|
|
52
|
+
const gradGate = new Float32Array(gate.length);
|
|
53
|
+
const gradUp = new Float32Array(gate.length);
|
|
54
|
+
for (let index = 0; index < gate.length; index += 1) {
|
|
55
|
+
const clamped = Math.max(-15, Math.min(15, gate[index]));
|
|
56
|
+
const sigmoid = 1 / (1 + Math.exp(-clamped));
|
|
57
|
+
const activated = gate[index] * sigmoid;
|
|
58
|
+
const product = activated * up[index];
|
|
59
|
+
if (swigluLimit > 0 && Math.abs(product) > swigluLimit) continue;
|
|
60
|
+
const derivative = sigmoid * (1 + (gate[index] * (1 - sigmoid)));
|
|
61
|
+
gradGate[index] = gradOutput[index] * derivative * up[index];
|
|
62
|
+
gradUp[index] = gradOutput[index] * activated;
|
|
63
|
+
}
|
|
64
|
+
return { gradGate, gradUp };
|
|
65
|
+
}
|
|
66
|
+
|
|
33
67
|
const MAX_RESIDUAL_ELEMENTS_PER_DISPATCH = 65535 * 256;
|
|
34
68
|
|
|
35
69
|
export class AutogradTape {
|
|
@@ -149,6 +183,24 @@ export class AutogradTape {
|
|
|
149
183
|
]);
|
|
150
184
|
}
|
|
151
185
|
|
|
186
|
+
if (backwardName === 'reshape_backward') {
|
|
187
|
+
const input = record.inputs[0];
|
|
188
|
+
const inputElements = input.shape.reduce((product, value) => product * value, 1);
|
|
189
|
+
const gradElements = gradOut.shape.reduce((product, value) => product * value, 1);
|
|
190
|
+
if (inputElements !== gradElements) {
|
|
191
|
+
throw new Error(
|
|
192
|
+
`reshape backward element mismatch: input=${inputElements}, grad=${gradElements}`
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
const gradInput = createTensor(
|
|
196
|
+
gradOut.buffer,
|
|
197
|
+
gradOut.dtype,
|
|
198
|
+
[...input.shape],
|
|
199
|
+
'reshape_backward_output'
|
|
200
|
+
);
|
|
201
|
+
return this.filterStoppedGradients(record, [{ input, grad: gradInput }]);
|
|
202
|
+
}
|
|
203
|
+
|
|
152
204
|
if (backwardName === 'row_slice_backward') {
|
|
153
205
|
const rows = Math.max(1, Math.floor(Number(record.options?.rows) || 0));
|
|
154
206
|
const cols = Math.max(1, Math.floor(Number(record.options?.cols) || 0));
|
|
@@ -170,6 +222,40 @@ export class AutogradTape {
|
|
|
170
222
|
return this.filterStoppedGradients(record, [{ input: record.inputs[0], grad: gradInput }]);
|
|
171
223
|
}
|
|
172
224
|
|
|
225
|
+
if (backwardName === 'silu_gated_backward') {
|
|
226
|
+
const [gate, up] = record.inputs;
|
|
227
|
+
const gateValues = await this.readTensorAsF32(gate);
|
|
228
|
+
const upValues = await this.readTensorAsF32(up);
|
|
229
|
+
const gradValues = await this.readTensorAsF32(gradOut);
|
|
230
|
+
const count = Math.max(1, Math.floor(Number(record.options?.count) || 0));
|
|
231
|
+
if (gateValues.length < count || upValues.length < count || gradValues.length < count) {
|
|
232
|
+
throw new Error('gated SiLU backward tensor is shorter than the declared count.');
|
|
233
|
+
}
|
|
234
|
+
const values = computeSiluGatedBackwardValues(
|
|
235
|
+
gateValues.subarray(0, count),
|
|
236
|
+
upValues.subarray(0, count),
|
|
237
|
+
gradValues.subarray(0, count),
|
|
238
|
+
Number.isFinite(record.options?.swigluLimit) ? record.options.swigluLimit : 0
|
|
239
|
+
);
|
|
240
|
+
const gradGate = createUploadedTensor(values.gradGate, 'f32', [...gate.shape], 'silu_gated_grad_gate');
|
|
241
|
+
const gradUp = createUploadedTensor(values.gradUp, 'f32', [...up.shape], 'silu_gated_grad_up');
|
|
242
|
+
return this.filterStoppedGradients(record, [
|
|
243
|
+
{ input: gate, grad: gradGate },
|
|
244
|
+
{ input: up, grad: gradUp },
|
|
245
|
+
]);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
if (backwardName === 'rope_backward') {
|
|
249
|
+
const [input, freqsCos, freqsSin] = record.inputs;
|
|
250
|
+
const gradInput = await backwardKernels.runRoPEBackward(
|
|
251
|
+
gradOut,
|
|
252
|
+
freqsCos,
|
|
253
|
+
freqsSin,
|
|
254
|
+
record.options
|
|
255
|
+
);
|
|
256
|
+
return this.filterStoppedGradients(record, [{ input, grad: gradInput }]);
|
|
257
|
+
}
|
|
258
|
+
|
|
173
259
|
if (backwardName === 'cross_entropy_backward') {
|
|
174
260
|
const [softmax, targets] = record.inputs;
|
|
175
261
|
const gradLogits = await backwardKernels.runCrossEntropyBackward(softmax, targets, gradOut, {
|
|
@@ -184,6 +270,10 @@ export class AutogradTape {
|
|
|
184
270
|
|
|
185
271
|
if (backwardName === 'embed_backward') {
|
|
186
272
|
const [indices, embeddings] = record.inputs;
|
|
273
|
+
if (Array.isArray(record.options?.stopGradInputs)
|
|
274
|
+
&& record.options.stopGradInputs.some((value) => Math.floor(Number(value)) === 1)) {
|
|
275
|
+
return [];
|
|
276
|
+
}
|
|
187
277
|
const gradWeight = await backwardKernels.runEmbedBackward(indices, gradOut, {
|
|
188
278
|
numTokens: Math.max(1, Math.floor(Number(record.options?.numTokens) || 0)),
|
|
189
279
|
hiddenSize: Math.max(1, Math.floor(Number(record.options?.hiddenSize) || 0)),
|
|
@@ -197,19 +287,19 @@ export class AutogradTape {
|
|
|
197
287
|
// Special case for attention which has CPU fallback and complex internal logic
|
|
198
288
|
if (backwardName === 'attention_backward') {
|
|
199
289
|
const [q, k, v, softmax] = record.inputs;
|
|
200
|
-
const { seqLen, numHeads, headDim, scale } = record.options;
|
|
290
|
+
const { seqLen, numHeads, numKVHeads, headDim, scale } = record.options;
|
|
201
291
|
const recomputeForward = record.options.recomputeForward === true || !softmax;
|
|
202
292
|
const { gradQ, gradK, gradV } = recomputeForward
|
|
203
293
|
? await attentionBackwardCpu(
|
|
204
294
|
q, k, v, null, gradOut,
|
|
205
|
-
{ seqLen, numHeads, headDim, scale, causal: record.options.causal }
|
|
295
|
+
{ seqLen, numHeads, numKVHeads, headDim, scale, causal: record.options.causal }
|
|
206
296
|
)
|
|
207
297
|
: await backwardKernels.runAttentionBackward(
|
|
208
298
|
q, k, v, softmax, gradOut,
|
|
209
|
-
{ seqLen, numHeads, headDim, scale, causal: record.options.causal }
|
|
299
|
+
{ seqLen, numHeads, numKVHeads, headDim, scale, causal: record.options.causal }
|
|
210
300
|
).catch(() => attentionBackwardCpu(
|
|
211
301
|
q, k, v, softmax, gradOut,
|
|
212
|
-
{ seqLen, numHeads, headDim, scale, causal: record.options.causal }
|
|
302
|
+
{ seqLen, numHeads, numKVHeads, headDim, scale, causal: record.options.causal }
|
|
213
303
|
));
|
|
214
304
|
return this.filterStoppedGradients(record, [
|
|
215
305
|
{ input: q, grad: gradQ },
|
|
@@ -226,7 +316,7 @@ export class AutogradTape {
|
|
|
226
316
|
}
|
|
227
317
|
|
|
228
318
|
// Prepare options from registry metadata
|
|
229
|
-
|
|
319
|
+
let options = { ...record.options };
|
|
230
320
|
if (entry.params) {
|
|
231
321
|
for (const param of entry.params) {
|
|
232
322
|
if (options[param] === undefined && record.options[param] !== undefined) {
|
|
@@ -234,6 +324,12 @@ export class AutogradTape {
|
|
|
234
324
|
}
|
|
235
325
|
}
|
|
236
326
|
}
|
|
327
|
+
if (backwardName === 'matmul_backward') {
|
|
328
|
+
options = resolveMatmulBackwardOptions(options);
|
|
329
|
+
if (!options.computeGradInput && !options.computeGradWeight) {
|
|
330
|
+
return [];
|
|
331
|
+
}
|
|
332
|
+
}
|
|
237
333
|
|
|
238
334
|
// Standard kernels: (input, weight, gradOut, options) or (input, gradOut, options)
|
|
239
335
|
// Map inputs based on registry "grads" metadata
|
|
@@ -41,7 +41,12 @@ export async function clipGradients(grads, config) {
|
|
|
41
41
|
const scale = maxNorm / (globalNorm + 1e-6);
|
|
42
42
|
const clipped = new Map();
|
|
43
43
|
for (const [param, grad] of grads.entries()) {
|
|
44
|
-
|
|
44
|
+
// WebGPU does not allow a buffer to be bound as both read-only storage and
|
|
45
|
+
// writable storage in the same compute pass. The scale kernel exposes
|
|
46
|
+
// separate input and output bindings, so clipping must use a distinct
|
|
47
|
+
// output buffer even though mutating the gradient would be semantically
|
|
48
|
+
// acceptable here.
|
|
49
|
+
const scaled = await runScale(grad, scale, { inplace: false });
|
|
45
50
|
clipped.set(param, scaled);
|
|
46
51
|
}
|
|
47
52
|
|
|
@@ -4,11 +4,38 @@ import { resolve } from 'node:path';
|
|
|
4
4
|
|
|
5
5
|
import { parseJsonl } from './jsonl.js';
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
export const CAUSAL_LM_IGNORE_TARGET_ID = 0xffffffff;
|
|
8
|
+
|
|
9
|
+
function normalizeTokenIds(values, label) {
|
|
10
|
+
const tokens = Array.from(values || []);
|
|
11
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
12
|
+
const token = tokens[index];
|
|
13
|
+
if (!Number.isInteger(token) || token < 0 || token > CAUSAL_LM_IGNORE_TARGET_ID) {
|
|
14
|
+
throw new Error(`${label} token ${index + 1} must be a uint32 integer.`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return tokens;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function commonPrefixLength(left, right) {
|
|
21
|
+
const limit = Math.min(left.length, right.length);
|
|
22
|
+
let index = 0;
|
|
23
|
+
while (index < limit && left[index] === right[index]) {
|
|
24
|
+
index += 1;
|
|
25
|
+
}
|
|
26
|
+
return index;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function truncatePromptTokens(tokens, budget) {
|
|
30
|
+
if (tokens.length <= budget) {
|
|
31
|
+
return tokens.slice();
|
|
10
32
|
}
|
|
11
|
-
|
|
33
|
+
const headCount = Math.ceil(budget / 2);
|
|
34
|
+
const tailCount = budget - headCount;
|
|
35
|
+
return [
|
|
36
|
+
...tokens.slice(0, headCount),
|
|
37
|
+
...(tailCount > 0 ? tokens.slice(tokens.length - tailCount) : []),
|
|
38
|
+
];
|
|
12
39
|
}
|
|
13
40
|
|
|
14
41
|
export function buildCausalPair(tokens) {
|
|
@@ -100,10 +127,49 @@ export async function tokenizeTextPairs(tokenizer, pairs, options = {}) {
|
|
|
100
127
|
const samples = [];
|
|
101
128
|
const normalizedPairs = mapTextPairs(pairs);
|
|
102
129
|
for (const pair of normalizedPairs) {
|
|
103
|
-
const
|
|
104
|
-
const
|
|
105
|
-
const
|
|
106
|
-
|
|
130
|
+
const promptText = `${pair.prompt}${joinWith}`;
|
|
131
|
+
const fullText = `${promptText}${pair.completion}`;
|
|
132
|
+
const promptTokens = normalizeTokenIds(
|
|
133
|
+
tokenizer.encode(promptText),
|
|
134
|
+
`text pair "${pair.id}" prompt`
|
|
135
|
+
);
|
|
136
|
+
const fullTokens = normalizeTokenIds(
|
|
137
|
+
tokenizer.encode(fullText),
|
|
138
|
+
`text pair "${pair.id}" full text`
|
|
139
|
+
);
|
|
140
|
+
const completionStart = commonPrefixLength(promptTokens, fullTokens);
|
|
141
|
+
const promptPrefixTokens = fullTokens.slice(0, completionStart);
|
|
142
|
+
const completionTokens = fullTokens.slice(completionStart);
|
|
143
|
+
|
|
144
|
+
if (promptPrefixTokens.length < 1) {
|
|
145
|
+
throw new Error(`text pair "${pair.id}" prompt produced no stable prefix tokens.`);
|
|
146
|
+
}
|
|
147
|
+
if (completionTokens.length < 1) {
|
|
148
|
+
throw new Error(`text pair "${pair.id}" completion produced no supervised tokens.`);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
let retainedPromptTokens = promptPrefixTokens;
|
|
152
|
+
if (maxLength != null) {
|
|
153
|
+
if (!Number.isInteger(maxLength) || maxLength < 2) {
|
|
154
|
+
throw new Error('tokenizeTextPairs maxLength must be an integer >= 2 when provided.');
|
|
155
|
+
}
|
|
156
|
+
if (completionTokens.length >= maxLength) {
|
|
157
|
+
throw new Error(
|
|
158
|
+
`text pair "${pair.id}" completion requires ${completionTokens.length} tokens, `
|
|
159
|
+
+ `but maxLength ${maxLength} must also retain at least one prompt token.`
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
retainedPromptTokens = truncatePromptTokens(
|
|
163
|
+
promptPrefixTokens,
|
|
164
|
+
maxLength - completionTokens.length
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const tokens = [...retainedPromptTokens, ...completionTokens];
|
|
169
|
+
const { inputIds, targetIds } = buildCausalPair(tokens);
|
|
170
|
+
const ignoredTargetCount = Math.max(0, retainedPromptTokens.length - 1);
|
|
171
|
+
targetIds.fill(CAUSAL_LM_IGNORE_TARGET_ID, 0, ignoredTargetCount);
|
|
172
|
+
const supervisedTokenCount = targetIds.length - ignoredTargetCount;
|
|
107
173
|
if (inputIds.length > 0) {
|
|
108
174
|
samples.push({
|
|
109
175
|
id: pair.id,
|
|
@@ -112,6 +178,12 @@ export async function tokenizeTextPairs(tokenizer, pairs, options = {}) {
|
|
|
112
178
|
text: fullText,
|
|
113
179
|
prompt: pair.prompt,
|
|
114
180
|
completion: pair.completion,
|
|
181
|
+
promptTokenCount: promptPrefixTokens.length,
|
|
182
|
+
retainedPromptTokenCount: retainedPromptTokens.length,
|
|
183
|
+
truncatedPromptTokenCount: promptPrefixTokens.length - retainedPromptTokens.length,
|
|
184
|
+
completionTokenCount: completionTokens.length,
|
|
185
|
+
ignoredTargetCount,
|
|
186
|
+
supervisedTokenCount,
|
|
115
187
|
});
|
|
116
188
|
}
|
|
117
189
|
}
|