doppler-gpu 0.4.11 → 0.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/README.md +79 -45
- package/models/catalog.json +29 -29
- package/package.json +12 -3
- package/src/client/doppler-registry.json +9 -9
- package/src/client/runtime/lora.js +5 -0
- package/src/client/runtime/model-session.d.ts +69 -1
- package/src/client/runtime/model-session.js +231 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +21 -2
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +9 -4
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/ffn/dense.js +16 -0
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +106 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +1 -0
- package/src/inference/pipelines/text/types.d.ts +28 -0
- package/src/inference/pipelines/text.d.ts +7 -2
- package/src/inference/pipelines/text.js +4 -0
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
import { createTrainingConfig } from '../../config/training-defaults.js';
|
|
2
|
+
import { loadBackwardRegistry } from '../../config/backward-registry-loader.js';
|
|
3
|
+
import { getWeightDtype, isWeightBuffer } from '../../gpu/weight-buffer.js';
|
|
4
|
+
import { createTensor, tensorBytes } from '../../gpu/tensor.js';
|
|
5
|
+
import { runMatmul, runRMSNorm, runResidualAdd } from '../../gpu/kernels/index.js';
|
|
6
|
+
import { readBuffer, releaseBuffer, uploadData } from '../../memory/buffer-pool.js';
|
|
7
|
+
import { LoraAdapter } from './lora.js';
|
|
8
|
+
import { AdamOptimizer } from './optimizer.js';
|
|
9
|
+
import { crossEntropyLoss } from './loss.js';
|
|
10
|
+
import { clipGradients } from './clip.js';
|
|
11
|
+
import { trainStep } from './trainer.js';
|
|
12
|
+
import { OpType } from './autograd.js';
|
|
13
|
+
import { createUploadedTensor } from './tensor-factory.js';
|
|
14
|
+
import { exportLoRAAdapter } from './export.js';
|
|
15
|
+
|
|
16
|
+
function asWeightTensor(weight, shape, label) {
|
|
17
|
+
if (!weight) throw new Error(`native_qwen_lora_missing_weight: ${label}.`);
|
|
18
|
+
if (isWeightBuffer(weight)) {
|
|
19
|
+
return { ...weight, shape: [...shape], label };
|
|
20
|
+
}
|
|
21
|
+
const dtype = getWeightDtype(weight) || 'f32';
|
|
22
|
+
return createTensor(weight, dtype, [...shape], label);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function releaseTensorSet(tensors, preserved) {
|
|
26
|
+
const released = new Set();
|
|
27
|
+
for (const tensor of tensors) {
|
|
28
|
+
const buffer = tensor?.buffer;
|
|
29
|
+
if (!buffer || preserved.has(buffer) || released.has(buffer)) continue;
|
|
30
|
+
released.add(buffer);
|
|
31
|
+
releaseBuffer(buffer);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function encodeBytes(bytes) {
|
|
36
|
+
const view = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
|
|
37
|
+
if (typeof btoa === 'function') {
|
|
38
|
+
let binary = '';
|
|
39
|
+
for (let index = 0; index < view.length; index += 1) binary += String.fromCharCode(view[index]);
|
|
40
|
+
return btoa(binary);
|
|
41
|
+
}
|
|
42
|
+
if (typeof Buffer !== 'undefined') return Buffer.from(view).toString('base64');
|
|
43
|
+
throw new Error('native_qwen_lora_checkpoint_base64_unavailable.');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function decodeBytes(value) {
|
|
47
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
48
|
+
throw new Error('native_qwen_lora_checkpoint_bytes_invalid.');
|
|
49
|
+
}
|
|
50
|
+
if (typeof atob === 'function') {
|
|
51
|
+
const binary = atob(value);
|
|
52
|
+
const bytes = new Uint8Array(binary.length);
|
|
53
|
+
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
|
|
54
|
+
return bytes;
|
|
55
|
+
}
|
|
56
|
+
if (typeof Buffer !== 'undefined') return new Uint8Array(Buffer.from(value, 'base64'));
|
|
57
|
+
throw new Error('native_qwen_lora_checkpoint_base64_unavailable.');
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async function serializeTensor(tensor) {
|
|
61
|
+
const byteLength = tensorBytes(tensor.shape, tensor.dtype);
|
|
62
|
+
return {
|
|
63
|
+
dtype: tensor.dtype,
|
|
64
|
+
shape: [...tensor.shape],
|
|
65
|
+
bytes: encodeBytes(await readBuffer(tensor.buffer, byteLength)),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function restoreTensor(tensor, snapshot, label) {
|
|
70
|
+
if (snapshot?.dtype !== tensor.dtype || JSON.stringify(snapshot?.shape) !== JSON.stringify(tensor.shape)) {
|
|
71
|
+
throw new Error(`native_qwen_lora_checkpoint_tensor_mismatch: ${label}.`);
|
|
72
|
+
}
|
|
73
|
+
const bytes = decodeBytes(snapshot.bytes);
|
|
74
|
+
const expectedBytes = tensorBytes(tensor.shape, tensor.dtype);
|
|
75
|
+
if (bytes.byteLength !== expectedBytes) {
|
|
76
|
+
throw new Error(`native_qwen_lora_checkpoint_tensor_size_mismatch: ${label}.`);
|
|
77
|
+
}
|
|
78
|
+
uploadData(tensor.buffer, bytes);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function createCausalLmObjective(owner) {
|
|
82
|
+
return {
|
|
83
|
+
name: 'native_qwen_completion_masked_sft',
|
|
84
|
+
async forward({ batch, tape }) {
|
|
85
|
+
return owner.forward(batch.inputIds, tape);
|
|
86
|
+
},
|
|
87
|
+
async computeLoss({ batch, config, tape, forwardState }) {
|
|
88
|
+
return {
|
|
89
|
+
loss: await crossEntropyLoss(forwardState.logits, batch.targets, config, tape),
|
|
90
|
+
components: {
|
|
91
|
+
supervised_token_count: batch.supervisedTokenCount,
|
|
92
|
+
ignored_target_count: batch.targets.shape[0] - batch.supervisedTokenCount,
|
|
93
|
+
},
|
|
94
|
+
};
|
|
95
|
+
},
|
|
96
|
+
backwardTargets({ batch, loss, lossScale }) {
|
|
97
|
+
const values = new Float32Array(loss.shape[0]);
|
|
98
|
+
values.fill(lossScale / batch.supervisedTokenCount);
|
|
99
|
+
return createUploadedTensor(values, 'f32', loss.shape, 'native_qwen_lora_loss_grad');
|
|
100
|
+
},
|
|
101
|
+
async metrics({ batch, loss }) {
|
|
102
|
+
const bytes = await readBuffer(loss.buffer, loss.shape[0] * Float32Array.BYTES_PER_ELEMENT);
|
|
103
|
+
const values = new Float32Array(bytes);
|
|
104
|
+
let sum = 0;
|
|
105
|
+
for (const value of values) sum += value;
|
|
106
|
+
return {
|
|
107
|
+
loss: sum / batch.supervisedTokenCount,
|
|
108
|
+
supervised_token_count: batch.supervisedTokenCount,
|
|
109
|
+
};
|
|
110
|
+
},
|
|
111
|
+
cleanup({ tape }) {
|
|
112
|
+
owner.cleanupStep(tape);
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export class NativeQwenLoRATrainer {
|
|
118
|
+
constructor(options) {
|
|
119
|
+
const pipeline = options?.pipeline;
|
|
120
|
+
const config = pipeline?.modelConfig;
|
|
121
|
+
if (!pipeline || !config || !(pipeline.weights instanceof Map)) {
|
|
122
|
+
throw new Error('NativeQwenLoRATrainer requires a loaded Doppler text pipeline.');
|
|
123
|
+
}
|
|
124
|
+
const layerIdx = Number(options.layerIdx);
|
|
125
|
+
if (!Number.isInteger(layerIdx) || layerIdx !== config.numLayers - 1) {
|
|
126
|
+
throw new Error(`native_lora_target_not_supported: layerIdx must be ${config.numLayers - 1}.`);
|
|
127
|
+
}
|
|
128
|
+
if (options.module !== 'down_proj') {
|
|
129
|
+
throw new Error('native_lora_target_not_supported: module must be down_proj.');
|
|
130
|
+
}
|
|
131
|
+
if (config.layerTypes?.[layerIdx] !== 'full_attention' || config.postFeedforwardNorm === true) {
|
|
132
|
+
throw new Error('native_lora_target_not_supported: target requires a final full-attention layer without post-FFN normalization.');
|
|
133
|
+
}
|
|
134
|
+
const lmHeadSource = pipeline.weights.get('lm_head')
|
|
135
|
+
|| (config.useTiedEmbeddings === true ? pipeline.weights.get('embed') : null);
|
|
136
|
+
const lmHead = asWeightTensor(lmHeadSource, [config.vocabSize, config.hiddenSize], 'native_qwen_lm_head');
|
|
137
|
+
if (lmHead.dtype !== 'q4k' || lmHead.layout !== 'row') {
|
|
138
|
+
throw new Error('native_lora_target_not_supported: Qwen LM head must be row-wise Q4K [vocab,hidden].');
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
this.pipeline = pipeline;
|
|
142
|
+
this.config = config;
|
|
143
|
+
this.layerIdx = layerIdx;
|
|
144
|
+
this.module = 'down_proj';
|
|
145
|
+
this.baseModelId = String(options.baseModelId || pipeline.manifest?.modelId || '');
|
|
146
|
+
this.lmHead = lmHead;
|
|
147
|
+
this.finalNorm = asWeightTensor(
|
|
148
|
+
pipeline.weights.get('final_norm'),
|
|
149
|
+
[config.hiddenSize],
|
|
150
|
+
'native_qwen_final_norm'
|
|
151
|
+
);
|
|
152
|
+
this.trainingConfig = createTrainingConfig({
|
|
153
|
+
training: {
|
|
154
|
+
enabled: true,
|
|
155
|
+
optimizer: options.optimizer,
|
|
156
|
+
gradient: options.gradient,
|
|
157
|
+
precision: options.precision,
|
|
158
|
+
lossScaling: { enabled: false },
|
|
159
|
+
distill: { freeze: { base: true, lora: false } },
|
|
160
|
+
},
|
|
161
|
+
});
|
|
162
|
+
this.adapter = new LoraAdapter({
|
|
163
|
+
inDim: config.intermediateSize,
|
|
164
|
+
outDim: config.hiddenSize,
|
|
165
|
+
rank: options.rank,
|
|
166
|
+
alpha: options.alpha,
|
|
167
|
+
dtype: this.trainingConfig.training.precision.loraParams,
|
|
168
|
+
});
|
|
169
|
+
this.optimizer = new AdamOptimizer(this.trainingConfig);
|
|
170
|
+
this.registry = loadBackwardRegistry();
|
|
171
|
+
this.activeCapture = null;
|
|
172
|
+
this.disposed = false;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
paramGroups() {
|
|
176
|
+
return {
|
|
177
|
+
base: [this.finalNorm, this.lmHead],
|
|
178
|
+
lora: [this.adapter.A, this.adapter.B],
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
async forward(inputIds, tape) {
|
|
183
|
+
if (this.activeCapture) throw new Error('native_qwen_lora_step_already_active.');
|
|
184
|
+
const capture = await this.pipeline.prefillForLoRATraining(inputIds, {
|
|
185
|
+
layerIdx: this.layerIdx,
|
|
186
|
+
module: this.module,
|
|
187
|
+
});
|
|
188
|
+
this.activeCapture = capture;
|
|
189
|
+
const delta = await this.adapter.forward(capture.activation, tape);
|
|
190
|
+
const adaptedHidden = await tape.record(
|
|
191
|
+
OpType.RESIDUAL_ADD,
|
|
192
|
+
(base, update) => runResidualAdd(base, update, inputIds.length * this.config.hiddenSize),
|
|
193
|
+
[capture.baseHidden, delta],
|
|
194
|
+
{ size: inputIds.length * this.config.hiddenSize, stopGradInputs: [0] }
|
|
195
|
+
);
|
|
196
|
+
const finalHidden = await tape.record(
|
|
197
|
+
OpType.RMSNORM,
|
|
198
|
+
(hidden, weight) => runRMSNorm(hidden, weight, this.config.rmsNormEps, {
|
|
199
|
+
batchSize: inputIds.length,
|
|
200
|
+
hiddenSize: this.config.hiddenSize,
|
|
201
|
+
rmsNormWeightOffset: this.config.rmsNormWeightOffset === true,
|
|
202
|
+
}),
|
|
203
|
+
[adaptedHidden, this.finalNorm],
|
|
204
|
+
{
|
|
205
|
+
numTokens: inputIds.length,
|
|
206
|
+
hiddenSize: this.config.hiddenSize,
|
|
207
|
+
eps: this.config.rmsNormEps,
|
|
208
|
+
rmsNormWeightOffset: this.config.rmsNormWeightOffset === true,
|
|
209
|
+
stopGradInputs: [1],
|
|
210
|
+
}
|
|
211
|
+
);
|
|
212
|
+
const logits = await tape.record(
|
|
213
|
+
OpType.MATMUL,
|
|
214
|
+
(hidden, weight) => runMatmul(
|
|
215
|
+
hidden,
|
|
216
|
+
weight,
|
|
217
|
+
inputIds.length,
|
|
218
|
+
this.config.vocabSize,
|
|
219
|
+
this.config.hiddenSize,
|
|
220
|
+
{
|
|
221
|
+
transposeB: true,
|
|
222
|
+
outputDtype: 'f32',
|
|
223
|
+
role: 'lm_head_prefill',
|
|
224
|
+
phaseOverride: 'prefill',
|
|
225
|
+
kernelPath: this.pipeline.resolvedKernelPath,
|
|
226
|
+
executionPolicies: this.pipeline.executionV1State?.policies ?? null,
|
|
227
|
+
}
|
|
228
|
+
),
|
|
229
|
+
[finalHidden, this.lmHead],
|
|
230
|
+
{
|
|
231
|
+
M: inputIds.length,
|
|
232
|
+
N: this.config.vocabSize,
|
|
233
|
+
K: this.config.hiddenSize,
|
|
234
|
+
transposeB: true,
|
|
235
|
+
stopGradInputs: [1],
|
|
236
|
+
}
|
|
237
|
+
);
|
|
238
|
+
return { logits };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
cleanupStep(tape) {
|
|
242
|
+
const capture = this.activeCapture;
|
|
243
|
+
const preserved = new Set([
|
|
244
|
+
this.adapter.A.buffer,
|
|
245
|
+
this.adapter.B.buffer,
|
|
246
|
+
this.finalNorm.buffer,
|
|
247
|
+
this.lmHead.buffer,
|
|
248
|
+
capture?.activation?.buffer,
|
|
249
|
+
capture?.baseHidden?.buffer,
|
|
250
|
+
]);
|
|
251
|
+
releaseTensorSet(tape.records.map((record) => record.output), preserved);
|
|
252
|
+
capture?.dispose();
|
|
253
|
+
this.activeCapture = null;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async trainStep(inputIds, targetIds, supervisedTokenCount) {
|
|
257
|
+
if (this.disposed) throw new Error('NativeQwenLoRATrainer is disposed.');
|
|
258
|
+
if (inputIds.length !== targetIds.length) {
|
|
259
|
+
throw new Error('native_qwen_lora_target_length_mismatch.');
|
|
260
|
+
}
|
|
261
|
+
if (!Number.isInteger(supervisedTokenCount) || supervisedTokenCount < 1) {
|
|
262
|
+
throw new Error('native_qwen_lora_supervised_token_count_invalid.');
|
|
263
|
+
}
|
|
264
|
+
const actualSupervisedTokenCount = Array.from(targetIds)
|
|
265
|
+
.filter((target) => Number.isInteger(target) && target >= 0 && target < this.config.vocabSize)
|
|
266
|
+
.length;
|
|
267
|
+
if (actualSupervisedTokenCount !== supervisedTokenCount) {
|
|
268
|
+
throw new Error(
|
|
269
|
+
`native_qwen_lora_supervised_token_count_mismatch: expected ${actualSupervisedTokenCount}, got ${supervisedTokenCount}.`
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
const targets = createUploadedTensor(
|
|
273
|
+
new Uint32Array(targetIds),
|
|
274
|
+
'f32',
|
|
275
|
+
[targetIds.length],
|
|
276
|
+
'native_qwen_lora_targets'
|
|
277
|
+
);
|
|
278
|
+
try {
|
|
279
|
+
const result = await trainStep(
|
|
280
|
+
this,
|
|
281
|
+
{ inputIds: Array.from(inputIds), targets, supervisedTokenCount },
|
|
282
|
+
this.trainingConfig,
|
|
283
|
+
{
|
|
284
|
+
registry: this.registry,
|
|
285
|
+
crossEntropyLoss,
|
|
286
|
+
clipGradients,
|
|
287
|
+
optimizer: this.optimizer,
|
|
288
|
+
trainingObjective: createCausalLmObjective(this),
|
|
289
|
+
}
|
|
290
|
+
);
|
|
291
|
+
const gradientNorm = result.clipMetrics?.gradient_norm_unclipped ?? null;
|
|
292
|
+
releaseTensorSet(result.grads.values(), new Set());
|
|
293
|
+
return {
|
|
294
|
+
loss: result.objectiveMetrics.loss,
|
|
295
|
+
gradientNorm,
|
|
296
|
+
optimizer: result.optimizerMetrics,
|
|
297
|
+
};
|
|
298
|
+
} finally {
|
|
299
|
+
releaseBuffer(targets.buffer);
|
|
300
|
+
if (this.activeCapture) {
|
|
301
|
+
this.activeCapture.dispose();
|
|
302
|
+
this.activeCapture = null;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
exportAdapter(options) {
|
|
308
|
+
return exportLoRAAdapter({
|
|
309
|
+
id: options.id,
|
|
310
|
+
name: options.name,
|
|
311
|
+
baseModel: this.baseModelId,
|
|
312
|
+
rank: this.adapter.rank,
|
|
313
|
+
alpha: this.adapter.alpha,
|
|
314
|
+
targetModules: ['down_proj'],
|
|
315
|
+
tensors: [
|
|
316
|
+
{ name: `layers.${this.layerIdx}.down_proj.lora_a`, tensor: this.adapter.A },
|
|
317
|
+
{ name: `layers.${this.layerIdx}.down_proj.lora_b`, tensor: this.adapter.B },
|
|
318
|
+
],
|
|
319
|
+
weightsFormat: 'safetensors',
|
|
320
|
+
weightsPath: options.weightsPath,
|
|
321
|
+
metadata: {
|
|
322
|
+
trainingBackend: 'webgpu_native',
|
|
323
|
+
trainingSurfaces: ['browser', 'node', 'bun'],
|
|
324
|
+
targetLayer: this.layerIdx,
|
|
325
|
+
},
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
async createCheckpoint() {
|
|
330
|
+
if (this.disposed) throw new Error('NativeQwenLoRATrainer is disposed.');
|
|
331
|
+
const params = [this.adapter.A, this.adapter.B];
|
|
332
|
+
const optimizer = [];
|
|
333
|
+
for (const param of params) {
|
|
334
|
+
const state = this.optimizer.getState(param);
|
|
335
|
+
optimizer.push({
|
|
336
|
+
m: await serializeTensor(state.m),
|
|
337
|
+
v: await serializeTensor(state.v),
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
return {
|
|
341
|
+
schemaVersion: 1,
|
|
342
|
+
backend: 'webgpu_native',
|
|
343
|
+
baseModelId: this.baseModelId,
|
|
344
|
+
target: { layerIdx: this.layerIdx, module: this.module },
|
|
345
|
+
rank: this.adapter.rank,
|
|
346
|
+
alpha: this.adapter.alpha,
|
|
347
|
+
stepCount: this.optimizer.stepCount,
|
|
348
|
+
adapter: {
|
|
349
|
+
A: await serializeTensor(this.adapter.A),
|
|
350
|
+
B: await serializeTensor(this.adapter.B),
|
|
351
|
+
},
|
|
352
|
+
optimizer,
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
restoreCheckpoint(checkpoint) {
|
|
357
|
+
if (this.disposed) throw new Error('NativeQwenLoRATrainer is disposed.');
|
|
358
|
+
if (
|
|
359
|
+
checkpoint?.schemaVersion !== 1
|
|
360
|
+
|| checkpoint.backend !== 'webgpu_native'
|
|
361
|
+
|| checkpoint.baseModelId !== this.baseModelId
|
|
362
|
+
|| checkpoint.target?.layerIdx !== this.layerIdx
|
|
363
|
+
|| checkpoint.target?.module !== this.module
|
|
364
|
+
|| checkpoint.rank !== this.adapter.rank
|
|
365
|
+
|| checkpoint.alpha !== this.adapter.alpha
|
|
366
|
+
|| !Number.isInteger(checkpoint.stepCount)
|
|
367
|
+
|| checkpoint.stepCount < 0
|
|
368
|
+
|| !Array.isArray(checkpoint.optimizer)
|
|
369
|
+
|| checkpoint.optimizer.length !== 2
|
|
370
|
+
) {
|
|
371
|
+
throw new Error('native_qwen_lora_checkpoint_identity_mismatch.');
|
|
372
|
+
}
|
|
373
|
+
const params = [this.adapter.A, this.adapter.B];
|
|
374
|
+
restoreTensor(this.adapter.A, checkpoint.adapter?.A, 'adapter.A');
|
|
375
|
+
restoreTensor(this.adapter.B, checkpoint.adapter?.B, 'adapter.B');
|
|
376
|
+
params.forEach((param, index) => {
|
|
377
|
+
const state = this.optimizer.getState(param);
|
|
378
|
+
restoreTensor(state.m, checkpoint.optimizer[index]?.m, `optimizer.${index}.m`);
|
|
379
|
+
restoreTensor(state.v, checkpoint.optimizer[index]?.v, `optimizer.${index}.v`);
|
|
380
|
+
});
|
|
381
|
+
this.optimizer.stepCount = checkpoint.stepCount;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
dispose() {
|
|
385
|
+
if (this.disposed) return;
|
|
386
|
+
this.disposed = true;
|
|
387
|
+
if (this.activeCapture) this.activeCapture.dispose();
|
|
388
|
+
for (const state of this.optimizer.state.values()) {
|
|
389
|
+
releaseBuffer(state.m.buffer);
|
|
390
|
+
releaseBuffer(state.v.buffer);
|
|
391
|
+
}
|
|
392
|
+
this.optimizer.state.clear();
|
|
393
|
+
this.adapter.dispose();
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
export function createNativeQwenLoRATrainer(options) {
|
|
398
|
+
return new NativeQwenLoRATrainer(options);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
export async function loadNativeQwenTrainingPipeline(modelUrl, options = {}) {
|
|
402
|
+
if (typeof modelUrl !== 'string' || modelUrl.trim().length === 0) {
|
|
403
|
+
throw new Error('loadNativeQwenTrainingPipeline requires an RDRR model URL.');
|
|
404
|
+
}
|
|
405
|
+
const { initializeInference } = await import('../../inference/test-harness.js');
|
|
406
|
+
const loaded = await initializeInference(modelUrl.replace(/\/$/, ''), options);
|
|
407
|
+
if (loaded.manifest?.modelId !== 'qwen-3-5-0-8b-q4k-ehaf16') {
|
|
408
|
+
await loaded.pipeline.unload();
|
|
409
|
+
throw new Error(
|
|
410
|
+
`native_qwen_lora_model_not_supported: expected qwen-3-5-0-8b-q4k-ehaf16, got ${String(loaded.manifest?.modelId)}.`
|
|
411
|
+
);
|
|
412
|
+
}
|
|
413
|
+
return loaded;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
export async function trainNativeQwenSftLoRA(options) {
|
|
417
|
+
if (!Array.isArray(options?.samples) || options.samples.length === 0) {
|
|
418
|
+
throw new Error('trainNativeQwenSftLoRA requires tokenized samples.');
|
|
419
|
+
}
|
|
420
|
+
const trainer = createNativeQwenLoRATrainer(options);
|
|
421
|
+
const metrics = [];
|
|
422
|
+
try {
|
|
423
|
+
const maxSteps = options.maxSteps === undefined ? options.samples.length : Number(options.maxSteps);
|
|
424
|
+
if (!Number.isInteger(maxSteps) || maxSteps < 1) {
|
|
425
|
+
throw new Error('trainNativeQwenSftLoRA maxSteps must be a positive integer.');
|
|
426
|
+
}
|
|
427
|
+
for (let step = 0; step < maxSteps; step += 1) {
|
|
428
|
+
const sample = options.samples[step % options.samples.length];
|
|
429
|
+
metrics.push(await trainer.trainStep(
|
|
430
|
+
sample.inputIds,
|
|
431
|
+
sample.targetIds,
|
|
432
|
+
sample.supervisedTokenCount
|
|
433
|
+
));
|
|
434
|
+
}
|
|
435
|
+
const adapter = options.export
|
|
436
|
+
? await trainer.exportAdapter(options.export)
|
|
437
|
+
: null;
|
|
438
|
+
return {
|
|
439
|
+
backend: 'webgpu_native',
|
|
440
|
+
surfaces: ['browser', 'node', 'bun'],
|
|
441
|
+
baseModelId: trainer.baseModelId,
|
|
442
|
+
target: { layerIdx: trainer.layerIdx, module: trainer.module },
|
|
443
|
+
metrics,
|
|
444
|
+
adapter,
|
|
445
|
+
};
|
|
446
|
+
} finally {
|
|
447
|
+
trainer.dispose();
|
|
448
|
+
}
|
|
449
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { TrainingConfigSchema } from '../../../config/training-defaults.d.ts';
|
|
2
|
+
import type { Tensor } from '../../../gpu/tensor.js';
|
|
3
|
+
|
|
4
|
+
export interface TrainingObjectiveContext {
|
|
5
|
+
model: unknown;
|
|
6
|
+
batch: Record<string, unknown>;
|
|
7
|
+
config: TrainingConfigSchema;
|
|
8
|
+
tape: unknown;
|
|
9
|
+
options: Record<string, unknown>;
|
|
10
|
+
lossScale: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface TrainingObjectiveLossResult {
|
|
14
|
+
loss: Tensor;
|
|
15
|
+
components?: Record<string, number | string | boolean | null | undefined>;
|
|
16
|
+
[key: string]: unknown;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface TrainingObjectiveBackwardSeed {
|
|
20
|
+
tensor: Tensor;
|
|
21
|
+
grad: Tensor;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export type TrainingObjectiveBackwardTargets =
|
|
25
|
+
| Tensor
|
|
26
|
+
| null
|
|
27
|
+
| Map<Tensor, Tensor>
|
|
28
|
+
| TrainingObjectiveBackwardSeed[]
|
|
29
|
+
| { seeds: TrainingObjectiveBackwardSeed[] };
|
|
30
|
+
|
|
31
|
+
export interface TrainingObjective {
|
|
32
|
+
name: string;
|
|
33
|
+
prepareBatch?: (
|
|
34
|
+
context: Omit<TrainingObjectiveContext, 'batch' | 'tape'> & { batch: Record<string, unknown> }
|
|
35
|
+
) => Promise<Record<string, unknown>> | Record<string, unknown>;
|
|
36
|
+
forward: (
|
|
37
|
+
context: TrainingObjectiveContext
|
|
38
|
+
) => Promise<Record<string, unknown>> | Record<string, unknown>;
|
|
39
|
+
computeLoss: (
|
|
40
|
+
context: TrainingObjectiveContext & { forwardState: Record<string, unknown> }
|
|
41
|
+
) => Promise<Tensor | TrainingObjectiveLossResult> | Tensor | TrainingObjectiveLossResult;
|
|
42
|
+
backwardTargets?: (
|
|
43
|
+
context: TrainingObjectiveContext & { loss: Tensor; lossResult: TrainingObjectiveLossResult }
|
|
44
|
+
) => Promise<TrainingObjectiveBackwardTargets> | TrainingObjectiveBackwardTargets;
|
|
45
|
+
metrics?: (
|
|
46
|
+
context: TrainingObjectiveContext & {
|
|
47
|
+
forwardState: Record<string, unknown>;
|
|
48
|
+
loss: Tensor;
|
|
49
|
+
lossResult: TrainingObjectiveLossResult;
|
|
50
|
+
}
|
|
51
|
+
) => Promise<Record<string, number | string | null | undefined>> | Record<string, number | string | null | undefined>;
|
|
52
|
+
cleanup?: (
|
|
53
|
+
context: TrainingObjectiveContext & { preparedBatch: Record<string, unknown> }
|
|
54
|
+
) => Promise<void> | void;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export declare function createTrainingObjective(definition: Partial<TrainingObjective>): TrainingObjective;
|
|
58
|
+
export declare function isTrainingObjective(value: unknown): value is TrainingObjective;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { TrainingConfigSchema } from '../../config/training-defaults.d.ts';
|
|
2
|
+
import type { Tensor } from '../../gpu/tensor.js';
|
|
3
|
+
|
|
4
|
+
import type { OptimizerMetrics } from './trainer.d.ts';
|
|
5
|
+
|
|
6
|
+
export declare class AdamOptimizer {
|
|
7
|
+
constructor(config: TrainingConfigSchema);
|
|
8
|
+
config: TrainingConfigSchema;
|
|
9
|
+
state: Map<Tensor, { m: Tensor; v: Tensor }>;
|
|
10
|
+
stepCount: number;
|
|
11
|
+
getState(param: Tensor): { m: Tensor; v: Tensor };
|
|
12
|
+
step(
|
|
13
|
+
params: Tensor[],
|
|
14
|
+
grads: Map<Tensor, Tensor>,
|
|
15
|
+
trainingConfig: TrainingConfigSchema,
|
|
16
|
+
context?: {
|
|
17
|
+
trainableGroups?: string[];
|
|
18
|
+
frozenGroups?: string[];
|
|
19
|
+
allGroups?: string[];
|
|
20
|
+
} | null
|
|
21
|
+
): Promise<OptimizerMetrics>;
|
|
22
|
+
}
|