doppler-gpu 0.4.11 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/README.md +13 -1
- package/models/catalog.json +29 -29
- package/package.json +12 -3
- package/src/client/doppler-registry.json +9 -9
- package/src/client/runtime/lora.js +5 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +21 -2
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +9 -4
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/ffn/dense.js +16 -0
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +106 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +1 -0
- package/src/inference/pipelines/text/types.d.ts +28 -0
- package/src/inference/pipelines/text.d.ts +7 -2
- package/src/inference/pipelines/text.js +4 -0
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -196,16 +196,28 @@ export function resolveMatmulBackwardDxVariant(weight) {
|
|
|
196
196
|
const dtype = String(weight?.dtype || 'f32').toLowerCase();
|
|
197
197
|
if (dtype === 'f16') return 'f16_weight';
|
|
198
198
|
if (dtype === 'f32') return 'default';
|
|
199
|
+
if (dtype === 'q4k') return 'q4k_weight';
|
|
199
200
|
throw new Error(`matmul backward dX does not support weight dtype "${dtype}".`);
|
|
200
201
|
}
|
|
201
202
|
|
|
203
|
+
function resolveMatmulBackwardDxDispatch(weight, K, transposeB) {
|
|
204
|
+
const variant = resolveMatmulBackwardDxVariant(weight);
|
|
205
|
+
if (variant === 'q4k_weight' && !transposeB) {
|
|
206
|
+
throw new Error('matmul backward dX requires transposeB=true for row-wise Q4K weights [N,K].');
|
|
207
|
+
}
|
|
208
|
+
return {
|
|
209
|
+
variant,
|
|
210
|
+
numBlocksPerRow: variant === 'q4k_weight' ? Math.ceil(K / 256) : 0,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
202
214
|
export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
|
|
203
215
|
const { alpha = 1.0, transposeB = false, outputBuffer = null } = options;
|
|
204
216
|
const device = getDevice();
|
|
205
217
|
const outputSize = M * K * 4;
|
|
206
218
|
const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
|
|
207
|
-
|
|
208
|
-
const pipeline = await createPipeline('matmul_backward',
|
|
219
|
+
const dispatchConfig = resolveMatmulBackwardDxDispatch(W, K, transposeB);
|
|
220
|
+
const pipeline = await createPipeline('matmul_backward', dispatchConfig.variant);
|
|
209
221
|
|
|
210
222
|
const uniformBuffer = createUniformBufferWithView(
|
|
211
223
|
'matmul_backward_uniforms',
|
|
@@ -216,6 +228,7 @@ export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
|
|
|
216
228
|
view.setUint32(8, K, true);
|
|
217
229
|
view.setFloat32(12, alpha, true);
|
|
218
230
|
view.setUint32(16, transposeB ? 1 : 0, true);
|
|
231
|
+
view.setUint32(20, dispatchConfig.numBlocksPerRow, true);
|
|
219
232
|
},
|
|
220
233
|
null,
|
|
221
234
|
device
|
|
@@ -250,8 +263,8 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
|
|
|
250
263
|
const device = recorder.device;
|
|
251
264
|
const outputSize = M * K * 4;
|
|
252
265
|
const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
|
|
253
|
-
|
|
254
|
-
const pipeline = await createPipeline('matmul_backward',
|
|
266
|
+
const dispatchConfig = resolveMatmulBackwardDxDispatch(W, K, transposeB);
|
|
267
|
+
const pipeline = await createPipeline('matmul_backward', dispatchConfig.variant);
|
|
255
268
|
|
|
256
269
|
const uniformBuffer = createUniformBufferWithView(
|
|
257
270
|
'matmul_backward_uniforms',
|
|
@@ -262,6 +275,7 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
|
|
|
262
275
|
view.setUint32(8, K, true);
|
|
263
276
|
view.setFloat32(12, alpha, true);
|
|
264
277
|
view.setUint32(16, transposeB ? 1 : 0, true);
|
|
278
|
+
view.setUint32(20, dispatchConfig.numBlocksPerRow, true);
|
|
265
279
|
},
|
|
266
280
|
recorder
|
|
267
281
|
);
|
|
@@ -49,6 +49,19 @@ function resolveActivationOp(hiddenActivation) {
|
|
|
49
49
|
return selectRuleValue('inference', 'ffn', 'activationOp', { hiddenActivation });
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
async function captureTrainingActivation(context, layerIdx, tensor, numTokens, hiddenSize) {
|
|
53
|
+
const capture = context.trainingCapture;
|
|
54
|
+
if (!capture || capture.layerIdx !== layerIdx || capture.stage !== 'ffn_act') return;
|
|
55
|
+
await capture.capture({
|
|
56
|
+
layerIdx,
|
|
57
|
+
stage: 'ffn_act',
|
|
58
|
+
tensor,
|
|
59
|
+
numTokens,
|
|
60
|
+
hiddenSize,
|
|
61
|
+
recorder: context.recorder ?? null,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
52
65
|
function hasQ4KMaterialization(weight) {
|
|
53
66
|
return isWeightBuffer(weight) && !!weight.materializations?.q4k?.buffer;
|
|
54
67
|
}
|
|
@@ -830,6 +843,7 @@ export async function runDenseFFNGPU(
|
|
|
830
843
|
});
|
|
831
844
|
}
|
|
832
845
|
enqueueRecordedDenseHealth(context, layerIdx, 'ffn_act', activatedOutput, numTokens * intermediateSize);
|
|
846
|
+
await captureTrainingActivation(context, layerIdx, activatedOutput, numTokens, intermediateSize);
|
|
833
847
|
|
|
834
848
|
if (recorder) {
|
|
835
849
|
recorder.trackTemporaryBuffer(gateUpOutput.buffer);
|
|
@@ -1056,6 +1070,7 @@ export async function runDenseFFNGPU(
|
|
|
1056
1070
|
dtype: fusedOutput.dtype,
|
|
1057
1071
|
});
|
|
1058
1072
|
enqueueRecordedDenseHealth(context, layerIdx, 'ffn_fused_gate_up', fusedOutput, numTokens * intermediateSize);
|
|
1073
|
+
await captureTrainingActivation(context, layerIdx, fusedOutput, numTokens, intermediateSize);
|
|
1059
1074
|
|
|
1060
1075
|
let downInput = fusedOutput;
|
|
1061
1076
|
if (downInputDtype && fusedOutput.dtype !== downInputDtype) {
|
|
@@ -1469,6 +1484,7 @@ export async function runDenseFFNGPU(
|
|
|
1469
1484
|
dtype: activatedOutput.dtype,
|
|
1470
1485
|
});
|
|
1471
1486
|
enqueueRecordedDenseHealth(context, layerIdx, 'ffn_act', activatedOutput, numTokens * intermediateSize);
|
|
1487
|
+
await captureTrainingActivation(context, layerIdx, activatedOutput, numTokens, intermediateSize);
|
|
1472
1488
|
|
|
1473
1489
|
if (recorder) {
|
|
1474
1490
|
recorder.trackTemporaryBuffer(gateOutput.buffer);
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import type { CommandRecorder, ProfileTimings } from '../../../gpu/command-recorder.js';
|
|
11
11
|
import type { PipelineState } from './state.js';
|
|
12
|
-
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext, WorkloadPhaseTiming } from './types.js';
|
|
12
|
+
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext, WorkloadPhaseTiming, NativeLoRAPrefillOptions, NativeLoRAPrefillResult } from './types.js';
|
|
13
13
|
import type { LogitsConfig, LogitsWeights } from './logits/index.js';
|
|
14
14
|
import type { WeightBufferConfig } from './weights.js';
|
|
15
15
|
import type { ChatMessage } from './chat-format.js';
|
|
@@ -65,6 +65,10 @@ export declare class PipelineGenerator {
|
|
|
65
65
|
resetGenerationState(): void;
|
|
66
66
|
resetToSeqLen(seqLen: number): void;
|
|
67
67
|
prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
|
|
68
|
+
prefillForLoRATraining(
|
|
69
|
+
inputIds: readonly number[] | Int32Array | Uint32Array,
|
|
70
|
+
options: NativeLoRAPrefillOptions
|
|
71
|
+
): Promise<NativeLoRAPrefillResult>;
|
|
68
72
|
computeDiffusionGemmaCanvasLogits(
|
|
69
73
|
args: DiffusionGemmaCanvasLogitsInput,
|
|
70
74
|
options?: GenerateOptions & { __internalGenerate?: boolean }
|
|
@@ -2115,6 +2115,111 @@ export class PipelineGenerator {
|
|
|
2115
2115
|
}
|
|
2116
2116
|
}
|
|
2117
2117
|
|
|
2118
|
+
async prefillForLoRATraining(inputIds, options = {}) {
|
|
2119
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2120
|
+
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
2121
|
+
if (!Array.isArray(inputIds) && !(inputIds instanceof Int32Array) && !(inputIds instanceof Uint32Array)) {
|
|
2122
|
+
throw new Error('prefillForLoRATraining requires token IDs.');
|
|
2123
|
+
}
|
|
2124
|
+
const tokens = Array.from(inputIds, (value) => Number(value));
|
|
2125
|
+
if (tokens.length === 0) throw new Error('prefillForLoRATraining requires at least one token.');
|
|
2126
|
+
this._assertTokenIdsInRange(tokens, 'prefillForLoRATraining.inputIds');
|
|
2127
|
+
|
|
2128
|
+
const config = this.#state.modelConfig;
|
|
2129
|
+
const layerIdx = Number(options.layerIdx);
|
|
2130
|
+
if (!Number.isInteger(layerIdx) || layerIdx !== config.numLayers - 1) {
|
|
2131
|
+
throw new Error(`native_lora_target_not_supported: layerIdx must be the final layer (${config.numLayers - 1}).`);
|
|
2132
|
+
}
|
|
2133
|
+
if (options.module !== 'down_proj') {
|
|
2134
|
+
throw new Error('native_lora_target_not_supported: module must be down_proj.');
|
|
2135
|
+
}
|
|
2136
|
+
if (config.layerTypes?.[layerIdx] !== 'full_attention') {
|
|
2137
|
+
throw new Error('native_lora_target_not_supported: final layer must be full_attention.');
|
|
2138
|
+
}
|
|
2139
|
+
if (config.postFeedforwardNorm === true) {
|
|
2140
|
+
throw new Error('native_lora_target_not_supported: final-layer post-feedforward normalization is not supported.');
|
|
2141
|
+
}
|
|
2142
|
+
if (this.#state.lora) {
|
|
2143
|
+
throw new Error('native_lora_base_capture_requires_inactive_adapter.');
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
this.resetToSeqLen(0);
|
|
2147
|
+
this._resetDecodeRuntimeState();
|
|
2148
|
+
const opts = resolvePrefillOptions(this.#state, {
|
|
2149
|
+
...options,
|
|
2150
|
+
inputIds: tokens,
|
|
2151
|
+
useChatTemplate: false,
|
|
2152
|
+
});
|
|
2153
|
+
let activationTensor = null;
|
|
2154
|
+
let hiddenTensor = null;
|
|
2155
|
+
let disposed = false;
|
|
2156
|
+
const releaseOwned = () => {
|
|
2157
|
+
if (disposed) return;
|
|
2158
|
+
disposed = true;
|
|
2159
|
+
if (activationTensor?.buffer) releaseBuffer(activationTensor.buffer);
|
|
2160
|
+
if (hiddenTensor?.buffer) releaseBuffer(hiddenTensor.buffer);
|
|
2161
|
+
};
|
|
2162
|
+
const capture = async ({ tensor, numTokens, hiddenSize, recorder }) => {
|
|
2163
|
+
if (activationTensor) {
|
|
2164
|
+
throw new Error('native_lora_training_capture_duplicated.');
|
|
2165
|
+
}
|
|
2166
|
+
const bytesPerElement = tensor.dtype === 'f16' ? 2 : 4;
|
|
2167
|
+
const byteLength = Math.ceil((numTokens * hiddenSize * bytesPerElement) / 4) * 4;
|
|
2168
|
+
const buffer = acquireBuffer(byteLength, undefined, `L${layerIdx}.native_lora_ffn_act`);
|
|
2169
|
+
if (recorder) {
|
|
2170
|
+
recorder.getEncoder().copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
|
|
2171
|
+
} else {
|
|
2172
|
+
const device = getDevice();
|
|
2173
|
+
const encoder = device.createCommandEncoder({ label: `L${layerIdx}.native_lora_capture` });
|
|
2174
|
+
encoder.copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
|
|
2175
|
+
device.queue.submit([encoder.finish()]);
|
|
2176
|
+
}
|
|
2177
|
+
activationTensor = createTensor(
|
|
2178
|
+
buffer,
|
|
2179
|
+
tensor.dtype,
|
|
2180
|
+
[numTokens, hiddenSize],
|
|
2181
|
+
`L${layerIdx}.native_lora_ffn_act`
|
|
2182
|
+
);
|
|
2183
|
+
};
|
|
2184
|
+
|
|
2185
|
+
try {
|
|
2186
|
+
const prefill = await this._prefillToHidden(tokens, {
|
|
2187
|
+
...opts,
|
|
2188
|
+
_trainingCapture: { layerIdx, stage: 'ffn_act', capture },
|
|
2189
|
+
});
|
|
2190
|
+
if (prefill.currentRecorder) {
|
|
2191
|
+
recordPrefillRecorderStats(this.#state, prefill.currentRecorder);
|
|
2192
|
+
await prefill.currentRecorder.submitAndWait();
|
|
2193
|
+
await prefill.recordProfile(prefill.currentRecorder);
|
|
2194
|
+
} else {
|
|
2195
|
+
await getDevice().queue.onSubmittedWorkDone();
|
|
2196
|
+
}
|
|
2197
|
+
if (!activationTensor) {
|
|
2198
|
+
releaseBuffer(prefill.currentHiddenBuffer);
|
|
2199
|
+
throw new Error('native_lora_training_activation_not_captured.');
|
|
2200
|
+
}
|
|
2201
|
+
hiddenTensor = createTensor(
|
|
2202
|
+
prefill.currentHiddenBuffer,
|
|
2203
|
+
prefill.activationDtype,
|
|
2204
|
+
[prefill.numTokens, config.hiddenSize],
|
|
2205
|
+
`L${layerIdx}.native_lora_base_hidden`
|
|
2206
|
+
);
|
|
2207
|
+
this.resetToSeqLen(0);
|
|
2208
|
+
return {
|
|
2209
|
+
inputIds: tokens,
|
|
2210
|
+
layerIdx,
|
|
2211
|
+
module: 'down_proj',
|
|
2212
|
+
activation: activationTensor,
|
|
2213
|
+
baseHidden: hiddenTensor,
|
|
2214
|
+
dispose: releaseOwned,
|
|
2215
|
+
};
|
|
2216
|
+
} catch (error) {
|
|
2217
|
+
releaseOwned();
|
|
2218
|
+
this.resetToSeqLen(0);
|
|
2219
|
+
throw error;
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2118
2223
|
async prefillWithEmbedding(prompt, options = {}) {
|
|
2119
2224
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2120
2225
|
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
@@ -3179,6 +3284,7 @@ export class PipelineGenerator {
|
|
|
3179
3284
|
debugCheckBuffer,
|
|
3180
3285
|
opts.executionPlan
|
|
3181
3286
|
);
|
|
3287
|
+
context.trainingCapture = opts?._trainingCapture ?? null;
|
|
3182
3288
|
context.currentTokenIds = inputIds;
|
|
3183
3289
|
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
3184
3290
|
context.skipKVCacheWrites = returnHidden
|
|
@@ -6,6 +6,7 @@ import type { LinearNormMode } from '../../../config/schema/index.js';
|
|
|
6
6
|
import type { ProbeConfigSchema } from '../../../config/schema/index.js';
|
|
7
7
|
import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
|
|
8
8
|
import type { KernelPathSchema } from '../../../config/schema/kernel-path.schema.js';
|
|
9
|
+
import type { LoRAAdapter } from './lora.js';
|
|
9
10
|
|
|
10
11
|
export interface LinearLayerRuntimeState {
|
|
11
12
|
layerIdx: number;
|
|
@@ -76,6 +77,7 @@ export interface RunLinearAttentionLayerOptions {
|
|
|
76
77
|
precomputedInputNorm?: Tensor | null;
|
|
77
78
|
debugProbes?: ProbeConfigSchema[] | null;
|
|
78
79
|
recorder?: CommandRecorder | null;
|
|
80
|
+
lora?: LoRAAdapter | null;
|
|
79
81
|
}
|
|
80
82
|
|
|
81
83
|
export declare function hasLinearAttentionLayers(layerTypes: unknown): boolean;
|
|
@@ -23,6 +23,8 @@ import { dequantizeQ4KM } from '../../../converter/quantizer.js';
|
|
|
23
23
|
import { getKernelPathMatmulPrecision } from '../../../config/kernel-path-loader.js';
|
|
24
24
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
25
25
|
import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
|
|
26
|
+
import { applyLoRA } from './lora-apply.js';
|
|
27
|
+
import { getLoRAModule } from './lora.js';
|
|
26
28
|
|
|
27
29
|
const LINEAR_RUNTIME_SCHEMA_VERSION = 1;
|
|
28
30
|
const QK_L2NORM_EPS = 1e-6;
|
|
@@ -172,6 +174,38 @@ function releaseOrTrackBuffer(recorder, buffer) {
|
|
|
172
174
|
}
|
|
173
175
|
}
|
|
174
176
|
|
|
177
|
+
async function applyProjectionLoRA({
|
|
178
|
+
inputTensor,
|
|
179
|
+
baseOutput,
|
|
180
|
+
loraModule,
|
|
181
|
+
dims,
|
|
182
|
+
getWeightBuffer,
|
|
183
|
+
recorder,
|
|
184
|
+
kernelPath,
|
|
185
|
+
}) {
|
|
186
|
+
if (!loraModule) {
|
|
187
|
+
return baseOutput;
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
const combined = await applyLoRA(
|
|
191
|
+
inputTensor,
|
|
192
|
+
baseOutput,
|
|
193
|
+
loraModule,
|
|
194
|
+
dims,
|
|
195
|
+
getWeightBuffer,
|
|
196
|
+
recorder ?? undefined,
|
|
197
|
+
{ kernelPath }
|
|
198
|
+
);
|
|
199
|
+
if (combined.buffer !== baseOutput.buffer) {
|
|
200
|
+
releaseOrTrackBuffer(recorder, baseOutput.buffer);
|
|
201
|
+
}
|
|
202
|
+
return combined;
|
|
203
|
+
} catch (error) {
|
|
204
|
+
releaseOrTrackBuffer(recorder, baseOutput.buffer);
|
|
205
|
+
throw error;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
175
209
|
function releaseResolvedWeightBuffer(originalWeight, resolvedWeight, recorder) {
|
|
176
210
|
if (isResolvedWeightShared(originalWeight)) {
|
|
177
211
|
return;
|
|
@@ -637,6 +671,7 @@ async function projectLinearTensor({
|
|
|
637
671
|
getWeightBuffer,
|
|
638
672
|
recorder,
|
|
639
673
|
executionPolicies = null,
|
|
674
|
+
loraModule = null,
|
|
640
675
|
}) {
|
|
641
676
|
const resolvedWeight = getWeightBuffer(sourceWeight, role);
|
|
642
677
|
const resolvedInputDtype = resolveMatmulStepDtype(
|
|
@@ -702,7 +737,16 @@ async function projectLinearTensor({
|
|
|
702
737
|
outputDtype: resolvedOutputDtype,
|
|
703
738
|
executionPolicies,
|
|
704
739
|
});
|
|
705
|
-
|
|
740
|
+
const projected = await settleProjectionOutputDtype(result);
|
|
741
|
+
return await applyProjectionLoRA({
|
|
742
|
+
inputTensor: matmulInput,
|
|
743
|
+
baseOutput: projected,
|
|
744
|
+
loraModule,
|
|
745
|
+
dims: { M: numTokens, N: outDim, K: hiddenSize },
|
|
746
|
+
getWeightBuffer,
|
|
747
|
+
recorder,
|
|
748
|
+
kernelPath,
|
|
749
|
+
});
|
|
706
750
|
}
|
|
707
751
|
const result = await runMatmul(matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
|
|
708
752
|
transposeB: 'auto',
|
|
@@ -712,7 +756,16 @@ async function projectLinearTensor({
|
|
|
712
756
|
outputDtype: resolvedOutputDtype,
|
|
713
757
|
executionPolicies,
|
|
714
758
|
});
|
|
715
|
-
|
|
759
|
+
const projected = await settleProjectionOutputDtype(result);
|
|
760
|
+
return await applyProjectionLoRA({
|
|
761
|
+
inputTensor: matmulInput,
|
|
762
|
+
baseOutput: projected,
|
|
763
|
+
loraModule,
|
|
764
|
+
dims: { M: numTokens, N: outDim, K: hiddenSize },
|
|
765
|
+
getWeightBuffer,
|
|
766
|
+
recorder: null,
|
|
767
|
+
kernelPath,
|
|
768
|
+
});
|
|
716
769
|
} finally {
|
|
717
770
|
if (matmulInput !== inputTensor) {
|
|
718
771
|
releaseOrTrackBuffer(recorder, matmulInput.buffer);
|
|
@@ -835,6 +888,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
835
888
|
recorder,
|
|
836
889
|
executionPolicies = null,
|
|
837
890
|
precomputedInputNorm = null,
|
|
891
|
+
lora = null,
|
|
838
892
|
} = options;
|
|
839
893
|
|
|
840
894
|
if (!layerWeights) {
|
|
@@ -909,7 +963,12 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
909
963
|
}
|
|
910
964
|
}
|
|
911
965
|
|
|
912
|
-
const
|
|
966
|
+
const qkvLoRA = getLoRAModule(lora, layerIdx, 'in_proj_qkv');
|
|
967
|
+
const zLoRA = getLoRAModule(lora, layerIdx, 'in_proj_z');
|
|
968
|
+
const aLoRA = getLoRAModule(lora, layerIdx, 'in_proj_a');
|
|
969
|
+
const bLoRA = getLoRAModule(lora, layerIdx, 'in_proj_b');
|
|
970
|
+
const outLoRA = getLoRAModule(lora, layerIdx, 'out_proj');
|
|
971
|
+
const qkvzProjection = qkvLoRA || zLoRA ? null : resolveLinearAttentionQKVZProjection(layerWeights, {
|
|
913
972
|
phase,
|
|
914
973
|
numTokens,
|
|
915
974
|
hiddenSize,
|
|
@@ -951,6 +1010,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
951
1010
|
getWeightBuffer,
|
|
952
1011
|
recorder,
|
|
953
1012
|
executionPolicies,
|
|
1013
|
+
loraModule: qkvLoRA,
|
|
954
1014
|
});
|
|
955
1015
|
const zTensor = qkvzTensor ?? await projectLinearTensor({
|
|
956
1016
|
inputTensor: normedTensor,
|
|
@@ -966,8 +1026,9 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
966
1026
|
getWeightBuffer,
|
|
967
1027
|
recorder,
|
|
968
1028
|
executionPolicies,
|
|
1029
|
+
loraModule: zLoRA,
|
|
969
1030
|
});
|
|
970
|
-
const abProjection = resolveLinearAttentionABProjection(layerWeights, {
|
|
1031
|
+
const abProjection = aLoRA || bLoRA ? null : resolveLinearAttentionABProjection(layerWeights, {
|
|
971
1032
|
phase,
|
|
972
1033
|
numTokens,
|
|
973
1034
|
hiddenSize,
|
|
@@ -1007,6 +1068,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
1007
1068
|
getWeightBuffer,
|
|
1008
1069
|
recorder,
|
|
1009
1070
|
executionPolicies,
|
|
1071
|
+
loraModule: aLoRA,
|
|
1010
1072
|
});
|
|
1011
1073
|
const bTensor = abTensor ?? await projectLinearTensor({
|
|
1012
1074
|
inputTensor: normedTensor,
|
|
@@ -1022,6 +1084,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
1022
1084
|
getWeightBuffer,
|
|
1023
1085
|
recorder,
|
|
1024
1086
|
executionPolicies,
|
|
1087
|
+
loraModule: bLoRA,
|
|
1025
1088
|
});
|
|
1026
1089
|
|
|
1027
1090
|
const outProjInputDtype = resolveMatmulStepDtype(
|
|
@@ -1160,9 +1223,17 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
1160
1223
|
? (recorder ? await recordCastF32ToF16(recorder, result) : await castF32ToF16(result))
|
|
1161
1224
|
: (recorder ? await recordCastF16ToF32(recorder, result) : await castF16ToF32(result));
|
|
1162
1225
|
releaseOrTrackBuffer(recorder, result.buffer);
|
|
1163
|
-
|
|
1226
|
+
result = casted;
|
|
1164
1227
|
}
|
|
1165
|
-
return
|
|
1228
|
+
return await applyProjectionLoRA({
|
|
1229
|
+
inputTensor: coreTensor,
|
|
1230
|
+
baseOutput: result,
|
|
1231
|
+
loraModule: outLoRA,
|
|
1232
|
+
dims: { M: numTokens, N: hiddenSize, K: projectionLayout.valueDim },
|
|
1233
|
+
getWeightBuffer,
|
|
1234
|
+
recorder,
|
|
1235
|
+
kernelPath,
|
|
1236
|
+
});
|
|
1166
1237
|
} finally {
|
|
1167
1238
|
releaseOrTrackBuffer(recorder, coreTensor.buffer);
|
|
1168
1239
|
releaseResolvedWeightBuffer(layerWeights.oProj, outProjWeight, recorder);
|
|
@@ -14,7 +14,12 @@ export type LoRAModuleName =
|
|
|
14
14
|
| 'gate_proj'
|
|
15
15
|
| 'up_proj'
|
|
16
16
|
| 'down_proj'
|
|
17
|
-
| 'gate_up_proj'
|
|
17
|
+
| 'gate_up_proj'
|
|
18
|
+
| 'in_proj_a'
|
|
19
|
+
| 'in_proj_b'
|
|
20
|
+
| 'in_proj_qkv'
|
|
21
|
+
| 'in_proj_z'
|
|
22
|
+
| 'out_proj';
|
|
18
23
|
|
|
19
24
|
export interface LoRAModuleWeights {
|
|
20
25
|
a: MaybeGPUBuffer;
|
|
@@ -34,6 +34,7 @@ import type { LinearAttentionRuntime } from './linear-attention.js';
|
|
|
34
34
|
import type { TokenizerLoadTiming } from '../../tokenizer.js';
|
|
35
35
|
import type { LoaderLoadTiming, PerLayerInputWeights } from '../../../loader/loader-types.js';
|
|
36
36
|
import type { UniformCacheStats } from '../../../gpu/uniform-cache.js';
|
|
37
|
+
import type { Tensor } from '../../../gpu/tensor.js';
|
|
37
38
|
|
|
38
39
|
// ============================================================================
|
|
39
40
|
// Core Context Types
|
|
@@ -85,6 +86,19 @@ export interface LayerContext {
|
|
|
85
86
|
debug: boolean;
|
|
86
87
|
/** Config-driven probes */
|
|
87
88
|
debugProbes?: ProbeConfigSchema[];
|
|
89
|
+
/** Internal native-LoRA activation capture owned by the training operator. */
|
|
90
|
+
trainingCapture?: {
|
|
91
|
+
layerIdx: number;
|
|
92
|
+
stage: 'ffn_act';
|
|
93
|
+
capture: (capture: {
|
|
94
|
+
layerIdx: number;
|
|
95
|
+
stage: 'ffn_act';
|
|
96
|
+
tensor: Tensor;
|
|
97
|
+
numTokens: number;
|
|
98
|
+
hiddenSize: number;
|
|
99
|
+
recorder: CommandRecorder | null;
|
|
100
|
+
}) => Promise<void>;
|
|
101
|
+
} | null;
|
|
88
102
|
/** Layers to debug (null = none, undefined/empty = layer 0 only for backward compat) */
|
|
89
103
|
debugLayers?: number[] | null;
|
|
90
104
|
/** Optional GPU buffer readback helper for debug checks */
|
|
@@ -398,6 +412,20 @@ export interface GenerateOptions {
|
|
|
398
412
|
__returnSequenceLogits?: boolean;
|
|
399
413
|
}
|
|
400
414
|
|
|
415
|
+
export interface NativeLoRAPrefillOptions extends GenerateOptions {
|
|
416
|
+
layerIdx: number;
|
|
417
|
+
module: 'down_proj';
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
export interface NativeLoRAPrefillResult {
|
|
421
|
+
inputIds: number[];
|
|
422
|
+
layerIdx: number;
|
|
423
|
+
module: 'down_proj';
|
|
424
|
+
activation: Tensor;
|
|
425
|
+
baseHidden: Tensor;
|
|
426
|
+
dispose(): void;
|
|
427
|
+
}
|
|
428
|
+
|
|
401
429
|
/**
|
|
402
430
|
* Result of a logits-only decode step.
|
|
403
431
|
*/
|
|
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
|
|
|
15
15
|
import { PipelineGenerator } from './text/generator.js';
|
|
16
16
|
import type { Manifest } from './text/config.js';
|
|
17
17
|
import type { WeightLoadResult, PipelineContexts } from './text/init.js';
|
|
18
|
-
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
|
|
18
|
+
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming, NativeLoRAPrefillOptions, NativeLoRAPrefillResult } from './text/types.js';
|
|
19
19
|
import type { ChatMessage } from './text/chat-format.js';
|
|
20
20
|
import type { LoRAAdapter } from './text/lora.js';
|
|
21
21
|
import type { DiffusionPipeline } from './diffusion/pipeline.js';
|
|
@@ -31,7 +31,7 @@ import type {
|
|
|
31
31
|
} from './text/generator.js';
|
|
32
32
|
|
|
33
33
|
// Re-export types for external use
|
|
34
|
-
export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
|
|
34
|
+
export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, NativeLoRAPrefillOptions, NativeLoRAPrefillResult };
|
|
35
35
|
export type { PipelineContexts };
|
|
36
36
|
|
|
37
37
|
export interface ChatRequestInput {
|
|
@@ -103,6 +103,11 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
103
103
|
|
|
104
104
|
prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
|
|
105
105
|
|
|
106
|
+
prefillForLoRATraining(
|
|
107
|
+
inputIds: readonly number[] | Int32Array | Uint32Array,
|
|
108
|
+
options: NativeLoRAPrefillOptions
|
|
109
|
+
): Promise<NativeLoRAPrefillResult>;
|
|
110
|
+
|
|
106
111
|
computeDiffusionGemmaCanvasLogits(
|
|
107
112
|
args: DiffusionGemmaCanvasLogitsInput,
|
|
108
113
|
options?: GenerateOptions & { __internalGenerate?: boolean }
|
|
@@ -1624,6 +1624,10 @@ export class InferencePipeline extends PipelineState {
|
|
|
1624
1624
|
return this.generator.prefillKVOnly(prompt, options);
|
|
1625
1625
|
}
|
|
1626
1626
|
|
|
1627
|
+
prefillForLoRATraining(inputIds, options) {
|
|
1628
|
+
return this.generator.prefillForLoRATraining(inputIds, options);
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1627
1631
|
computeDiffusionGemmaCanvasLogits(args, options = {}) {
|
|
1628
1632
|
assertNotAborted(options?.signal);
|
|
1629
1633
|
return this.generator.computeDiffusionGemmaCanvasLogits(args, options);
|
package/src/models/gemma3.js
CHANGED
package/src/models/gemma4.js
CHANGED
package/src/models/qwen3.js
CHANGED
|
@@ -2,7 +2,7 @@ import fs from 'node:fs/promises';
|
|
|
2
2
|
import { existsSync } from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
|
|
5
|
-
const DEFAULT_HF_REPO_ID = '
|
|
5
|
+
const DEFAULT_HF_REPO_ID = 'clocksmith/rdrr';
|
|
6
6
|
export const DEFAULT_HF_REGISTRY_PATH = 'registry/catalog.json';
|
|
7
7
|
export const DEFAULT_HF_REGISTRY_URL = `https://huggingface.co/${DEFAULT_HF_REPO_ID}/resolve/main/${DEFAULT_HF_REGISTRY_PATH}`;
|
|
8
8
|
|
|
@@ -32,6 +32,22 @@ export interface BrowserRelayLocalModelResolution {
|
|
|
32
32
|
}>;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
export interface StaticFileServerOptions {
|
|
36
|
+
rootDir?: string;
|
|
37
|
+
staticMounts?: NodeBrowserCommandRunOptions['staticMounts'];
|
|
38
|
+
host?: string;
|
|
39
|
+
port?: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface StaticFileServerHandle {
|
|
43
|
+
baseUrl: string;
|
|
44
|
+
close: () => Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export declare function createStaticFileServer(
|
|
48
|
+
options?: StaticFileServerOptions
|
|
49
|
+
): Promise<StaticFileServerHandle>;
|
|
50
|
+
|
|
35
51
|
export declare function resolveLocalFileModelUrlForBrowserRelay(
|
|
36
52
|
commandRequest: ToolingCommandRequestInput,
|
|
37
53
|
options?: Pick<NodeBrowserCommandRunOptions, 'baseUrl' | 'staticMounts'>
|
|
@@ -152,7 +152,7 @@ async function resolveFileForRequest(rootDir, mounts, requestPath) {
|
|
|
152
152
|
return { filePath: resolved, size: stats.size };
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
-
async function createStaticFileServer(options = {}) {
|
|
155
|
+
export async function createStaticFileServer(options = {}) {
|
|
156
156
|
const rootDir = path.resolve(
|
|
157
157
|
options.rootDir || fileURLToPath(new URL('../..', import.meta.url))
|
|
158
158
|
);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export { AutogradTape, OpType } from './experimental/training/autograd.js';
|
|
2
|
+
export { LoraAdapter } from './experimental/training/lora.js';
|
|
3
|
+
export { AdamOptimizer } from './experimental/training/optimizer.js';
|
|
4
|
+
export { trainStep } from './experimental/training/trainer.js';
|
|
5
|
+
export { crossEntropyLoss } from './experimental/training/loss.js';
|
|
6
|
+
export { clipGradients } from './experimental/training/clip.js';
|
|
7
|
+
export { exportLoRAAdapter, serializeLoRASafetensors } from './experimental/training/export.js';
|
|
8
|
+
export {
|
|
9
|
+
NativeQwenLoRATrainer,
|
|
10
|
+
createNativeQwenLoRATrainer,
|
|
11
|
+
loadNativeQwenTrainingPipeline,
|
|
12
|
+
trainNativeQwenSftLoRA,
|
|
13
|
+
} from './experimental/training/native-qwen-lora.js';
|
|
14
|
+
export type {
|
|
15
|
+
NativeQwenLoRATrainerOptions,
|
|
16
|
+
NativeQwenLoRACheckpoint,
|
|
17
|
+
NativeQwenLoRAStepResult,
|
|
18
|
+
NativeQwenSftLoRAOptions,
|
|
19
|
+
} from './experimental/training/native-qwen-lora.js';
|