doppler-gpu 0.4.10 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +40 -1
- package/models/catalog.json +637 -32
- package/package.json +24 -3
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +135 -13
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +69 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +73 -11
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/adapter.schema.d.ts +6 -1
- package/src/config/schema/adapter.schema.js +5 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/artifact-contract.d.ts +83 -0
- package/src/experimental/adapters/artifact-contract.js +254 -0
- package/src/experimental/adapters/index.d.ts +25 -0
- package/src/experimental/adapters/index.js +12 -0
- package/src/experimental/adapters/lora-loader.js +24 -4
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/experimental/training/artifacts.d.ts +160 -0
- package/src/experimental/training/attention-backward.d.ts +64 -0
- package/src/experimental/training/attention-forward.d.ts +22 -0
- package/src/experimental/training/attention-forward.js +82 -0
- package/src/experimental/training/autograd.d.ts +76 -0
- package/src/experimental/training/checkpoint.d.ts +36 -0
- package/src/experimental/training/clip.d.ts +9 -0
- package/src/experimental/training/dataloader.d.ts +8 -0
- package/src/experimental/training/distillation/student-fixture.js +1 -0
- package/src/experimental/training/export.d.ts +41 -0
- package/src/experimental/training/lora-pipeline.d.ts +186 -0
- package/src/experimental/training/lora-pipeline.js +5 -0
- package/src/experimental/training/lora.d.ts +20 -0
- package/src/experimental/training/lora.js +4 -3
- package/src/experimental/training/loss-scaling.d.ts +21 -0
- package/src/experimental/training/loss.d.ts +10 -0
- package/src/experimental/training/native-qwen-lora.d.ts +106 -0
- package/src/experimental/training/native-qwen-lora.js +449 -0
- package/src/experimental/training/objectives/base.d.ts +58 -0
- package/src/experimental/training/optimizer.d.ts +22 -0
- package/src/experimental/training/optimizer.js +1 -0
- package/src/experimental/training/runner.d.ts +249 -0
- package/src/experimental/training/trainer.d.ts +89 -0
- package/src/experimental/training/workloads.d.ts +205 -0
- package/src/gpu/kernels/backward/adam.d.ts +1 -0
- package/src/gpu/kernels/backward/adam.js +6 -2
- package/src/gpu/kernels/backward/adam.wgsl +3 -2
- package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
- package/src/gpu/kernels/backward/utils.js +18 -4
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +205 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.d.ts +5 -1
- package/src/inference/pipelines/text/generator.js +196 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
- package/src/inference/pipelines/text/linear-attention.js +77 -6
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/lora-types.d.ts +6 -1
- package/src/inference/pipelines/text/lora-types.js +5 -0
- package/src/inference/pipelines/text/ops.js +33 -0
- package/src/inference/pipelines/text/types.d.ts +66 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +9 -2
- package/src/inference/pipelines/text.js +116 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/models/diffusiongemma.js +1 -1
- package/src/models/embeddinggemma.js +1 -1
- package/src/models/gemma3.js +1 -1
- package/src/models/gemma4.js +1 -1
- package/src/models/qwen3.js +1 -1
- package/src/tooling/hf-registry-utils.js +1 -1
- package/src/tooling/node-browser-command-runner.d.ts +16 -0
- package/src/tooling/node-browser-command-runner.js +1 -1
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/training-browser.d.ts +19 -0
- package/src/training-browser.js +13 -0
- package/src/training.d.ts +137 -0
- package/src/training.js +219 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +9 -1
- package/tests/kernels/browser/test-page.js +4 -2
- package/tests/kernels/reference/adam.js +4 -2
- package/NOTICE +0 -5
|
@@ -25,6 +25,28 @@ export function rmsNormCPU(
|
|
|
25
25
|
return result;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
export function layerNormCPU(x, weight, bias, eps = 1e-5) {
|
|
29
|
+
if (weight.length !== x.length || bias.length !== x.length) {
|
|
30
|
+
throw new Error(
|
|
31
|
+
`LayerNorm parameter length mismatch: input=${x.length}, weight=${weight.length}, bias=${bias.length}.`
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
let sum = 0;
|
|
35
|
+
for (let i = 0; i < x.length; i++) sum += x[i];
|
|
36
|
+
const mean = sum / x.length;
|
|
37
|
+
let squaredDeviation = 0;
|
|
38
|
+
for (let i = 0; i < x.length; i++) {
|
|
39
|
+
const centered = x[i] - mean;
|
|
40
|
+
squaredDeviation += centered * centered;
|
|
41
|
+
}
|
|
42
|
+
const invStd = 1 / Math.sqrt(squaredDeviation / x.length + eps);
|
|
43
|
+
const result = new Float32Array(x.length);
|
|
44
|
+
for (let i = 0; i < x.length; i++) {
|
|
45
|
+
result[i] = (x[i] - mean) * invStd * weight[i] + bias[i];
|
|
46
|
+
}
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
|
|
28
50
|
|
|
29
51
|
export function f16BufferToF32(data) {
|
|
30
52
|
const u16 = new Uint16Array(data);
|
|
@@ -17,7 +17,7 @@ import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
|
17
17
|
export type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
18
18
|
|
|
19
19
|
// Re-export CPU functions
|
|
20
|
-
export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
20
|
+
export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
21
21
|
|
|
22
22
|
// Re-export GPU functions
|
|
23
23
|
export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
// Re-export CPU functions
|
|
4
|
-
export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
4
|
+
export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
5
5
|
|
|
6
6
|
// Re-export GPU functions
|
|
7
7
|
export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
|
|
|
12
12
|
// Imports for computeLogits orchestrator
|
|
13
13
|
import { getDevice } from '../../../../gpu/device.js';
|
|
14
14
|
import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
15
|
-
import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
|
|
15
|
+
import { runLayerNorm, runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
|
|
16
16
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
17
17
|
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
|
|
18
18
|
import { kernelTrace, traceStep } from '../kernel-trace.js';
|
|
@@ -72,6 +72,45 @@ function normalizeSelectedLogitTokenIds(value, vocabSize) {
|
|
|
72
72
|
return tokenIds;
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
+
function applyLmHeadBias(logits, bias, rowCount, rowSize, selectedTokenIds = null) {
|
|
76
|
+
if (bias == null) {
|
|
77
|
+
return logits;
|
|
78
|
+
}
|
|
79
|
+
if (!(bias instanceof Float32Array)) {
|
|
80
|
+
throw new Error('[Logits] LM head bias must be a Float32Array.');
|
|
81
|
+
}
|
|
82
|
+
if (selectedTokenIds !== null) {
|
|
83
|
+
if (logits.length !== selectedTokenIds.length) {
|
|
84
|
+
throw new Error('[Logits] Selected-logit bias application length mismatch.');
|
|
85
|
+
}
|
|
86
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
87
|
+
const tokenId = selectedTokenIds[index];
|
|
88
|
+
if (tokenId >= bias.length) {
|
|
89
|
+
throw new Error(`[Logits] LM head bias is missing token ${tokenId}.`);
|
|
90
|
+
}
|
|
91
|
+
logits[index] += bias[tokenId];
|
|
92
|
+
}
|
|
93
|
+
return logits;
|
|
94
|
+
}
|
|
95
|
+
if (bias.length < rowSize) {
|
|
96
|
+
throw new Error(
|
|
97
|
+
`[Logits] LM head bias length mismatch: expected at least ${rowSize}, got ${bias.length}.`
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
if (logits.length !== rowCount * rowSize) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
`[Logits] LM head bias matrix mismatch: expected ${rowCount * rowSize} logits, got ${logits.length}.`
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
for (let row = 0; row < rowCount; row += 1) {
|
|
106
|
+
const offset = row * rowSize;
|
|
107
|
+
for (let column = 0; column < rowSize; column += 1) {
|
|
108
|
+
logits[offset + column] += bias[column];
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return logits;
|
|
112
|
+
}
|
|
113
|
+
|
|
75
114
|
async function coerceTensorDtype(tensor, targetDtype, options = {}) {
|
|
76
115
|
if (!targetDtype || tensor.dtype === targetDtype) {
|
|
77
116
|
return tensor;
|
|
@@ -164,7 +203,7 @@ export async function computeLogits(
|
|
|
164
203
|
activationDtype: activationDtypeOverride,
|
|
165
204
|
} = config;
|
|
166
205
|
const activationDtype = activationDtypeOverride ?? getRuntimeConfig().inference.compute.activationDtype;
|
|
167
|
-
const { finalNorm, lmHead } = weights;
|
|
206
|
+
const { finalNorm, finalNormBias = null, lmHead, lmHeadBias = null } = weights;
|
|
168
207
|
const device = getDevice();
|
|
169
208
|
|
|
170
209
|
// Consistency check: warn if lmHead weight dtype disagrees with layer activation dtype.
|
|
@@ -269,6 +308,7 @@ export async function computeLogits(
|
|
|
269
308
|
cpuWeightLayout === 'column' ? cpuWeightVocabSize : null
|
|
270
309
|
)
|
|
271
310
|
: matmulCPU(normed, (lmHead), numTokens, matmulVocabSize, hiddenSize);
|
|
311
|
+
applyLmHeadBias(rawLogits, lmHeadBias, numTokens, matmulVocabSize);
|
|
272
312
|
return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
|
|
273
313
|
}
|
|
274
314
|
|
|
@@ -293,7 +333,7 @@ export async function computeLogits(
|
|
|
293
333
|
dtype: inputDtype,
|
|
294
334
|
});
|
|
295
335
|
|
|
296
|
-
// 2. Apply final
|
|
336
|
+
// 2. Apply the manifest-owned final normalization.
|
|
297
337
|
|
|
298
338
|
let normWeightBuffer;
|
|
299
339
|
let normWeightBufferOwned = false;
|
|
@@ -319,7 +359,10 @@ export async function computeLogits(
|
|
|
319
359
|
const kernelPath = config.kernelPath ?? null;
|
|
320
360
|
const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
|
|
321
361
|
const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
|
|
322
|
-
const forceStableF32Logits = !hasExplicitFinalNormPrecision &&
|
|
362
|
+
const forceStableF32Logits = !hasExplicitFinalNormPrecision && (
|
|
363
|
+
config.normalizationType === 'layernorm'
|
|
364
|
+
|| shouldForceStableF32Logits(config, inputDtype)
|
|
365
|
+
);
|
|
323
366
|
const stableKernelPath = forceStableF32Logits
|
|
324
367
|
? createStableF32LogitsKernelPath(kernelPath)
|
|
325
368
|
: kernelPath;
|
|
@@ -346,11 +389,39 @@ export async function computeLogits(
|
|
|
346
389
|
: inputTensor;
|
|
347
390
|
normInputBufferOwned = normInputTensor !== inputTensor;
|
|
348
391
|
}
|
|
349
|
-
let
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
392
|
+
let finalNormBiasBuffer = null;
|
|
393
|
+
let normedTensor;
|
|
394
|
+
if (config.normalizationType === 'layernorm') {
|
|
395
|
+
if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
|
|
396
|
+
throw new Error(
|
|
397
|
+
`[Logits] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
const effectiveBias = finalNormBias instanceof Float32Array
|
|
401
|
+
? finalNormBias
|
|
402
|
+
: new Float32Array(hiddenSize);
|
|
403
|
+
if (effectiveBias.length !== hiddenSize) {
|
|
404
|
+
throw new Error(
|
|
405
|
+
`[Logits] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveBias.length}.`
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
finalNormBiasBuffer = acquireBuffer(effectiveBias.byteLength, undefined, 'final_norm_bias');
|
|
409
|
+
device.queue.writeBuffer(finalNormBiasBuffer, 0, effectiveBias);
|
|
410
|
+
normedTensor = await runLayerNorm(
|
|
411
|
+
normInputTensor,
|
|
412
|
+
normWeightBuffer,
|
|
413
|
+
finalNormBiasBuffer,
|
|
414
|
+
rmsNormEps,
|
|
415
|
+
{ batchSize: numTokens, hiddenSize }
|
|
416
|
+
);
|
|
417
|
+
} else {
|
|
418
|
+
normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
|
|
419
|
+
batchSize: numTokens,
|
|
420
|
+
hiddenSize,
|
|
421
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
if (finalNormBiasBuffer) releaseBuffer(finalNormBiasBuffer);
|
|
354
425
|
if (normInputBufferOwned) {
|
|
355
426
|
releaseBuffer(normInputTensor.buffer);
|
|
356
427
|
}
|
|
@@ -386,7 +457,7 @@ export async function computeLogits(
|
|
|
386
457
|
|
|
387
458
|
// Trace final norm output
|
|
388
459
|
if (kernelTrace.enabled) {
|
|
389
|
-
await traceStep(
|
|
460
|
+
await traceStep(config.normalizationType, 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
|
|
390
461
|
}
|
|
391
462
|
|
|
392
463
|
// Debug: Check hidden state after final norm
|
|
@@ -461,6 +532,7 @@ export async function computeLogits(
|
|
|
461
532
|
if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
|
|
462
533
|
if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
|
|
463
534
|
|
|
535
|
+
applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
|
|
464
536
|
return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
|
|
465
537
|
}
|
|
466
538
|
|
|
@@ -533,6 +605,7 @@ export async function computeLogits(
|
|
|
533
605
|
if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
|
|
534
606
|
});
|
|
535
607
|
const selectedLogits = new Float32Array(selectedData);
|
|
608
|
+
applyLmHeadBias(selectedLogits, lmHeadBias, 1, selectedTokenIds.length, selectedTokenIds);
|
|
536
609
|
await runProbes('logits_final', selectedLogits, {
|
|
537
610
|
numTokens: 1,
|
|
538
611
|
hiddenSize: selectedTokenIds.length,
|
|
@@ -579,6 +652,7 @@ export async function computeLogits(
|
|
|
579
652
|
const rawLogits = logitsTensor.dtype === 'f16'
|
|
580
653
|
? f16BufferToF32(logitsData)
|
|
581
654
|
: new Float32Array(logitsData);
|
|
655
|
+
applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
|
|
582
656
|
if (isTraceEnabled('logits')) {
|
|
583
657
|
trace.logits('LM_HEAD_RAW_LOGITS_HEALTH', getLogitsHealth(rawLogits));
|
|
584
658
|
}
|
|
@@ -17,6 +17,8 @@ import type { WeightBuffer, CpuWeightBuffer, SplitWeightBuffer } from '../../../
|
|
|
17
17
|
export interface LogitsConfig {
|
|
18
18
|
hiddenSize: number;
|
|
19
19
|
vocabSize: number;
|
|
20
|
+
normalizationType: 'rmsnorm' | 'layernorm';
|
|
21
|
+
finalNormBiasTensor: string | null;
|
|
20
22
|
rmsNormEps: number;
|
|
21
23
|
useTiedEmbeddings: boolean;
|
|
22
24
|
embeddingVocabSize: number | null;
|
|
@@ -38,7 +40,9 @@ export interface LogitsConfig {
|
|
|
38
40
|
*/
|
|
39
41
|
export interface LogitsWeights {
|
|
40
42
|
finalNorm: GPUBuffer | Float32Array;
|
|
43
|
+
finalNormBias?: Float32Array | null;
|
|
41
44
|
lmHead: GPUBuffer | Float32Array | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer;
|
|
45
|
+
lmHeadBias?: Float32Array | null;
|
|
42
46
|
}
|
|
43
47
|
|
|
44
48
|
/**
|
|
@@ -14,7 +14,12 @@ export type LoRAModuleName =
|
|
|
14
14
|
| 'gate_proj'
|
|
15
15
|
| 'up_proj'
|
|
16
16
|
| 'down_proj'
|
|
17
|
-
| 'gate_up_proj'
|
|
17
|
+
| 'gate_up_proj'
|
|
18
|
+
| 'in_proj_a'
|
|
19
|
+
| 'in_proj_b'
|
|
20
|
+
| 'in_proj_qkv'
|
|
21
|
+
| 'in_proj_z'
|
|
22
|
+
| 'out_proj';
|
|
18
23
|
|
|
19
24
|
export interface LoRAModuleWeights {
|
|
20
25
|
a: MaybeGPUBuffer;
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
4
|
runRMSNorm, runResidualAdd, runMatmul, runSiLU, runGeLU,
|
|
5
|
+
runLayerNorm, runBiasAdd,
|
|
5
6
|
recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
|
|
7
|
+
recordLayerNorm, recordBiasAdd,
|
|
6
8
|
runSiLURowSplit, recordSiLURowSplit,
|
|
7
9
|
runMatmulRMSNormFused, recordMatmulRMSNormFused,
|
|
8
10
|
runRMSNormStats, recordRMSNormStats,
|
|
@@ -60,6 +62,36 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
|
|
|
60
62
|
return result;
|
|
61
63
|
}
|
|
62
64
|
|
|
65
|
+
export async function doLayerNorm(input, weight, bias, eps, options, recorder) {
|
|
66
|
+
const result = recorder
|
|
67
|
+
? await recordLayerNorm(recorder, input, weight, bias, eps, options)
|
|
68
|
+
: await runLayerNorm(input, weight, bias, eps, options);
|
|
69
|
+
|
|
70
|
+
if (kernelTrace.enabled && !recorder) {
|
|
71
|
+
const layer = options.layerIdx ?? -1;
|
|
72
|
+
const label = options.label ?? 'layernorm';
|
|
73
|
+
await traceStep('layernorm', label, layer, result.buffer, [options.batchSize, options.hiddenSize]);
|
|
74
|
+
}
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export async function doBiasAdd(data, bias, numTokens, dim, options, recorder) {
|
|
79
|
+
const result = recorder
|
|
80
|
+
? await recordBiasAdd(recorder, data, bias, numTokens, dim, options)
|
|
81
|
+
: await runBiasAdd(data, bias, numTokens, dim, options);
|
|
82
|
+
|
|
83
|
+
if (kernelTrace.enabled && !recorder) {
|
|
84
|
+
await traceStep(
|
|
85
|
+
'bias_add',
|
|
86
|
+
options?.label ?? 'bias_add',
|
|
87
|
+
options?.layerIdx ?? -1,
|
|
88
|
+
result.buffer,
|
|
89
|
+
[numTokens, dim]
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
return result;
|
|
93
|
+
}
|
|
94
|
+
|
|
63
95
|
export async function doRMSNormStats(input, residual, eps, options, recorder) {
|
|
64
96
|
return recorder
|
|
65
97
|
? await recordRMSNormStats(recorder, input, residual, eps, options)
|
|
@@ -472,6 +504,7 @@ export async function doAttention(
|
|
|
472
504
|
getWeightBuffer: getWeightBufferFn,
|
|
473
505
|
getNormWeightBuffer: getNormWeightBufferFn,
|
|
474
506
|
recorder: recorder ?? null,
|
|
507
|
+
lora: lora ?? null,
|
|
475
508
|
}),
|
|
476
509
|
residualFused: false,
|
|
477
510
|
};
|
|
@@ -34,6 +34,7 @@ import type { LinearAttentionRuntime } from './linear-attention.js';
|
|
|
34
34
|
import type { TokenizerLoadTiming } from '../../tokenizer.js';
|
|
35
35
|
import type { LoaderLoadTiming, PerLayerInputWeights } from '../../../loader/loader-types.js';
|
|
36
36
|
import type { UniformCacheStats } from '../../../gpu/uniform-cache.js';
|
|
37
|
+
import type { Tensor } from '../../../gpu/tensor.js';
|
|
37
38
|
|
|
38
39
|
// ============================================================================
|
|
39
40
|
// Core Context Types
|
|
@@ -85,6 +86,19 @@ export interface LayerContext {
|
|
|
85
86
|
debug: boolean;
|
|
86
87
|
/** Config-driven probes */
|
|
87
88
|
debugProbes?: ProbeConfigSchema[];
|
|
89
|
+
/** Internal native-LoRA activation capture owned by the training operator. */
|
|
90
|
+
trainingCapture?: {
|
|
91
|
+
layerIdx: number;
|
|
92
|
+
stage: 'ffn_act';
|
|
93
|
+
capture: (capture: {
|
|
94
|
+
layerIdx: number;
|
|
95
|
+
stage: 'ffn_act';
|
|
96
|
+
tensor: Tensor;
|
|
97
|
+
numTokens: number;
|
|
98
|
+
hiddenSize: number;
|
|
99
|
+
recorder: CommandRecorder | null;
|
|
100
|
+
}) => Promise<void>;
|
|
101
|
+
} | null;
|
|
88
102
|
/** Layers to debug (null = none, undefined/empty = layer 0 only for backward compat) */
|
|
89
103
|
debugLayers?: number[] | null;
|
|
90
104
|
/** Optional GPU buffer readback helper for debug checks */
|
|
@@ -390,6 +404,26 @@ export interface GenerateOptions {
|
|
|
390
404
|
offset: number;
|
|
391
405
|
length: number;
|
|
392
406
|
} | null;
|
|
407
|
+
|
|
408
|
+
/** @internal Return final-normalized embeddings for every input token. */
|
|
409
|
+
__returnTokenEmbeddings?: boolean;
|
|
410
|
+
|
|
411
|
+
/** @internal Return full per-token vocabulary logits. */
|
|
412
|
+
__returnSequenceLogits?: boolean;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
export interface NativeLoRAPrefillOptions extends GenerateOptions {
|
|
416
|
+
layerIdx: number;
|
|
417
|
+
module: 'down_proj';
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
export interface NativeLoRAPrefillResult {
|
|
421
|
+
inputIds: number[];
|
|
422
|
+
layerIdx: number;
|
|
423
|
+
module: 'down_proj';
|
|
424
|
+
activation: Tensor;
|
|
425
|
+
baseHidden: Tensor;
|
|
426
|
+
dispose(): void;
|
|
393
427
|
}
|
|
394
428
|
|
|
395
429
|
/**
|
|
@@ -438,6 +472,30 @@ export interface PrefillEmbeddingResult extends KVCacheSnapshot {
|
|
|
438
472
|
/** Pooling mode used to construct embedding */
|
|
439
473
|
embeddingMode: 'last' | 'mean';
|
|
440
474
|
|
|
475
|
+
/** Final-normalized token matrix when explicitly requested. */
|
|
476
|
+
tokenEmbeddings?: Float32Array | null;
|
|
477
|
+
|
|
478
|
+
/** Full token-by-vocabulary logits when explicitly requested. */
|
|
479
|
+
logits?: Float32Array | null;
|
|
480
|
+
|
|
481
|
+
phase?: WorkloadPhaseTiming | null;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
export interface SequenceEncodeOptions extends GenerateOptions {
|
|
485
|
+
includeTokenEmbeddings?: boolean;
|
|
486
|
+
includeLogits?: boolean;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
export interface SequenceEncodeResult {
|
|
490
|
+
alphabet: 'amino_acid' | 'nucleotide';
|
|
491
|
+
tokens: number[];
|
|
492
|
+
tokenMask: Uint8Array;
|
|
493
|
+
includedTokenCount: number;
|
|
494
|
+
tokenEmbeddings: Float32Array | null;
|
|
495
|
+
pooledEmbedding: Float32Array | null;
|
|
496
|
+
logits: Float32Array | null;
|
|
497
|
+
embeddingDim: number;
|
|
498
|
+
vocabSize: number;
|
|
441
499
|
phase?: WorkloadPhaseTiming | null;
|
|
442
500
|
}
|
|
443
501
|
|
|
@@ -495,16 +553,23 @@ export type LayerWeightBuffer = GPUBuffer | WeightBuffer | Float32Array | CpuWei
|
|
|
495
553
|
export interface LayerWeights {
|
|
496
554
|
// Attention
|
|
497
555
|
inputNorm: GPUBuffer | Float32Array;
|
|
556
|
+
inputNormBias?: GPUBuffer | Float32Array | null;
|
|
498
557
|
qProj: LayerWeightBuffer;
|
|
558
|
+
qProjBias?: GPUBuffer | Float32Array | null;
|
|
499
559
|
kProj: LayerWeightBuffer;
|
|
560
|
+
kProjBias?: GPUBuffer | Float32Array | null;
|
|
500
561
|
vProj?: LayerWeightBuffer;
|
|
562
|
+
vProjBias?: GPUBuffer | Float32Array | null;
|
|
501
563
|
oProj: LayerWeightBuffer;
|
|
564
|
+
oProjBias?: GPUBuffer | Float32Array | null;
|
|
502
565
|
qGateProj?: LayerWeightBuffer | null;
|
|
503
566
|
convInProj?: LayerWeightBuffer;
|
|
504
567
|
convKernel?: LayerWeightBuffer;
|
|
505
568
|
convOutProj?: LayerWeightBuffer;
|
|
506
569
|
/** Fused Q/K/V projection (runtime-generated for 3->1 matmul optimization) */
|
|
507
570
|
qkvProj?: GPUBuffer | WeightBuffer | null;
|
|
571
|
+
/** Concatenated Q/K/V projection bias matching qkvSizes. */
|
|
572
|
+
qkvProjBias?: GPUBuffer | Float32Array | null;
|
|
508
573
|
/** Sizes for splitting fused QKV output: [qSize, kSize, vSize] in elements */
|
|
509
574
|
qkvSizes?: [number, number, number];
|
|
510
575
|
/** Data type of fused QKV weights (f16, f32, or q4k) */
|
|
@@ -516,6 +581,7 @@ export interface LayerWeights {
|
|
|
516
581
|
|
|
517
582
|
// FFN (dense layers)
|
|
518
583
|
postAttentionNorm?: GPUBuffer | Float32Array;
|
|
584
|
+
postAttentionNormBias?: GPUBuffer | Float32Array | null;
|
|
519
585
|
postAttnNorm?: GPUBuffer | Float32Array; // LLaMA-style pre-FFN norm
|
|
520
586
|
gate?: LayerWeightBuffer;
|
|
521
587
|
up?: LayerWeightBuffer;
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import type { LayerWeights } from './types.js';
|
|
14
14
|
import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
15
|
+
import type { Tensor } from '../../../gpu/tensor.js';
|
|
15
16
|
|
|
16
17
|
// ============================================================================
|
|
17
18
|
// Types
|
|
@@ -60,3 +61,12 @@ export function getNormWeightBuffer(
|
|
|
60
61
|
debugFlags?: WeightDebugFlags,
|
|
61
62
|
deviceOverride?: GPUDevice | null
|
|
62
63
|
): GPUBuffer;
|
|
64
|
+
|
|
65
|
+
export function getVectorTensor(
|
|
66
|
+
weight: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | { buffer: ArrayBuffer; byteOffset: number; byteLength: number } | CpuWeightBuffer,
|
|
67
|
+
label: string,
|
|
68
|
+
length: number,
|
|
69
|
+
config: WeightBufferConfig,
|
|
70
|
+
debugFlags?: WeightDebugFlags,
|
|
71
|
+
deviceOverride?: GPUDevice | null
|
|
72
|
+
): { tensor: Tensor; owned: boolean };
|
|
@@ -3,7 +3,14 @@
|
|
|
3
3
|
import { getDevice } from '../../../gpu/device.js';
|
|
4
4
|
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
5
5
|
import { log } from '../../../debug/index.js';
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
getBufferDtype,
|
|
8
|
+
isWeightBuffer,
|
|
9
|
+
isCpuWeightBuffer,
|
|
10
|
+
isGpuBufferInstance,
|
|
11
|
+
tagBufferDtype,
|
|
12
|
+
} from '../../../gpu/weight-buffer.js';
|
|
13
|
+
import { createTensor } from '../../../gpu/tensor.js';
|
|
7
14
|
|
|
8
15
|
// ============================================================================
|
|
9
16
|
// Type Guards
|
|
@@ -116,6 +123,31 @@ export function getNormWeightBuffer(weight, label, config, debugFlags, deviceOve
|
|
|
116
123
|
}
|
|
117
124
|
}
|
|
118
125
|
|
|
126
|
+
export function getVectorTensor(weight, label, length, config, debugFlags, deviceOverride = null) {
|
|
127
|
+
if (!Number.isInteger(length) || length <= 0) {
|
|
128
|
+
throw new Error(`Vector tensor "${label}" requires a positive integer length.`);
|
|
129
|
+
}
|
|
130
|
+
const buffer = getNormWeightBuffer(weight, label, config, debugFlags, deviceOverride);
|
|
131
|
+
let dtype = getBufferDtype(buffer);
|
|
132
|
+
if (dtype == null) {
|
|
133
|
+
const bytesPerElement = buffer.size / length;
|
|
134
|
+
if (bytesPerElement === 2) dtype = 'f16';
|
|
135
|
+
else if (bytesPerElement === 4) dtype = 'f32';
|
|
136
|
+
else {
|
|
137
|
+
throw new Error(
|
|
138
|
+
`Vector tensor "${label}" cannot infer dtype from ${buffer.size} bytes for ${length} elements.`
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (dtype !== 'f16' && dtype !== 'f32') {
|
|
143
|
+
throw new Error(`Vector tensor "${label}" has unsupported dtype "${String(dtype)}".`);
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
tensor: createTensor(buffer, dtype, [length], label),
|
|
147
|
+
owned: !isGpuBufferInstance(weight) && !isWeightBuffer(weight),
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
119
151
|
|
|
120
152
|
function getGPUWeightBuffer(weight, label) {
|
|
121
153
|
// Handle WeightBuffer by extracting underlying GPUBuffer
|
|
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
|
|
|
15
15
|
import { PipelineGenerator } from './text/generator.js';
|
|
16
16
|
import type { Manifest } from './text/config.js';
|
|
17
17
|
import type { WeightLoadResult, PipelineContexts } from './text/init.js';
|
|
18
|
-
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
|
|
18
|
+
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming, NativeLoRAPrefillOptions, NativeLoRAPrefillResult } from './text/types.js';
|
|
19
19
|
import type { ChatMessage } from './text/chat-format.js';
|
|
20
20
|
import type { LoRAAdapter } from './text/lora.js';
|
|
21
21
|
import type { DiffusionPipeline } from './diffusion/pipeline.js';
|
|
@@ -31,7 +31,7 @@ import type {
|
|
|
31
31
|
} from './text/generator.js';
|
|
32
32
|
|
|
33
33
|
// Re-export types for external use
|
|
34
|
-
export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
|
|
34
|
+
export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, NativeLoRAPrefillOptions, NativeLoRAPrefillResult };
|
|
35
35
|
export type { PipelineContexts };
|
|
36
36
|
|
|
37
37
|
export interface ChatRequestInput {
|
|
@@ -103,6 +103,11 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
103
103
|
|
|
104
104
|
prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
|
|
105
105
|
|
|
106
|
+
prefillForLoRATraining(
|
|
107
|
+
inputIds: readonly number[] | Int32Array | Uint32Array,
|
|
108
|
+
options: NativeLoRAPrefillOptions
|
|
109
|
+
): Promise<NativeLoRAPrefillResult>;
|
|
110
|
+
|
|
106
111
|
computeDiffusionGemmaCanvasLogits(
|
|
107
112
|
args: DiffusionGemmaCanvasLogitsInput,
|
|
108
113
|
options?: GenerateOptions & { __internalGenerate?: boolean }
|
|
@@ -130,6 +135,8 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
130
135
|
embeddingMode: string;
|
|
131
136
|
}>>;
|
|
132
137
|
|
|
138
|
+
encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise<SequenceEncodeResult>;
|
|
139
|
+
|
|
133
140
|
prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
|
|
134
141
|
|
|
135
142
|
prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
|