doppler-gpu 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +62 -12
- package/models/catalog.json +21 -7
- package/package.json +25 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +8 -4
- package/src/config/kernels/registry.json +1035 -2677
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/workloads.js +11 -0
- package/src/gpu/device.js +100 -32
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/index.d.ts +5 -0
- package/src/gpu/kernels/index.js +5 -0
- package/src/gpu/kernels/matmul-selection.js +5 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-v1.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +2 -0
- package/src/inference/pipelines/text/generator.js +29 -0
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/tokenizers/bundled.js +105 -32
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling/runtime-optimization.d.ts +109 -0
- package/src/tooling/runtime-optimization.js +872 -0
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/tooling-exports.shared.d.ts +19 -0
- package/src/tooling-exports.shared.js +11 -0
- package/src/version.js +1 -1
|
@@ -35,6 +35,7 @@ import { LORA_MODULE_ALIASES } from '../../inference/pipelines/text/lora.js';
|
|
|
35
35
|
import { loadDistillModelHandle } from './suite.js';
|
|
36
36
|
import { createDistillStudentRuntimeModelFixture } from './distillation/student-fixture.js';
|
|
37
37
|
import { f16ToF32Array } from '../../inference/kv-cache/types.js';
|
|
38
|
+
import { sha256BytesHex, sha256Hex } from '../../utils/sha256.js';
|
|
38
39
|
|
|
39
40
|
const CAUSAL_LM_TEXT_PAIR_RUNNER_KEYS = Object.freeze([
|
|
40
41
|
'gemma-3-270m-it-f16-af32::text-pairs::text_generation',
|
|
@@ -44,6 +45,7 @@ const CAUSAL_LM_TEXT_PAIR_RUNNER_KEYS = Object.freeze([
|
|
|
44
45
|
'gemma-4-e2b-it-q4k-ehf16-af32-int4ple::text-pairs::text_generation',
|
|
45
46
|
'qwen-3-5-0-8b-q4k-ehaf16::text-pairs::text_generation',
|
|
46
47
|
'qwen-3-5-2b-q4k-ehaf16::text-pairs::text_generation',
|
|
48
|
+
'qwen-3-5-9b-hf-bf16::text-pairs::text_generation',
|
|
47
49
|
'qwen-3-6-27b-q4k-ehaf16::text-pairs::text_generation',
|
|
48
50
|
'qwen-3-6-27b-q4k-eaf16::text-pairs::text_generation',
|
|
49
51
|
]);
|
|
@@ -60,6 +62,7 @@ export const LORA_RUNNER_SUPPORT_CONTRACT = Object.freeze({
|
|
|
60
62
|
'gemma-4-e2b-it-q4k-ehf16-af32-int4ple',
|
|
61
63
|
'qwen-3-5-0-8b-q4k-ehaf16',
|
|
62
64
|
'qwen-3-5-2b-q4k-ehaf16',
|
|
65
|
+
'qwen-3-5-9b-hf-bf16',
|
|
63
66
|
'qwen-3-6-27b-q4k-ehaf16',
|
|
64
67
|
'qwen-3-6-27b-q4k-eaf16',
|
|
65
68
|
]),
|
|
@@ -127,6 +130,13 @@ export const LORA_RUNNER_BASE_MODEL_REGISTRY = Object.freeze({
|
|
|
127
130
|
runnerKind: 'causal_lm_text_generation',
|
|
128
131
|
requiresExternalTrainer: true,
|
|
129
132
|
}),
|
|
133
|
+
'qwen-3-5-9b-hf-bf16': Object.freeze({
|
|
134
|
+
baseModelId: 'qwen-3-5-9b-hf-bf16',
|
|
135
|
+
modelRef: 'Qwen/Qwen3.5-9B',
|
|
136
|
+
family: 'qwen3',
|
|
137
|
+
runnerKind: 'causal_lm_text_generation',
|
|
138
|
+
requiresExternalTrainer: true,
|
|
139
|
+
}),
|
|
130
140
|
'qwen-3-6-27b-q4k-ehaf16': Object.freeze({
|
|
131
141
|
baseModelId: 'qwen-3-6-27b-q4k-ehaf16',
|
|
132
142
|
modelRef: 'qwen-3-6-27b-q4k-ehaf16',
|
|
@@ -537,10 +547,19 @@ function createCausalLmTrainingObjective() {
|
|
|
537
547
|
},
|
|
538
548
|
async computeLoss({ batch, config, tape, forwardState }) {
|
|
539
549
|
const loss = await crossEntropyLoss(forwardState.logits, batch.targets, config, tape);
|
|
540
|
-
return {
|
|
550
|
+
return {
|
|
551
|
+
loss,
|
|
552
|
+
components: {
|
|
553
|
+
supervised_token_count: batch.supervisedTokenCount,
|
|
554
|
+
ignored_target_count: batch.ignoredTargetCount,
|
|
555
|
+
},
|
|
556
|
+
};
|
|
541
557
|
},
|
|
542
|
-
backwardTargets({ loss, lossScale }) {
|
|
543
|
-
|
|
558
|
+
backwardTargets({ batch, loss, lossScale }) {
|
|
559
|
+
if (!Number.isInteger(batch.supervisedTokenCount) || batch.supervisedTokenCount < 1) {
|
|
560
|
+
throw new Error('Causal-LM training batch requires supervisedTokenCount >= 1.');
|
|
561
|
+
}
|
|
562
|
+
return createLossGradient(loss, lossScale / batch.supervisedTokenCount);
|
|
544
563
|
},
|
|
545
564
|
};
|
|
546
565
|
}
|
|
@@ -566,6 +585,8 @@ function createCausalLmDatasetBatches(samples) {
|
|
|
566
585
|
targets: targetTensor,
|
|
567
586
|
prompt: sample.prompt,
|
|
568
587
|
completion: sample.completion,
|
|
588
|
+
supervisedTokenCount: sample.supervisedTokenCount,
|
|
589
|
+
ignoredTargetCount: sample.ignoredTargetCount,
|
|
569
590
|
};
|
|
570
591
|
} finally {
|
|
571
592
|
releaseTensor(inputTensor);
|
|
@@ -598,20 +619,41 @@ async function loadCausalLmTextPairSamples(workload, datasetPath, tokenizer) {
|
|
|
598
619
|
if (samples.length < 1) {
|
|
599
620
|
throw new Error(`Causal-LM LoRA dataset ${dataset.absolutePath} produced no tokenized samples.`);
|
|
600
621
|
}
|
|
622
|
+
const tokenizationContract = {
|
|
623
|
+
version: 2,
|
|
624
|
+
objective: 'completion_only_causal_lm',
|
|
625
|
+
truncation: 'preserve_completion_head_tail_prompt',
|
|
626
|
+
maxLength,
|
|
627
|
+
joinWith,
|
|
628
|
+
};
|
|
629
|
+
const supervisedTokenCount = samples.reduce(
|
|
630
|
+
(sum, sample) => sum + sample.supervisedTokenCount,
|
|
631
|
+
0
|
|
632
|
+
);
|
|
633
|
+
const ignoredTargetCount = samples.reduce(
|
|
634
|
+
(sum, sample) => sum + sample.ignoredTargetCount,
|
|
635
|
+
0
|
|
636
|
+
);
|
|
637
|
+
const truncatedPromptTokenCount = samples.reduce(
|
|
638
|
+
(sum, sample) => sum + sample.truncatedPromptTokenCount,
|
|
639
|
+
0
|
|
640
|
+
);
|
|
601
641
|
return {
|
|
602
642
|
...dataset,
|
|
603
643
|
samples,
|
|
604
644
|
datasetHash: hashArtifactPayload({
|
|
605
645
|
rows: dataset.rows,
|
|
606
|
-
tokenization:
|
|
607
|
-
maxLength,
|
|
608
|
-
joinWith,
|
|
609
|
-
},
|
|
646
|
+
tokenization: tokenizationContract,
|
|
610
647
|
}),
|
|
611
648
|
tokenization: {
|
|
612
|
-
|
|
613
|
-
joinWith,
|
|
649
|
+
...tokenizationContract,
|
|
614
650
|
sampleCount: samples.length,
|
|
651
|
+
supervisedTokenCount,
|
|
652
|
+
ignoredTargetCount,
|
|
653
|
+
truncatedPromptTokenCount,
|
|
654
|
+
truncatedSampleCount: samples.filter(
|
|
655
|
+
(sample) => sample.truncatedPromptTokenCount > 0
|
|
656
|
+
).length,
|
|
615
657
|
},
|
|
616
658
|
};
|
|
617
659
|
}
|
|
@@ -690,6 +732,90 @@ function collectProtectedBuffers(model) {
|
|
|
690
732
|
return protectedBuffers;
|
|
691
733
|
}
|
|
692
734
|
|
|
735
|
+
async function captureLoraParameterState(model) {
|
|
736
|
+
const entries = typeof model?.loraTensorEntries === 'function'
|
|
737
|
+
? model.loraTensorEntries()
|
|
738
|
+
: [];
|
|
739
|
+
if (entries.length === 0) {
|
|
740
|
+
throw new Error('Causal-LM LoRA parameter receipt requires adapter tensors.');
|
|
741
|
+
}
|
|
742
|
+
const tensors = [];
|
|
743
|
+
for (const entry of [...entries].sort((left, right) => left.name.localeCompare(right.name))) {
|
|
744
|
+
const tensor = entry.tensor;
|
|
745
|
+
const elementCount = tensor.shape.reduce((product, value) => product * value, 1);
|
|
746
|
+
const bytesPerElement = tensor.dtype === 'f16' ? 2 : 4;
|
|
747
|
+
const raw = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
|
|
748
|
+
const values = tensor.dtype === 'f16'
|
|
749
|
+
? f16ToF32Array(new Uint16Array(raw))
|
|
750
|
+
: new Float32Array(raw);
|
|
751
|
+
let nonzeroCount = 0;
|
|
752
|
+
let sumSquares = 0;
|
|
753
|
+
let maxAbs = 0;
|
|
754
|
+
for (const value of values) {
|
|
755
|
+
if (value !== 0) nonzeroCount += 1;
|
|
756
|
+
sumSquares += value * value;
|
|
757
|
+
maxAbs = Math.max(maxAbs, Math.abs(value));
|
|
758
|
+
}
|
|
759
|
+
tensors.push({
|
|
760
|
+
name: entry.name,
|
|
761
|
+
dtype: tensor.dtype,
|
|
762
|
+
shape: [...tensor.shape],
|
|
763
|
+
elementCount,
|
|
764
|
+
hash: sha256BytesHex(new Uint8Array(raw)),
|
|
765
|
+
nonzeroCount,
|
|
766
|
+
maxAbs,
|
|
767
|
+
l2Norm: Math.sqrt(sumSquares),
|
|
768
|
+
values,
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
return {
|
|
772
|
+
aggregateHash: sha256Hex(tensors.map((tensor) => `${tensor.name}\0${tensor.hash}\n`).join('')),
|
|
773
|
+
tensors,
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
function buildLoraParameterReceipt(initial, final) {
|
|
778
|
+
const initialByName = new Map(initial.tensors.map((tensor) => [tensor.name, tensor]));
|
|
779
|
+
let totalSquaredDelta = 0;
|
|
780
|
+
const tensors = final.tensors.map((finalTensor) => {
|
|
781
|
+
const initialTensor = initialByName.get(finalTensor.name);
|
|
782
|
+
if (!initialTensor || initialTensor.elementCount !== finalTensor.elementCount) {
|
|
783
|
+
throw new Error(`LoRA parameter receipt shape mismatch for ${finalTensor.name}.`);
|
|
784
|
+
}
|
|
785
|
+
let squaredDelta = 0;
|
|
786
|
+
let maxAbsDelta = 0;
|
|
787
|
+
for (let index = 0; index < finalTensor.values.length; index += 1) {
|
|
788
|
+
const delta = finalTensor.values[index] - initialTensor.values[index];
|
|
789
|
+
squaredDelta += delta * delta;
|
|
790
|
+
maxAbsDelta = Math.max(maxAbsDelta, Math.abs(delta));
|
|
791
|
+
}
|
|
792
|
+
totalSquaredDelta += squaredDelta;
|
|
793
|
+
return {
|
|
794
|
+
name: finalTensor.name,
|
|
795
|
+
dtype: finalTensor.dtype,
|
|
796
|
+
shape: finalTensor.shape,
|
|
797
|
+
elementCount: finalTensor.elementCount,
|
|
798
|
+
initialHash: initialTensor.hash,
|
|
799
|
+
finalHash: finalTensor.hash,
|
|
800
|
+
changed: initialTensor.hash !== finalTensor.hash,
|
|
801
|
+
initialNonzeroCount: initialTensor.nonzeroCount,
|
|
802
|
+
finalNonzeroCount: finalTensor.nonzeroCount,
|
|
803
|
+
l2Delta: Math.sqrt(squaredDelta),
|
|
804
|
+
maxAbsDelta,
|
|
805
|
+
};
|
|
806
|
+
});
|
|
807
|
+
return {
|
|
808
|
+
tensorCount: tensors.length,
|
|
809
|
+
changedTensorCount: tensors.filter((tensor) => tensor.changed).length,
|
|
810
|
+
nonzeroFinalTensorCount: tensors.filter((tensor) => tensor.finalNonzeroCount > 0).length,
|
|
811
|
+
initialAggregateHash: initial.aggregateHash,
|
|
812
|
+
finalAggregateHash: final.aggregateHash,
|
|
813
|
+
aggregateChanged: initial.aggregateHash !== final.aggregateHash,
|
|
814
|
+
l2Delta: Math.sqrt(totalSquaredDelta),
|
|
815
|
+
tensors,
|
|
816
|
+
};
|
|
817
|
+
}
|
|
818
|
+
|
|
693
819
|
function disposeTapeOutputs(tape, protectedBuffers = new Set()) {
|
|
694
820
|
if (!Array.isArray(tape?.records)) return;
|
|
695
821
|
const released = new Set();
|
|
@@ -946,17 +1072,42 @@ async function exportCausalLmLoraModel(loadedWorkload, layout, fixture, checkpoi
|
|
|
946
1072
|
};
|
|
947
1073
|
}
|
|
948
1074
|
|
|
949
|
-
async function
|
|
950
|
-
const
|
|
1075
|
+
async function readSupervisedLossStats(loss, targets, vocabSize) {
|
|
1076
|
+
const lossElementCount = loss.shape.reduce((product, value) => product * value, 1);
|
|
1077
|
+
const targetElementCount = targets.shape.reduce((product, value) => product * value, 1);
|
|
1078
|
+
const [rawLoss, rawTargets] = await Promise.all([
|
|
1079
|
+
readBuffer(loss.buffer, lossElementCount * (loss.dtype === 'f16' ? 2 : 4)),
|
|
1080
|
+
readBuffer(targets.buffer, targetElementCount * 4),
|
|
1081
|
+
]);
|
|
951
1082
|
const data = loss.dtype === 'f16'
|
|
952
|
-
? f16ToF32Array(new Uint16Array(
|
|
953
|
-
: new Float32Array(
|
|
954
|
-
|
|
1083
|
+
? f16ToF32Array(new Uint16Array(rawLoss))
|
|
1084
|
+
: new Float32Array(rawLoss);
|
|
1085
|
+
const targetIds = new Uint32Array(rawTargets);
|
|
1086
|
+
if (data.length !== targetIds.length) {
|
|
1087
|
+
throw new Error(
|
|
1088
|
+
`Causal-LM loss/target length mismatch: ${data.length} losses for ${targetIds.length} targets.`
|
|
1089
|
+
);
|
|
1090
|
+
}
|
|
955
1091
|
let sum = 0;
|
|
956
|
-
|
|
957
|
-
|
|
1092
|
+
let supervisedTokenCount = 0;
|
|
1093
|
+
for (let index = 0; index < data.length; index += 1) {
|
|
1094
|
+
if (targetIds[index] >= vocabSize) continue;
|
|
1095
|
+
const value = data[index];
|
|
1096
|
+
if (!Number.isFinite(value)) {
|
|
1097
|
+
throw new Error(`Causal-LM loss is non-finite at supervised target ${index}.`);
|
|
1098
|
+
}
|
|
1099
|
+
sum += value;
|
|
1100
|
+
supervisedTokenCount += 1;
|
|
958
1101
|
}
|
|
959
|
-
|
|
1102
|
+
if (supervisedTokenCount < 1) {
|
|
1103
|
+
throw new Error('Causal-LM evaluation batch contains no supervised targets.');
|
|
1104
|
+
}
|
|
1105
|
+
return {
|
|
1106
|
+
lossSum: sum,
|
|
1107
|
+
meanLoss: sum / supervisedTokenCount,
|
|
1108
|
+
supervisedTokenCount,
|
|
1109
|
+
ignoredTargetCount: data.length - supervisedTokenCount,
|
|
1110
|
+
};
|
|
960
1111
|
}
|
|
961
1112
|
|
|
962
1113
|
function finiteMetric(value) {
|
|
@@ -994,19 +1145,37 @@ function buildLossQualityClaim(evalDataset, meanLoss, baselineReport) {
|
|
|
994
1145
|
}
|
|
995
1146
|
|
|
996
1147
|
async function computeCausalLmTextPairLosses(workload, fixture, samples, protectedBuffers) {
|
|
997
|
-
const
|
|
1148
|
+
const sampleLosses = [];
|
|
1149
|
+
let lossSum = 0;
|
|
1150
|
+
let supervisedTokenCount = 0;
|
|
1151
|
+
let ignoredTargetCount = 0;
|
|
998
1152
|
for await (const resolvedBatch of createCausalLmDatasetBatches(samples).batches()) {
|
|
999
1153
|
const tape = new AutogradTape(loadBackwardRegistry());
|
|
1000
1154
|
let loss = null;
|
|
1001
1155
|
try {
|
|
1002
1156
|
const { logits } = await fixture.model.forwardCausalLm(resolvedBatch, tape);
|
|
1003
1157
|
loss = await crossEntropyLoss(logits, resolvedBatch.targets, { training: { precision: workload.training.precision } }, tape);
|
|
1004
|
-
|
|
1158
|
+
const stats = await readSupervisedLossStats(
|
|
1159
|
+
loss,
|
|
1160
|
+
resolvedBatch.targets,
|
|
1161
|
+
logits.shape.at(-1)
|
|
1162
|
+
);
|
|
1163
|
+
sampleLosses.push(stats.meanLoss);
|
|
1164
|
+
lossSum += stats.lossSum;
|
|
1165
|
+
supervisedTokenCount += stats.supervisedTokenCount;
|
|
1166
|
+
ignoredTargetCount += stats.ignoredTargetCount;
|
|
1005
1167
|
} finally {
|
|
1006
1168
|
disposeTapeOutputs(tape, protectedBuffers);
|
|
1007
1169
|
}
|
|
1008
1170
|
}
|
|
1009
|
-
return
|
|
1171
|
+
return {
|
|
1172
|
+
sampleLosses,
|
|
1173
|
+
sampleCount: sampleLosses.length,
|
|
1174
|
+
lossSum,
|
|
1175
|
+
supervisedTokenCount,
|
|
1176
|
+
ignoredTargetCount,
|
|
1177
|
+
meanLoss: supervisedTokenCount > 0 ? lossSum / supervisedTokenCount : null,
|
|
1178
|
+
};
|
|
1010
1179
|
}
|
|
1011
1180
|
|
|
1012
1181
|
async function evaluateCausalLmLoraModel(workload, fixture, dataset, layout = null, checkpointMeta = {}) {
|
|
@@ -1023,15 +1192,13 @@ async function evaluateCausalLmLoraModel(workload, fixture, dataset, layout = nu
|
|
|
1023
1192
|
const evalDatasetMaterialized = evalDataset.datasetPath === dataset.absolutePath
|
|
1024
1193
|
? dataset
|
|
1025
1194
|
: await loadCausalLmTextPairSamples(workload, evalDatasetPath, fixture.tokenizer);
|
|
1026
|
-
const
|
|
1195
|
+
const lossStats = await computeCausalLmTextPairLosses(
|
|
1027
1196
|
workload,
|
|
1028
1197
|
fixture,
|
|
1029
1198
|
evalDatasetMaterialized.samples,
|
|
1030
1199
|
protectedBuffers
|
|
1031
1200
|
);
|
|
1032
|
-
const meanLoss =
|
|
1033
|
-
? losses.reduce((sum, value) => sum + value, 0) / losses.length
|
|
1034
|
-
: null;
|
|
1201
|
+
const meanLoss = lossStats.meanLoss;
|
|
1035
1202
|
const baselineReports = checkpointMeta.baselineReportsByEvalDatasetId || {};
|
|
1036
1203
|
const baselineReport = baselineReports[evalDataset.id] || null;
|
|
1037
1204
|
const qualityClaim = buildLossQualityClaim(evalDataset, meanLoss, baselineReport);
|
|
@@ -1054,7 +1221,9 @@ async function evaluateCausalLmLoraModel(workload, fixture, dataset, layout = nu
|
|
|
1054
1221
|
metrics: {
|
|
1055
1222
|
loss: {
|
|
1056
1223
|
score: meanLoss,
|
|
1057
|
-
samples:
|
|
1224
|
+
samples: lossStats.sampleCount,
|
|
1225
|
+
supervisedTokens: lossStats.supervisedTokenCount,
|
|
1226
|
+
ignoredTargets: lossStats.ignoredTargetCount,
|
|
1058
1227
|
},
|
|
1059
1228
|
},
|
|
1060
1229
|
primaryMetric: 'loss',
|
|
@@ -1302,8 +1471,7 @@ function assertCausalLmTensorCoverage(tensors, adapter) {
|
|
|
1302
1471
|
}
|
|
1303
1472
|
}
|
|
1304
1473
|
for (const [layerIndex, modules] of layerModules.entries()) {
|
|
1305
|
-
for (const moduleName of
|
|
1306
|
-
const kinds = modules.get(moduleName);
|
|
1474
|
+
for (const [moduleName, kinds] of modules.entries()) {
|
|
1307
1475
|
if (!kinds?.has('a') || !kinds?.has('b')) {
|
|
1308
1476
|
throw new Error(
|
|
1309
1477
|
`Causal-LM trainer layer ${layerIndex} module ${moduleName} must include both lora_a and lora_b tensors.`
|
|
@@ -1618,6 +1786,7 @@ async function runInternalCausalLmLoraPipeline(options, layout, compatibility) {
|
|
|
1618
1786
|
const fixture = await createCausalLmLoraFixture(workload);
|
|
1619
1787
|
try {
|
|
1620
1788
|
const dataset = await loadCausalLmTextPairSamples(workload, datasetPath, fixture.tokenizer);
|
|
1789
|
+
const initialParameterState = await captureLoraParameterState(fixture.model);
|
|
1621
1790
|
const evalReports = [];
|
|
1622
1791
|
const checkpointArtifacts = [];
|
|
1623
1792
|
const exports = [];
|
|
@@ -1754,6 +1923,11 @@ async function runInternalCausalLmLoraPipeline(options, layout, compatibility) {
|
|
|
1754
1923
|
tokenizerHash: fixture.baseManifest?.tokenizerHash || null,
|
|
1755
1924
|
}
|
|
1756
1925
|
);
|
|
1926
|
+
const finalParameterState = await captureLoraParameterState(fixture.model);
|
|
1927
|
+
const parameterReceipt = buildLoraParameterReceipt(
|
|
1928
|
+
initialParameterState,
|
|
1929
|
+
finalParameterState
|
|
1930
|
+
);
|
|
1757
1931
|
const finalCheckpointId = runner.lastCheckpoint
|
|
1758
1932
|
? `checkpoint-${String(runner.lastCheckpoint.step).padStart(6, '0')}`
|
|
1759
1933
|
: null;
|
|
@@ -1783,6 +1957,7 @@ async function runInternalCausalLmLoraPipeline(options, layout, compatibility) {
|
|
|
1783
1957
|
evalReports,
|
|
1784
1958
|
exports,
|
|
1785
1959
|
metrics,
|
|
1960
|
+
parameterReceipt,
|
|
1786
1961
|
lastCheckpoint: runner.lastCheckpoint,
|
|
1787
1962
|
dataset: {
|
|
1788
1963
|
path: dataset.absolutePath,
|
|
@@ -68,15 +68,21 @@ export class LoraAdapter {
|
|
|
68
68
|
const [tokens] = input.shape;
|
|
69
69
|
const down = await tape.record(
|
|
70
70
|
OpType.MATMUL,
|
|
71
|
-
(a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0], {
|
|
71
|
+
(a, b) => runMatmul(a, b, tokens, this.rank, this.A.shape[0], {
|
|
72
|
+
transposeB: false,
|
|
73
|
+
outputDtype: input.dtype,
|
|
74
|
+
}),
|
|
72
75
|
[input, this.A],
|
|
73
|
-
{ M: tokens, N: this.rank, K: this.A.shape[0] }
|
|
76
|
+
{ M: tokens, N: this.rank, K: this.A.shape[0], transposeB: false }
|
|
74
77
|
);
|
|
75
78
|
const up = await tape.record(
|
|
76
79
|
OpType.MATMUL,
|
|
77
|
-
(a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank, {
|
|
80
|
+
(a, b) => runMatmul(a, b, tokens, this.B.shape[1], this.rank, {
|
|
81
|
+
transposeB: false,
|
|
82
|
+
outputDtype: down.dtype,
|
|
83
|
+
}),
|
|
78
84
|
[down, this.B],
|
|
79
|
-
{ M: tokens, N: this.B.shape[1], K: this.rank }
|
|
85
|
+
{ M: tokens, N: this.B.shape[1], K: this.rank, transposeB: false }
|
|
80
86
|
);
|
|
81
87
|
return tape.record(
|
|
82
88
|
OpType.SCALE,
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
acquireBuffer,
|
|
3
|
+
releaseBuffer,
|
|
4
|
+
uploadData,
|
|
5
|
+
BufferUsage,
|
|
6
|
+
} from '../../memory/buffer-pool.js';
|
|
2
7
|
import { createTensor, tensorBytes } from '../../gpu/tensor.js';
|
|
3
8
|
import { runAdam } from '../../gpu/kernels/backward/adam.js';
|
|
4
9
|
|
|
@@ -81,6 +86,9 @@ export class AdamOptimizer {
|
|
|
81
86
|
m: createTensor(mBuf, param.dtype, [...param.shape], 'adam_m'),
|
|
82
87
|
v: createTensor(vBuf, param.dtype, [...param.shape], 'adam_v'),
|
|
83
88
|
};
|
|
89
|
+
const zeroState = new Uint8Array(bytes);
|
|
90
|
+
uploadData(mBuf, zeroState);
|
|
91
|
+
uploadData(vBuf, zeroState);
|
|
84
92
|
} catch (error) {
|
|
85
93
|
if (mBuf) {
|
|
86
94
|
releaseBuffer(mBuf);
|
|
@@ -33,16 +33,27 @@ function toFloat32(buffer, dtype) {
|
|
|
33
33
|
return new Float32Array(buffer);
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
async function computeLossMean(loss) {
|
|
37
|
-
const
|
|
36
|
+
async function computeLossMean(loss, elementCount = null) {
|
|
37
|
+
const lossElementCount = loss.shape.reduce((product, value) => product * value, 1);
|
|
38
|
+
const bytesPerElement = loss.dtype === 'f16' ? 2 : 4;
|
|
39
|
+
const data = toFloat32(
|
|
40
|
+
await readBuffer(loss.buffer, lossElementCount * bytesPerElement),
|
|
41
|
+
loss.dtype
|
|
42
|
+
);
|
|
38
43
|
if (!data.length) {
|
|
39
44
|
return 0;
|
|
40
45
|
}
|
|
46
|
+
const divisor = elementCount == null ? data.length : elementCount;
|
|
47
|
+
if (!Number.isInteger(divisor) || divisor < 1 || divisor > data.length) {
|
|
48
|
+
throw new Error(
|
|
49
|
+
`TrainingRunner loss elementCount must be an integer from 1 to ${data.length}, got ${divisor}.`
|
|
50
|
+
);
|
|
51
|
+
}
|
|
41
52
|
let sum = 0;
|
|
42
53
|
for (let i = 0; i < data.length; i += 1) {
|
|
43
54
|
sum += data[i];
|
|
44
55
|
}
|
|
45
|
-
return sum /
|
|
56
|
+
return sum / divisor;
|
|
46
57
|
}
|
|
47
58
|
|
|
48
59
|
async function resolveBatches(dataset, batchSize, shuffle) {
|
|
@@ -1010,7 +1021,10 @@ export class TrainingRunner {
|
|
|
1010
1021
|
globalThis.performance.now() - runStartMs,
|
|
1011
1022
|
progressContext
|
|
1012
1023
|
);
|
|
1013
|
-
const meanLoss = await computeLossMean(
|
|
1024
|
+
const meanLoss = await computeLossMean(
|
|
1025
|
+
stepResult.loss,
|
|
1026
|
+
stepResult.objectiveMetrics?.supervised_token_count ?? null
|
|
1027
|
+
);
|
|
1014
1028
|
pushRolling(lossWindow, meanLoss, telemetry.windowSize);
|
|
1015
1029
|
pushRolling(stepTimeWindow, step_time_ms, telemetry.windowSize);
|
|
1016
1030
|
const objectiveName = stepResult.objectiveName || this.trainingObjective?.name || 'cross_entropy';
|
|
@@ -14,6 +14,7 @@ export const TRAINING_WORKLOAD_SCHEMA_VERSION = 1;
|
|
|
14
14
|
export const TRAINING_WORKLOAD_KINDS = Object.freeze(['lora', 'distill', 'ul']);
|
|
15
15
|
export const TRAINING_WORKLOAD_SURFACE_SUPPORT = Object.freeze(['node', 'browser', 'both']);
|
|
16
16
|
export const TRAINING_SELECTION_GOALS = Object.freeze(['max', 'min']);
|
|
17
|
+
export const TRAINING_ROW_ORDERS = Object.freeze(['dataset_order', 'seed_hash_sorted_v1']);
|
|
17
18
|
export const TRAINING_EVAL_KINDS = Object.freeze([
|
|
18
19
|
'translation',
|
|
19
20
|
'text_generation',
|
|
@@ -385,6 +386,10 @@ function normalizeLoraConfig(value, label) {
|
|
|
385
386
|
const exportConfig = asObject(lora.export, `${label}.export`, { optional: true });
|
|
386
387
|
const activation = asObject(lora.activation, `${label}.activation`, { optional: true });
|
|
387
388
|
const targetModules = asStringArray(adapter.targetModules, `${label}.adapter.targetModules`);
|
|
389
|
+
const rowOrder = asNonEmptyString(lora.rowOrder, `${label}.rowOrder`, { optional: true });
|
|
390
|
+
if (rowOrder && !TRAINING_ROW_ORDERS.includes(rowOrder)) {
|
|
391
|
+
throw new Error(`${label}.rowOrder must be one of: ${TRAINING_ROW_ORDERS.join(', ')}.`);
|
|
392
|
+
}
|
|
388
393
|
for (const moduleName of targetModules) {
|
|
389
394
|
if (!VALID_LORA_TARGET_MODULES.includes(moduleName)) {
|
|
390
395
|
throw new Error(`${label}.adapter.targetModules contains unsupported module "${moduleName}".`);
|
|
@@ -394,9 +399,15 @@ function normalizeLoraConfig(value, label) {
|
|
|
394
399
|
datasetFormat: asNonEmptyString(lora.datasetFormat, `${label}.datasetFormat`),
|
|
395
400
|
taskType: asNonEmptyString(lora.taskType, `${label}.taskType`),
|
|
396
401
|
baseModelRef: asNonEmptyString(lora.baseModelRef, `${label}.baseModelRef`, { optional: true }),
|
|
402
|
+
baseModelRevision: asNonEmptyString(
|
|
403
|
+
lora.baseModelRevision,
|
|
404
|
+
`${label}.baseModelRevision`,
|
|
405
|
+
{ optional: true }
|
|
406
|
+
),
|
|
397
407
|
maxLength: asPositiveInteger(lora.maxLength, `${label}.maxLength`, { optional: true }),
|
|
398
408
|
sequenceLength: asPositiveInteger(lora.sequenceLength, `${label}.sequenceLength`, { optional: true }),
|
|
399
409
|
joinWith: asStringValue(lora.joinWith, `${label}.joinWith`, { optional: true }),
|
|
410
|
+
...(rowOrder ? { rowOrder } : {}),
|
|
400
411
|
adapter: {
|
|
401
412
|
rank: asPositiveInteger(adapter.rank, `${label}.adapter.rank`),
|
|
402
413
|
alpha: asFiniteNumber(adapter.alpha, `${label}.adapter.alpha`),
|