doppler-gpu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -31
- package/assets/doppler-runtime-map.svg +1 -3
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +4 -1
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.js +6 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
- package/assets/doppler-readme-hero.svg +0 -84
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
|
|
4
4
|
import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
|
|
5
|
-
import { runMatmul, runRMSNorm } from '../../../../gpu/kernel-selector.js';
|
|
5
|
+
import { runMatmul, runRMSNorm, runScale, recordScale } from '../../../../gpu/kernel-selector.js';
|
|
6
6
|
import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
|
|
7
7
|
import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
|
|
8
8
|
import { recordLmHeadArgmaxF16 } from '../../../../gpu/kernels/lm-head-argmax.js';
|
|
@@ -21,25 +21,9 @@ import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
|
21
21
|
import { runProbes } from '../probes.js';
|
|
22
22
|
import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
|
|
23
23
|
import { f16BufferToF32 } from './cpu.js';
|
|
24
|
-
import { readBufferWithCleanup } from './utils.js';
|
|
24
|
+
import { readBufferWithCleanup, resolveLogitInputScale } from './utils.js';
|
|
25
25
|
import { f16ToF32 } from '../../../../loader/dtype-utils.js';
|
|
26
|
-
|
|
27
|
-
function shouldForceStableF32Logits(config, inputDtype) {
|
|
28
|
-
if (inputDtype !== 'f16') {
|
|
29
|
-
return false;
|
|
30
|
-
}
|
|
31
|
-
// Softcapped output heads are numerically sensitive in pure F16 on the
|
|
32
|
-
// final RMSNorm + LM-head path. Widen only the logits tail so the main
|
|
33
|
-
// layer stack and KV cache can stay on the faster F16 lane.
|
|
34
|
-
if (Number.isFinite(config.finalLogitSoftcapping) && config.finalLogitSoftcapping > 0) {
|
|
35
|
-
return true;
|
|
36
|
-
}
|
|
37
|
-
// Small Gemma-family checkpoints can also overflow in pure F16 logits path
|
|
38
|
-
// after RMSNorm offset even without output softcapping.
|
|
39
|
-
return config.rmsNormWeightOffset === true
|
|
40
|
-
&& Number.isFinite(config.hiddenSize)
|
|
41
|
-
&& config.hiddenSize <= 768;
|
|
42
|
-
}
|
|
26
|
+
import { shouldForceStableF32Logits, createStableF32LogitsKernelPath } from './precision-policy.js';
|
|
43
27
|
|
|
44
28
|
function resolvePrecisionFieldDtype(precision, fallback, field) {
|
|
45
29
|
const requested = precision?.[field] ?? fallback;
|
|
@@ -84,68 +68,6 @@ async function coerceTensorDtype(tensor, targetDtype, recorder = null, options =
|
|
|
84
68
|
throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
|
|
85
69
|
}
|
|
86
70
|
|
|
87
|
-
const STABLE_F32_LOGITS_KERNEL_MAP = new Map([
|
|
88
|
-
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
89
|
-
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
90
|
-
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
91
|
-
]);
|
|
92
|
-
|
|
93
|
-
function createStableF32LogitsKernelPath(kernelPath) {
|
|
94
|
-
if (!kernelPath?.postLayer) {
|
|
95
|
-
return kernelPath;
|
|
96
|
-
}
|
|
97
|
-
let changed = false;
|
|
98
|
-
const postLayer = kernelPath.postLayer.map((step) => {
|
|
99
|
-
if (step?.op === 'final_norm') {
|
|
100
|
-
const precision = {
|
|
101
|
-
...(step.precision ?? {}),
|
|
102
|
-
inputDtype: 'f32',
|
|
103
|
-
outputDtype: 'f32',
|
|
104
|
-
};
|
|
105
|
-
if (
|
|
106
|
-
step.precision?.inputDtype === precision.inputDtype
|
|
107
|
-
&& step.precision?.outputDtype === precision.outputDtype
|
|
108
|
-
) {
|
|
109
|
-
return step;
|
|
110
|
-
}
|
|
111
|
-
changed = true;
|
|
112
|
-
return {
|
|
113
|
-
...step,
|
|
114
|
-
precision,
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
|
|
118
|
-
return step;
|
|
119
|
-
}
|
|
120
|
-
const replacement = STABLE_F32_LOGITS_KERNEL_MAP.get(step.kernel) ?? step.kernel;
|
|
121
|
-
const precision = {
|
|
122
|
-
...(step.precision ?? {}),
|
|
123
|
-
inputDtype: 'f32',
|
|
124
|
-
outputDtype: 'f32',
|
|
125
|
-
};
|
|
126
|
-
if (
|
|
127
|
-
replacement === step.kernel
|
|
128
|
-
&& step.precision?.inputDtype === precision.inputDtype
|
|
129
|
-
&& step.precision?.outputDtype === precision.outputDtype
|
|
130
|
-
) {
|
|
131
|
-
return step;
|
|
132
|
-
}
|
|
133
|
-
changed = true;
|
|
134
|
-
return {
|
|
135
|
-
...step,
|
|
136
|
-
kernel: replacement,
|
|
137
|
-
precision,
|
|
138
|
-
};
|
|
139
|
-
});
|
|
140
|
-
if (!changed) {
|
|
141
|
-
return kernelPath;
|
|
142
|
-
}
|
|
143
|
-
return {
|
|
144
|
-
...kernelPath,
|
|
145
|
-
postLayer,
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
|
|
149
71
|
const bf16ScratchU32 = new Uint32Array(1);
|
|
150
72
|
const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
|
|
151
73
|
const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
|
|
@@ -175,6 +97,26 @@ function writeBufferInChunks(queue, buffer, bytes) {
|
|
|
175
97
|
}
|
|
176
98
|
}
|
|
177
99
|
|
|
100
|
+
function resolveFinalNormGpuBuffer(finalNorm, queue, label) {
|
|
101
|
+
if (isWeightBuffer(finalNorm)) {
|
|
102
|
+
return { buffer: finalNorm.buffer, owned: false };
|
|
103
|
+
}
|
|
104
|
+
if (isGpuBufferInstance(finalNorm)) {
|
|
105
|
+
return { buffer: finalNorm, owned: false };
|
|
106
|
+
}
|
|
107
|
+
if (!ArrayBuffer.isView(finalNorm)) {
|
|
108
|
+
throw new Error('[Logits] final_norm must be a GPU buffer, typed array, or WeightBuffer.');
|
|
109
|
+
}
|
|
110
|
+
const buffer = acquireBuffer(finalNorm.byteLength, undefined, label);
|
|
111
|
+
try {
|
|
112
|
+
queue.writeBuffer(buffer, 0, finalNorm);
|
|
113
|
+
return { buffer, owned: true };
|
|
114
|
+
} catch (error) {
|
|
115
|
+
releaseBuffer(buffer);
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
178
120
|
function normalizeRangeBytes(value, label) {
|
|
179
121
|
if (value instanceof Uint8Array) return value;
|
|
180
122
|
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
@@ -304,9 +246,12 @@ export async function extractLmHeadChunk(
|
|
|
304
246
|
vocabSize,
|
|
305
247
|
rowOffset,
|
|
306
248
|
rowCount,
|
|
307
|
-
sourceDtype
|
|
249
|
+
sourceDtype
|
|
308
250
|
) {
|
|
309
|
-
|
|
251
|
+
if (typeof sourceDtype !== 'string' || sourceDtype.trim() === '') {
|
|
252
|
+
throw new Error('[Logits] CPU LM head source dtype is required.');
|
|
253
|
+
}
|
|
254
|
+
const normalizedSourceDtype = sourceDtype.trim().toLowerCase();
|
|
310
255
|
if (normalizedSourceDtype !== 'f32' && normalizedSourceDtype !== 'f16' && normalizedSourceDtype !== 'bf16') {
|
|
311
256
|
throw new Error(`[Logits] Unsupported CPU LM head source dtype "${sourceDtype}".`);
|
|
312
257
|
}
|
|
@@ -800,6 +745,8 @@ export async function computeLogitsGPU(
|
|
|
800
745
|
let normInputOwned = false;
|
|
801
746
|
let normedTensor;
|
|
802
747
|
let finalNormTensor;
|
|
748
|
+
let logitInputTensor;
|
|
749
|
+
let logitInputOwned = false;
|
|
803
750
|
let lmHeadInputTensor;
|
|
804
751
|
let lmHeadInputOwned = false;
|
|
805
752
|
let lmHeadBuffer;
|
|
@@ -815,13 +762,9 @@ export async function computeLogitsGPU(
|
|
|
815
762
|
}
|
|
816
763
|
|
|
817
764
|
// Apply final RMSNorm
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
normWeightBuffer = acquireBuffer( (finalNorm).byteLength, undefined, 'final_norm_w');
|
|
822
|
-
device.queue.writeBuffer(normWeightBuffer, 0, (finalNorm));
|
|
823
|
-
normWeightBufferOwned = true;
|
|
824
|
-
}
|
|
765
|
+
const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, device.queue, 'final_norm_w');
|
|
766
|
+
normWeightBuffer = resolvedFinalNorm.buffer;
|
|
767
|
+
normWeightBufferOwned = resolvedFinalNorm.owned;
|
|
825
768
|
|
|
826
769
|
const inputDtype = isGpuBufferInstance(hiddenStates) ? activationDtype : 'f32';
|
|
827
770
|
const inputTensor = createTensor(inputBuffer, inputDtype, [numTokens, hiddenSize], 'logits_input');
|
|
@@ -899,21 +842,29 @@ export async function computeLogitsGPU(
|
|
|
899
842
|
releaseBuffer(normInputTensor.buffer);
|
|
900
843
|
normInputOwned = false;
|
|
901
844
|
}
|
|
845
|
+
const logitInputScale = resolveLogitInputScale(config);
|
|
846
|
+
logitInputTensor = finalNormTensor;
|
|
847
|
+
if (logitInputScale !== 1) {
|
|
848
|
+
logitInputTensor = await runScale(finalNormTensor, logitInputScale, {
|
|
849
|
+
count: numTokens * hiddenSize,
|
|
850
|
+
});
|
|
851
|
+
logitInputOwned = true;
|
|
852
|
+
}
|
|
902
853
|
const lmHeadRole = resolveLmHeadMatmulRole(phase);
|
|
903
854
|
const lmHeadInputDtype = forceStableF32Logits
|
|
904
|
-
?
|
|
905
|
-
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath,
|
|
855
|
+
? logitInputTensor.dtype
|
|
856
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'inputDtype');
|
|
906
857
|
const lmHeadOutputDtype = forceStableF32Logits
|
|
907
|
-
?
|
|
908
|
-
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath,
|
|
909
|
-
lmHeadInputTensor = lmHeadInputDtype !==
|
|
910
|
-
? await coerceTensorDtype(
|
|
858
|
+
? logitInputTensor.dtype
|
|
859
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'outputDtype');
|
|
860
|
+
lmHeadInputTensor = lmHeadInputDtype !== logitInputTensor.dtype
|
|
861
|
+
? await coerceTensorDtype(logitInputTensor, lmHeadInputDtype, null, {
|
|
911
862
|
executionPolicies: config.executionPolicies ?? null,
|
|
912
863
|
op: 'lm_head',
|
|
913
864
|
transitionDeclaredBy: 'step_precision',
|
|
914
865
|
})
|
|
915
|
-
:
|
|
916
|
-
lmHeadInputOwned = lmHeadInputTensor !==
|
|
866
|
+
: logitInputTensor;
|
|
867
|
+
lmHeadInputOwned = lmHeadInputTensor !== logitInputTensor;
|
|
917
868
|
|
|
918
869
|
// Project to vocab via LM head
|
|
919
870
|
if (isGpuBufferInstance(lmHead)) {
|
|
@@ -948,6 +899,7 @@ export async function computeLogitsGPU(
|
|
|
948
899
|
// Cleanup intermediate buffers (but keep logitsBuffer)
|
|
949
900
|
if (inputBufferOwned) { releaseBuffer(inputBuffer); inputBufferOwned = false; }
|
|
950
901
|
if (lmHeadInputOwned) { releaseBuffer(lmHeadInputTensor.buffer); lmHeadInputOwned = false; }
|
|
902
|
+
if (logitInputOwned) { releaseBuffer(logitInputTensor.buffer); logitInputOwned = false; }
|
|
951
903
|
if (finalNormTensor) {
|
|
952
904
|
releaseBuffer(finalNormTensor.buffer);
|
|
953
905
|
finalNormTensor = null;
|
|
@@ -961,6 +913,7 @@ export async function computeLogitsGPU(
|
|
|
961
913
|
if (inputBufferOwned && inputBuffer) releaseBuffer(inputBuffer);
|
|
962
914
|
if (normInputOwned && normInputTensor) releaseBuffer(normInputTensor.buffer);
|
|
963
915
|
if (lmHeadInputOwned && lmHeadInputTensor) releaseBuffer(lmHeadInputTensor.buffer);
|
|
916
|
+
if (logitInputOwned && logitInputTensor) releaseBuffer(logitInputTensor.buffer);
|
|
964
917
|
if (finalNormTensor) releaseBuffer(finalNormTensor.buffer);
|
|
965
918
|
if (normedTensor) releaseBuffer(normedTensor.buffer);
|
|
966
919
|
if (normWeightBufferOwned && normWeightBuffer) releaseBuffer(normWeightBuffer);
|
|
@@ -999,13 +952,9 @@ async function recordLogitsTailGPU(
|
|
|
999
952
|
|
|
1000
953
|
let normWeightBuffer;
|
|
1001
954
|
let normWeightOwned = false;
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
normWeightBuffer = acquireBuffer( (finalNorm).byteLength, undefined, 'final_norm_w');
|
|
1006
|
-
recorder.device.queue.writeBuffer(normWeightBuffer, 0, (finalNorm));
|
|
1007
|
-
normWeightOwned = true;
|
|
1008
|
-
}
|
|
955
|
+
const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, recorder.device.queue, 'final_norm_w');
|
|
956
|
+
normWeightBuffer = resolvedFinalNorm.buffer;
|
|
957
|
+
normWeightOwned = resolvedFinalNorm.owned;
|
|
1009
958
|
|
|
1010
959
|
|
|
1011
960
|
const inputDtype = activationDtype;
|
|
@@ -1080,20 +1029,29 @@ async function recordLogitsTailGPU(
|
|
|
1080
1029
|
operatorDiagnostics,
|
|
1081
1030
|
dtype: finalNormTensor.dtype,
|
|
1082
1031
|
});
|
|
1032
|
+
const logitInputScale = resolveLogitInputScale(config);
|
|
1033
|
+
let logitInputTensor = finalNormTensor;
|
|
1034
|
+
let logitInputOwned = false;
|
|
1035
|
+
if (logitInputScale !== 1) {
|
|
1036
|
+
logitInputTensor = await recordScale(recorder, finalNormTensor, logitInputScale, {
|
|
1037
|
+
count: numTokens * hiddenSize,
|
|
1038
|
+
});
|
|
1039
|
+
logitInputOwned = true;
|
|
1040
|
+
}
|
|
1083
1041
|
const lmHeadRole = resolveLmHeadMatmulRole(phase);
|
|
1084
1042
|
const lmHeadInputDtype = forceStableF32Logits
|
|
1085
|
-
?
|
|
1086
|
-
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath,
|
|
1043
|
+
? logitInputTensor.dtype
|
|
1044
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'inputDtype');
|
|
1087
1045
|
const lmHeadOutputDtype = forceStableF32Logits
|
|
1088
|
-
?
|
|
1089
|
-
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath,
|
|
1090
|
-
const lmHeadInputTensor = lmHeadInputDtype !==
|
|
1091
|
-
? await coerceTensorDtype(
|
|
1046
|
+
? logitInputTensor.dtype
|
|
1047
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'outputDtype');
|
|
1048
|
+
const lmHeadInputTensor = lmHeadInputDtype !== logitInputTensor.dtype
|
|
1049
|
+
? await coerceTensorDtype(logitInputTensor, lmHeadInputDtype, recorder, {
|
|
1092
1050
|
executionPolicies: config.executionPolicies ?? null,
|
|
1093
1051
|
op: 'lm_head',
|
|
1094
1052
|
transitionDeclaredBy: 'step_precision',
|
|
1095
1053
|
})
|
|
1096
|
-
:
|
|
1054
|
+
: logitInputTensor;
|
|
1097
1055
|
|
|
1098
1056
|
// Get LM head buffer
|
|
1099
1057
|
|
|
@@ -1119,6 +1077,8 @@ async function recordLogitsTailGPU(
|
|
|
1119
1077
|
lmHeadOutputDtype,
|
|
1120
1078
|
normedTensor,
|
|
1121
1079
|
finalNormTensor,
|
|
1080
|
+
logitInputTensor,
|
|
1081
|
+
logitInputOwned,
|
|
1122
1082
|
normInputTensor,
|
|
1123
1083
|
normInputOwned,
|
|
1124
1084
|
normWeightBuffer,
|
|
@@ -1142,7 +1102,10 @@ function trackRecordedLogitsTail(recorder, tail) {
|
|
|
1142
1102
|
trackTempBufferOnce(tail.normedTensor.buffer);
|
|
1143
1103
|
}
|
|
1144
1104
|
trackTempBufferOnce(tail.finalNormTensor.buffer);
|
|
1145
|
-
if (tail.
|
|
1105
|
+
if (tail.logitInputOwned) {
|
|
1106
|
+
trackTempBufferOnce(tail.logitInputTensor.buffer);
|
|
1107
|
+
}
|
|
1108
|
+
if (tail.lmHeadInputTensor !== tail.logitInputTensor) {
|
|
1146
1109
|
trackTempBufferOnce(tail.lmHeadInputTensor.buffer);
|
|
1147
1110
|
}
|
|
1148
1111
|
if (tail.normWeightOwned) {
|
|
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
|
|
|
12
12
|
// Imports for computeLogits orchestrator
|
|
13
13
|
import { getDevice } from '../../../../gpu/device.js';
|
|
14
14
|
import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
15
|
-
import { runMatmul, runRMSNorm, castF16ToF32, castF32ToF16 } from '../../../../gpu/kernel-selector.js';
|
|
15
|
+
import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16 } from '../../../../gpu/kernel-selector.js';
|
|
16
16
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
17
17
|
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
|
|
18
18
|
import { kernelTrace, traceStep } from '../kernel-trace.js';
|
|
@@ -20,29 +20,13 @@ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
|
|
|
20
20
|
import { runProbes } from '../probes.js';
|
|
21
21
|
import { rmsNormCPU, matmulCPU, f16BufferToF32 } from './cpu.js';
|
|
22
22
|
import { resolveCpuWeightDims, computeChunkedLogitsGPU, computeSplitLogitsGPU } from './gpu.js';
|
|
23
|
-
import { finalizeLogits, readBufferWithCleanup } from './utils.js';
|
|
23
|
+
import { finalizeLogits, readBufferWithCleanup, resolveLogitInputScale } from './utils.js';
|
|
24
24
|
import { getLogitsHealth } from '../debug-utils/index.js';
|
|
25
25
|
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
26
26
|
import { getKernelPathMatmulPrecision, getKernelPathStepPrecision } from '../../../../config/kernel-path-loader.js';
|
|
27
27
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
28
28
|
import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
|
|
29
|
-
|
|
30
|
-
function shouldForceStableF32Logits(config, inputDtype) {
|
|
31
|
-
if (inputDtype !== 'f16') {
|
|
32
|
-
return false;
|
|
33
|
-
}
|
|
34
|
-
// Softcapped output heads are numerically sensitive in pure F16 on the
|
|
35
|
-
// final RMSNorm + LM-head path. Widen only the logits tail so the main
|
|
36
|
-
// layer stack and KV cache can stay on the faster F16 lane.
|
|
37
|
-
if (Number.isFinite(config.finalLogitSoftcapping) && config.finalLogitSoftcapping > 0) {
|
|
38
|
-
return true;
|
|
39
|
-
}
|
|
40
|
-
// Small Gemma-family checkpoints can also overflow in pure F16 logits path
|
|
41
|
-
// after RMSNorm offset even without output softcapping.
|
|
42
|
-
return config.rmsNormWeightOffset === true
|
|
43
|
-
&& Number.isFinite(config.hiddenSize)
|
|
44
|
-
&& config.hiddenSize <= 768;
|
|
45
|
-
}
|
|
29
|
+
import { shouldForceStableF32Logits, createStableF32LogitsKernelPath } from './precision-policy.js';
|
|
46
30
|
|
|
47
31
|
function resolvePrecisionFieldDtype(precision, fallback, field) {
|
|
48
32
|
const requested = precision?.[field] ?? fallback;
|
|
@@ -87,68 +71,6 @@ async function coerceTensorDtype(tensor, targetDtype, options = {}) {
|
|
|
87
71
|
throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
|
|
88
72
|
}
|
|
89
73
|
|
|
90
|
-
const STABLE_F32_LOGITS_KERNEL_MAP = new Map([
|
|
91
|
-
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
92
|
-
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
93
|
-
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
94
|
-
]);
|
|
95
|
-
|
|
96
|
-
function createStableF32LogitsKernelPath(kernelPath) {
|
|
97
|
-
if (!kernelPath?.postLayer) {
|
|
98
|
-
return kernelPath;
|
|
99
|
-
}
|
|
100
|
-
let changed = false;
|
|
101
|
-
const postLayer = kernelPath.postLayer.map((step) => {
|
|
102
|
-
if (step?.op === 'final_norm') {
|
|
103
|
-
const precision = {
|
|
104
|
-
...(step.precision ?? {}),
|
|
105
|
-
inputDtype: 'f32',
|
|
106
|
-
outputDtype: 'f32',
|
|
107
|
-
};
|
|
108
|
-
if (
|
|
109
|
-
step.precision?.inputDtype === precision.inputDtype
|
|
110
|
-
&& step.precision?.outputDtype === precision.outputDtype
|
|
111
|
-
) {
|
|
112
|
-
return step;
|
|
113
|
-
}
|
|
114
|
-
changed = true;
|
|
115
|
-
return {
|
|
116
|
-
...step,
|
|
117
|
-
precision,
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
|
|
121
|
-
return step;
|
|
122
|
-
}
|
|
123
|
-
const replacement = STABLE_F32_LOGITS_KERNEL_MAP.get(step.kernel) ?? step.kernel;
|
|
124
|
-
const precision = {
|
|
125
|
-
...(step.precision ?? {}),
|
|
126
|
-
inputDtype: 'f32',
|
|
127
|
-
outputDtype: 'f32',
|
|
128
|
-
};
|
|
129
|
-
if (
|
|
130
|
-
replacement === step.kernel
|
|
131
|
-
&& step.precision?.inputDtype === precision.inputDtype
|
|
132
|
-
&& step.precision?.outputDtype === precision.outputDtype
|
|
133
|
-
) {
|
|
134
|
-
return step;
|
|
135
|
-
}
|
|
136
|
-
changed = true;
|
|
137
|
-
return {
|
|
138
|
-
...step,
|
|
139
|
-
kernel: replacement,
|
|
140
|
-
precision,
|
|
141
|
-
};
|
|
142
|
-
});
|
|
143
|
-
if (!changed) {
|
|
144
|
-
return kernelPath;
|
|
145
|
-
}
|
|
146
|
-
return {
|
|
147
|
-
...kernelPath,
|
|
148
|
-
postLayer,
|
|
149
|
-
};
|
|
150
|
-
}
|
|
151
|
-
|
|
152
74
|
async function traceTensorHealth(label, tensor, elementCount) {
|
|
153
75
|
if (!isTraceEnabled('logits')) {
|
|
154
76
|
return;
|
|
@@ -173,6 +95,26 @@ export function resolveLmHeadMatmulConfig(numTokens, options = null) {
|
|
|
173
95
|
};
|
|
174
96
|
}
|
|
175
97
|
|
|
98
|
+
function resolveFinalNormGpuBuffer(finalNorm, device, label) {
|
|
99
|
+
if (isWeightBuffer(finalNorm)) {
|
|
100
|
+
return { buffer: finalNorm.buffer, owned: false };
|
|
101
|
+
}
|
|
102
|
+
if (isGpuBufferInstance(finalNorm)) {
|
|
103
|
+
return { buffer: finalNorm, owned: false };
|
|
104
|
+
}
|
|
105
|
+
if (!ArrayBuffer.isView(finalNorm)) {
|
|
106
|
+
throw new Error('[Logits] final_norm must be a GPU buffer, typed array, or WeightBuffer.');
|
|
107
|
+
}
|
|
108
|
+
const buffer = acquireBuffer(finalNorm.byteLength, undefined, label);
|
|
109
|
+
try {
|
|
110
|
+
device.queue.writeBuffer(buffer, 0, finalNorm);
|
|
111
|
+
return { buffer, owned: true };
|
|
112
|
+
} catch (error) {
|
|
113
|
+
releaseBuffer(buffer);
|
|
114
|
+
throw error;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
176
118
|
|
|
177
119
|
export async function computeLogits(
|
|
178
120
|
hiddenStates,
|
|
@@ -281,6 +223,12 @@ export async function computeLogits(
|
|
|
281
223
|
rmsNormEps,
|
|
282
224
|
config.rmsNormWeightOffset
|
|
283
225
|
);
|
|
226
|
+
const logitInputScale = resolveLogitInputScale(config);
|
|
227
|
+
if (logitInputScale !== 1) {
|
|
228
|
+
for (let i = 0; i < normed.length; i += 1) {
|
|
229
|
+
normed[i] *= logitInputScale;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
284
232
|
const rawLogits = isCpuWeightBuffer(lmHead)
|
|
285
233
|
? matmulCPU(
|
|
286
234
|
normed,
|
|
@@ -319,13 +267,13 @@ export async function computeLogits(
|
|
|
319
267
|
// 2. Apply final RMSNorm
|
|
320
268
|
|
|
321
269
|
let normWeightBuffer;
|
|
270
|
+
let normWeightBufferOwned = false;
|
|
322
271
|
if (getNormWeightBuffer) {
|
|
323
272
|
normWeightBuffer = getNormWeightBuffer(finalNorm, 'final_norm_w');
|
|
324
|
-
} else if (isGpuBufferInstance(finalNorm)) {
|
|
325
|
-
normWeightBuffer = finalNorm;
|
|
326
273
|
} else {
|
|
327
|
-
|
|
328
|
-
|
|
274
|
+
const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, device, 'final_norm_w');
|
|
275
|
+
normWeightBuffer = resolvedFinalNorm.buffer;
|
|
276
|
+
normWeightBufferOwned = resolvedFinalNorm.owned;
|
|
329
277
|
}
|
|
330
278
|
|
|
331
279
|
// Debug: Check hidden state before final norm
|
|
@@ -418,6 +366,15 @@ export async function computeLogits(
|
|
|
418
366
|
await debugCheckBuffer(finalNormTensor.buffer, 'After final norm', numTokens, hiddenSize);
|
|
419
367
|
}
|
|
420
368
|
|
|
369
|
+
const logitInputScale = resolveLogitInputScale(config);
|
|
370
|
+
if (logitInputScale !== 1) {
|
|
371
|
+
const unscaledFinalNormTensor = finalNormTensor;
|
|
372
|
+
finalNormTensor = await runScale(finalNormTensor, logitInputScale, {
|
|
373
|
+
count: numTokens * hiddenSize,
|
|
374
|
+
});
|
|
375
|
+
releaseBuffer(unscaledFinalNormTensor.buffer);
|
|
376
|
+
}
|
|
377
|
+
|
|
421
378
|
const lastTokenMatmul = resolveLmHeadMatmulConfig(numTokens, options);
|
|
422
379
|
const { lastPositionOnly, matmulRows } = lastTokenMatmul;
|
|
423
380
|
const matmulPhaseOverride = lastTokenMatmul.phaseOverride;
|
|
@@ -473,7 +430,7 @@ export async function computeLogits(
|
|
|
473
430
|
if (inputBufferOwned) releaseBuffer(inputBuffer);
|
|
474
431
|
releaseBuffer(finalNormTensor.buffer);
|
|
475
432
|
if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
|
|
476
|
-
if (
|
|
433
|
+
if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
|
|
477
434
|
|
|
478
435
|
return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
|
|
479
436
|
}
|
|
@@ -554,7 +511,7 @@ export async function computeLogits(
|
|
|
554
511
|
releaseBuffer(finalNormTensor.buffer);
|
|
555
512
|
if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
|
|
556
513
|
releaseBuffer(logitsTensor.buffer);
|
|
557
|
-
if (
|
|
514
|
+
if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
|
|
558
515
|
if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
|
|
559
516
|
});
|
|
560
517
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { KernelPathSchema } from '../../../../config/schema/index.js';
|
|
2
|
+
import type { LogitsConfig } from './types.js';
|
|
3
|
+
|
|
4
|
+
export function shouldForceStableF32Logits(
|
|
5
|
+
config: Pick<LogitsConfig, 'finalLogitSoftcapping' | 'rmsNormWeightOffset' | 'hiddenSize'>,
|
|
6
|
+
inputDtype: 'f16' | 'f32'
|
|
7
|
+
): boolean;
|
|
8
|
+
|
|
9
|
+
export function createStableF32LogitsKernelPath(
|
|
10
|
+
kernelPath: KernelPathSchema | null | undefined
|
|
11
|
+
): KernelPathSchema | null | undefined;
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getKernelFilePrecisionPatch,
|
|
3
|
+
resolveF16ToF32ActivationKernel,
|
|
4
|
+
} from '../../../../config/transforms/execution-graph-transforms.js';
|
|
5
|
+
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
6
|
+
|
|
7
|
+
function requireLogitsConfigValue(condition, label, value) {
|
|
8
|
+
if (!condition) {
|
|
9
|
+
throw new Error(`[Logits] stable F32 policy requires ${label}, got ${String(value)}.`);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function shouldForceStableF32Logits(config, inputDtype) {
|
|
14
|
+
requireLogitsConfigValue(inputDtype === 'f16' || inputDtype === 'f32', 'input dtype "f16" or "f32"', inputDtype);
|
|
15
|
+
requireLogitsConfigValue(config && typeof config === 'object', 'a logits config object', config);
|
|
16
|
+
requireLogitsConfigValue(
|
|
17
|
+
config.finalLogitSoftcapping === null || Number.isFinite(config.finalLogitSoftcapping),
|
|
18
|
+
'config.finalLogitSoftcapping to be null or finite',
|
|
19
|
+
config.finalLogitSoftcapping
|
|
20
|
+
);
|
|
21
|
+
requireLogitsConfigValue(
|
|
22
|
+
typeof config.rmsNormWeightOffset === 'boolean',
|
|
23
|
+
'config.rmsNormWeightOffset to be boolean',
|
|
24
|
+
config.rmsNormWeightOffset
|
|
25
|
+
);
|
|
26
|
+
requireLogitsConfigValue(
|
|
27
|
+
Number.isFinite(config.hiddenSize) && config.hiddenSize > 0,
|
|
28
|
+
'config.hiddenSize to be a positive finite number',
|
|
29
|
+
config.hiddenSize
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
return selectRuleValue('inference', 'config', 'stableF32Logits', {
|
|
33
|
+
inputDtype,
|
|
34
|
+
finalLogitSoftcapping: config.finalLogitSoftcapping,
|
|
35
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
36
|
+
hiddenSize: config.hiddenSize,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function createStableF32LogitsKernelPath(kernelPath) {
|
|
41
|
+
if (!kernelPath?.postLayer) {
|
|
42
|
+
return kernelPath;
|
|
43
|
+
}
|
|
44
|
+
let changed = false;
|
|
45
|
+
const postLayer = kernelPath.postLayer.map((step) => {
|
|
46
|
+
if (step?.op === 'final_norm') {
|
|
47
|
+
const precision = {
|
|
48
|
+
...(step.precision ?? {}),
|
|
49
|
+
inputDtype: 'f32',
|
|
50
|
+
outputDtype: 'f32',
|
|
51
|
+
};
|
|
52
|
+
if (
|
|
53
|
+
step.precision?.inputDtype === precision.inputDtype
|
|
54
|
+
&& step.precision?.outputDtype === precision.outputDtype
|
|
55
|
+
) {
|
|
56
|
+
return step;
|
|
57
|
+
}
|
|
58
|
+
changed = true;
|
|
59
|
+
return {
|
|
60
|
+
...step,
|
|
61
|
+
precision,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
|
|
65
|
+
return step;
|
|
66
|
+
}
|
|
67
|
+
const replacement = resolveF16ToF32ActivationKernel(step.kernel);
|
|
68
|
+
const existingPrecision = getKernelFilePrecisionPatch(step.kernel);
|
|
69
|
+
const kernelAlreadyF32 = existingPrecision?.inputDtype === 'f32'
|
|
70
|
+
&& existingPrecision?.outputDtype === 'f32';
|
|
71
|
+
if (!replacement && !kernelAlreadyF32) {
|
|
72
|
+
throw new Error(
|
|
73
|
+
`[Logits] stable F32 policy cannot map LM-head kernel "${step.kernel}" to an F32 activation kernel.`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
const kernel = replacement ?? step.kernel;
|
|
77
|
+
const precision = {
|
|
78
|
+
...(step.precision ?? {}),
|
|
79
|
+
inputDtype: 'f32',
|
|
80
|
+
outputDtype: 'f32',
|
|
81
|
+
};
|
|
82
|
+
if (
|
|
83
|
+
kernel === step.kernel
|
|
84
|
+
&& step.precision?.inputDtype === precision.inputDtype
|
|
85
|
+
&& step.precision?.outputDtype === precision.outputDtype
|
|
86
|
+
) {
|
|
87
|
+
return step;
|
|
88
|
+
}
|
|
89
|
+
changed = true;
|
|
90
|
+
return {
|
|
91
|
+
...step,
|
|
92
|
+
kernel,
|
|
93
|
+
precision,
|
|
94
|
+
};
|
|
95
|
+
});
|
|
96
|
+
if (!changed) {
|
|
97
|
+
return kernelPath;
|
|
98
|
+
}
|
|
99
|
+
return {
|
|
100
|
+
...kernelPath,
|
|
101
|
+
postLayer,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
@@ -21,6 +21,7 @@ export interface LogitsConfig {
|
|
|
21
21
|
useTiedEmbeddings: boolean;
|
|
22
22
|
embeddingVocabSize: number | null;
|
|
23
23
|
finalLogitSoftcapping: number | null; // Gemma 2: 30.0 - applies tanh(x/cap)*cap
|
|
24
|
+
logitInputScale: number;
|
|
24
25
|
largeWeights?: LargeWeightConfigSchema;
|
|
25
26
|
/** Dtype for hidden state activations */
|
|
26
27
|
activationDtype?: 'f16' | 'f32';
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
import type { ProbeConfigSchema } from '../../../../config/schema/index.js';
|
|
10
10
|
import type { LogitsConfig } from './types.js';
|
|
11
11
|
|
|
12
|
+
export function resolveLogitInputScale(config: LogitsConfig): number;
|
|
13
|
+
|
|
12
14
|
/**
|
|
13
15
|
* Extract logits for only the last position.
|
|
14
16
|
*
|
|
@@ -4,6 +4,17 @@ import { readBuffer } from '../../../../memory/buffer-pool.js';
|
|
|
4
4
|
import { runProbes } from '../probes.js';
|
|
5
5
|
import { applySoftcapping } from './cpu.js';
|
|
6
6
|
|
|
7
|
+
export function resolveLogitInputScale(config) {
|
|
8
|
+
const value = config?.logitInputScale;
|
|
9
|
+
if (value == null) {
|
|
10
|
+
throw new Error('[Logits] logitInputScale is required.');
|
|
11
|
+
}
|
|
12
|
+
const scale = Number(value);
|
|
13
|
+
if (!Number.isFinite(scale) || scale <= 0) {
|
|
14
|
+
throw new Error(`[Logits] logitInputScale must be a positive finite number; got "${String(value)}".`);
|
|
15
|
+
}
|
|
16
|
+
return scale;
|
|
17
|
+
}
|
|
7
18
|
|
|
8
19
|
export function extractLastPositionLogits(
|
|
9
20
|
logits,
|
|
@@ -39,17 +39,19 @@ export async function applyLoRA(input, baseOutput, lora, dims, getWeightBuffer,
|
|
|
39
39
|
// Extract underlying GPUBuffer for WeightBuffers
|
|
40
40
|
const aBufGPU = isWeightBuffer(aBuf) ? aBuf.buffer : aBuf;
|
|
41
41
|
const bBufGPU = isWeightBuffer(bBuf) ? bBuf.buffer : bBuf;
|
|
42
|
+
const intermediateDtype = input.dtype;
|
|
43
|
+
const outputDtype = baseOutput.dtype;
|
|
42
44
|
let loraIntermediate = null;
|
|
43
45
|
let loraOutput = null;
|
|
44
46
|
let scaled = null;
|
|
45
47
|
try {
|
|
46
48
|
loraIntermediate = recorder
|
|
47
|
-
? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath })
|
|
48
|
-
: await runMatmul(input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath });
|
|
49
|
+
? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
|
|
50
|
+
: await runMatmul(input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
|
|
49
51
|
|
|
50
52
|
loraOutput = recorder
|
|
51
|
-
? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath })
|
|
52
|
-
: await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath });
|
|
53
|
+
? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype })
|
|
54
|
+
: await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype });
|
|
53
55
|
|
|
54
56
|
scaled = recorder
|
|
55
57
|
? await recordScale(recorder, loraOutput, lora.scale, { outputBuffer: null })
|