doppler-gpu 0.4.6 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/README.md +43 -31
  2. package/assets/doppler-runtime-map.svg +1 -3
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/config/conversion/README.md +4 -0
  11. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  12. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  13. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  14. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  19. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  21. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  25. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  26. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  29. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  30. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  31. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  32. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  33. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  34. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  35. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  36. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  37. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  39. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  40. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  41. package/src/config/execution-contract-check.js +14 -10
  42. package/src/config/merge-contract-check.js +1 -0
  43. package/src/config/merge.js +30 -0
  44. package/src/config/param-validator.d.ts +5 -0
  45. package/src/config/param-validator.js +42 -8
  46. package/src/config/required-inference-fields-contract-check.js +14 -0
  47. package/src/config/schema/debug.schema.d.ts +2 -0
  48. package/src/config/schema/debug.schema.js +1 -0
  49. package/src/config/schema/doppler.schema.d.ts +5 -1
  50. package/src/config/schema/doppler.schema.js +82 -292
  51. package/src/config/schema/inference.schema.d.ts +17 -3
  52. package/src/config/schema/kvcache.schema.d.ts +19 -3
  53. package/src/config/schema/kvcache.schema.js +2 -1
  54. package/src/config/schema/manifest.schema.d.ts +38 -1
  55. package/src/config/schema/manifest.schema.js +18 -0
  56. package/src/config/schema/policy-schema-registry.json +5 -0
  57. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  58. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  59. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  60. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  61. package/src/config/transforms/execution-graph-transforms.js +8 -0
  62. package/src/converter/conversion-plan.js +2 -0
  63. package/src/converter/parsers/transformer.js +27 -1
  64. package/src/converter/quantization-info.js +16 -4
  65. package/src/converter/rope-config.js +55 -3
  66. package/src/experimental/training/distillation/student-fixture.js +1 -1
  67. package/src/experimental/training/lora-pipeline.js +1 -1
  68. package/src/experimental/training/lora.js +2 -2
  69. package/src/experimental/training/suite.js +1 -1
  70. package/src/gpu/kernels/README.md +8 -1
  71. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  72. package/src/gpu/kernels/dequant.d.ts +6 -6
  73. package/src/gpu/kernels/dequant.js +17 -6
  74. package/src/gpu/kernels/linear-attention-core.js +1 -6
  75. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  76. package/src/gpu/kernels/matmul-selection.js +4 -1
  77. package/src/gpu/kernels/matmul.d.ts +1 -1
  78. package/src/gpu/kernels/matmul.js +24 -6
  79. package/src/gpu/kernels/moe.d.ts +5 -1
  80. package/src/gpu/kernels/moe.js +5 -2
  81. package/src/gpu/kernels/softmax.d.ts +3 -3
  82. package/src/gpu/kernels/softmax.js +9 -2
  83. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  84. package/src/gpu/kernels/types.d.ts +1 -1
  85. package/src/gpu/profiler.js +11 -1
  86. package/src/inference/browser-harness-model-helpers.js +4 -1
  87. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  88. package/src/inference/browser-harness-text-helpers.js +357 -116
  89. package/src/inference/browser-harness.d.ts +3 -3
  90. package/src/inference/browser-harness.js +266 -11
  91. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  92. package/src/inference/kv-cache/quantized.js +16 -2
  93. package/src/inference/kv-cache/tiered.js +50 -15
  94. package/src/inference/moe-router.js +38 -7
  95. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  96. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  97. package/src/inference/pipelines/text/config.d.ts +5 -1
  98. package/src/inference/pipelines/text/config.js +135 -112
  99. package/src/inference/pipelines/text/embed.d.ts +1 -0
  100. package/src/inference/pipelines/text/embed.js +39 -12
  101. package/src/inference/pipelines/text/execution-plan.js +8 -2
  102. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  103. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  104. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  105. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  106. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  107. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  108. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  109. package/src/inference/pipelines/text/generator-steps.js +3 -0
  110. package/src/inference/pipelines/text/generator.js +6 -1
  111. package/src/inference/pipelines/text/init-rope.js +54 -6
  112. package/src/inference/pipelines/text/init.js +63 -15
  113. package/src/inference/pipelines/text/layer.js +32 -5
  114. package/src/inference/pipelines/text/linear-attention.js +12 -8
  115. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  116. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  117. package/src/inference/pipelines/text/logits/index.js +44 -87
  118. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  119. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  120. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  121. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  122. package/src/inference/pipelines/text/logits/utils.js +11 -0
  123. package/src/inference/pipelines/text/lora-apply.js +6 -4
  124. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  125. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  126. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  127. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  128. package/src/inference/pipelines/text/ops.d.ts +3 -0
  129. package/src/inference/pipelines/text/ops.js +26 -1
  130. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  131. package/src/inference/pipelines/text/weights.js +7 -2
  132. package/src/inference/pipelines/text.js +24 -4
  133. package/src/inference/tokenizers/behavior-flags.js +23 -2
  134. package/src/loader/final-weights-loader.js +12 -1
  135. package/src/rules/execution-rules-contract-check.js +12 -3
  136. package/src/rules/inference/capability-transforms.rules.json +14 -2
  137. package/src/rules/inference/config.rules.json +26 -2
  138. package/src/rules/inference/dtype.rules.json +12 -5
  139. package/src/rules/inference/execution.rules.json +5 -19
  140. package/src/rules/inference/moe.rules.json +97 -0
  141. package/src/tooling/command-api-constants.js +2 -2
  142. package/src/tooling/command-api-family-normalizers.js +2 -1
  143. package/src/tooling/command-api.d.ts +1 -1
  144. package/src/tooling/command-runner-shared.d.ts +1 -1
  145. package/src/tooling/command-runner-shared.js +3 -1
  146. package/src/tooling/node-converter.js +20 -0
  147. package/src/tooling/node-source-runtime.js +5 -1
  148. package/src/version.js +1 -1
  149. package/assets/doppler-readme-hero.svg +0 -84
@@ -2,7 +2,7 @@
2
2
 
3
3
  import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
4
4
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
5
- import { runMatmul, runRMSNorm } from '../../../../gpu/kernel-selector.js';
5
+ import { runMatmul, runRMSNorm, runScale, recordScale } from '../../../../gpu/kernel-selector.js';
6
6
  import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
7
7
  import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
8
8
  import { recordLmHeadArgmaxF16 } from '../../../../gpu/kernels/lm-head-argmax.js';
@@ -21,25 +21,9 @@ import { selectRuleValue } from '../../../../rules/rule-registry.js';
21
21
  import { runProbes } from '../probes.js';
22
22
  import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
23
23
  import { f16BufferToF32 } from './cpu.js';
24
- import { readBufferWithCleanup } from './utils.js';
24
+ import { readBufferWithCleanup, resolveLogitInputScale } from './utils.js';
25
25
  import { f16ToF32 } from '../../../../loader/dtype-utils.js';
26
-
27
- function shouldForceStableF32Logits(config, inputDtype) {
28
- if (inputDtype !== 'f16') {
29
- return false;
30
- }
31
- // Softcapped output heads are numerically sensitive in pure F16 on the
32
- // final RMSNorm + LM-head path. Widen only the logits tail so the main
33
- // layer stack and KV cache can stay on the faster F16 lane.
34
- if (Number.isFinite(config.finalLogitSoftcapping) && config.finalLogitSoftcapping > 0) {
35
- return true;
36
- }
37
- // Small Gemma-family checkpoints can also overflow in pure F16 logits path
38
- // after RMSNorm offset even without output softcapping.
39
- return config.rmsNormWeightOffset === true
40
- && Number.isFinite(config.hiddenSize)
41
- && config.hiddenSize <= 768;
42
- }
26
+ import { shouldForceStableF32Logits, createStableF32LogitsKernelPath } from './precision-policy.js';
43
27
 
44
28
  function resolvePrecisionFieldDtype(precision, fallback, field) {
45
29
  const requested = precision?.[field] ?? fallback;
@@ -84,68 +68,6 @@ async function coerceTensorDtype(tensor, targetDtype, recorder = null, options =
84
68
  throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
85
69
  }
86
70
 
87
- const STABLE_F32_LOGITS_KERNEL_MAP = new Map([
88
- ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
89
- ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
90
- ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
91
- ]);
92
-
93
- function createStableF32LogitsKernelPath(kernelPath) {
94
- if (!kernelPath?.postLayer) {
95
- return kernelPath;
96
- }
97
- let changed = false;
98
- const postLayer = kernelPath.postLayer.map((step) => {
99
- if (step?.op === 'final_norm') {
100
- const precision = {
101
- ...(step.precision ?? {}),
102
- inputDtype: 'f32',
103
- outputDtype: 'f32',
104
- };
105
- if (
106
- step.precision?.inputDtype === precision.inputDtype
107
- && step.precision?.outputDtype === precision.outputDtype
108
- ) {
109
- return step;
110
- }
111
- changed = true;
112
- return {
113
- ...step,
114
- precision,
115
- };
116
- }
117
- if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
118
- return step;
119
- }
120
- const replacement = STABLE_F32_LOGITS_KERNEL_MAP.get(step.kernel) ?? step.kernel;
121
- const precision = {
122
- ...(step.precision ?? {}),
123
- inputDtype: 'f32',
124
- outputDtype: 'f32',
125
- };
126
- if (
127
- replacement === step.kernel
128
- && step.precision?.inputDtype === precision.inputDtype
129
- && step.precision?.outputDtype === precision.outputDtype
130
- ) {
131
- return step;
132
- }
133
- changed = true;
134
- return {
135
- ...step,
136
- kernel: replacement,
137
- precision,
138
- };
139
- });
140
- if (!changed) {
141
- return kernelPath;
142
- }
143
- return {
144
- ...kernelPath,
145
- postLayer,
146
- };
147
- }
148
-
149
71
  const bf16ScratchU32 = new Uint32Array(1);
150
72
  const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
151
73
  const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
@@ -175,6 +97,26 @@ function writeBufferInChunks(queue, buffer, bytes) {
175
97
  }
176
98
  }
177
99
 
100
+ function resolveFinalNormGpuBuffer(finalNorm, queue, label) {
101
+ if (isWeightBuffer(finalNorm)) {
102
+ return { buffer: finalNorm.buffer, owned: false };
103
+ }
104
+ if (isGpuBufferInstance(finalNorm)) {
105
+ return { buffer: finalNorm, owned: false };
106
+ }
107
+ if (!ArrayBuffer.isView(finalNorm)) {
108
+ throw new Error('[Logits] final_norm must be a GPU buffer, typed array, or WeightBuffer.');
109
+ }
110
+ const buffer = acquireBuffer(finalNorm.byteLength, undefined, label);
111
+ try {
112
+ queue.writeBuffer(buffer, 0, finalNorm);
113
+ return { buffer, owned: true };
114
+ } catch (error) {
115
+ releaseBuffer(buffer);
116
+ throw error;
117
+ }
118
+ }
119
+
178
120
  function normalizeRangeBytes(value, label) {
179
121
  if (value instanceof Uint8Array) return value;
180
122
  if (value instanceof ArrayBuffer) return new Uint8Array(value);
@@ -304,9 +246,12 @@ export async function extractLmHeadChunk(
304
246
  vocabSize,
305
247
  rowOffset,
306
248
  rowCount,
307
- sourceDtype = 'f32'
249
+ sourceDtype
308
250
  ) {
309
- const normalizedSourceDtype = String(sourceDtype ?? 'f32').toLowerCase();
251
+ if (typeof sourceDtype !== 'string' || sourceDtype.trim() === '') {
252
+ throw new Error('[Logits] CPU LM head source dtype is required.');
253
+ }
254
+ const normalizedSourceDtype = sourceDtype.trim().toLowerCase();
310
255
  if (normalizedSourceDtype !== 'f32' && normalizedSourceDtype !== 'f16' && normalizedSourceDtype !== 'bf16') {
311
256
  throw new Error(`[Logits] Unsupported CPU LM head source dtype "${sourceDtype}".`);
312
257
  }
@@ -800,6 +745,8 @@ export async function computeLogitsGPU(
800
745
  let normInputOwned = false;
801
746
  let normedTensor;
802
747
  let finalNormTensor;
748
+ let logitInputTensor;
749
+ let logitInputOwned = false;
803
750
  let lmHeadInputTensor;
804
751
  let lmHeadInputOwned = false;
805
752
  let lmHeadBuffer;
@@ -815,13 +762,9 @@ export async function computeLogitsGPU(
815
762
  }
816
763
 
817
764
  // Apply final RMSNorm
818
- if (isGpuBufferInstance(finalNorm)) {
819
- normWeightBuffer = finalNorm;
820
- } else {
821
- normWeightBuffer = acquireBuffer( (finalNorm).byteLength, undefined, 'final_norm_w');
822
- device.queue.writeBuffer(normWeightBuffer, 0, (finalNorm));
823
- normWeightBufferOwned = true;
824
- }
765
+ const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, device.queue, 'final_norm_w');
766
+ normWeightBuffer = resolvedFinalNorm.buffer;
767
+ normWeightBufferOwned = resolvedFinalNorm.owned;
825
768
 
826
769
  const inputDtype = isGpuBufferInstance(hiddenStates) ? activationDtype : 'f32';
827
770
  const inputTensor = createTensor(inputBuffer, inputDtype, [numTokens, hiddenSize], 'logits_input');
@@ -899,21 +842,29 @@ export async function computeLogitsGPU(
899
842
  releaseBuffer(normInputTensor.buffer);
900
843
  normInputOwned = false;
901
844
  }
845
+ const logitInputScale = resolveLogitInputScale(config);
846
+ logitInputTensor = finalNormTensor;
847
+ if (logitInputScale !== 1) {
848
+ logitInputTensor = await runScale(finalNormTensor, logitInputScale, {
849
+ count: numTokens * hiddenSize,
850
+ });
851
+ logitInputOwned = true;
852
+ }
902
853
  const lmHeadRole = resolveLmHeadMatmulRole(phase);
903
854
  const lmHeadInputDtype = forceStableF32Logits
904
- ? finalNormTensor.dtype
905
- : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
855
+ ? logitInputTensor.dtype
856
+ : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'inputDtype');
906
857
  const lmHeadOutputDtype = forceStableF32Logits
907
- ? finalNormTensor.dtype
908
- : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
909
- lmHeadInputTensor = lmHeadInputDtype !== finalNormTensor.dtype
910
- ? await coerceTensorDtype(finalNormTensor, lmHeadInputDtype, null, {
858
+ ? logitInputTensor.dtype
859
+ : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'outputDtype');
860
+ lmHeadInputTensor = lmHeadInputDtype !== logitInputTensor.dtype
861
+ ? await coerceTensorDtype(logitInputTensor, lmHeadInputDtype, null, {
911
862
  executionPolicies: config.executionPolicies ?? null,
912
863
  op: 'lm_head',
913
864
  transitionDeclaredBy: 'step_precision',
914
865
  })
915
- : finalNormTensor;
916
- lmHeadInputOwned = lmHeadInputTensor !== finalNormTensor;
866
+ : logitInputTensor;
867
+ lmHeadInputOwned = lmHeadInputTensor !== logitInputTensor;
917
868
 
918
869
  // Project to vocab via LM head
919
870
  if (isGpuBufferInstance(lmHead)) {
@@ -948,6 +899,7 @@ export async function computeLogitsGPU(
948
899
  // Cleanup intermediate buffers (but keep logitsBuffer)
949
900
  if (inputBufferOwned) { releaseBuffer(inputBuffer); inputBufferOwned = false; }
950
901
  if (lmHeadInputOwned) { releaseBuffer(lmHeadInputTensor.buffer); lmHeadInputOwned = false; }
902
+ if (logitInputOwned) { releaseBuffer(logitInputTensor.buffer); logitInputOwned = false; }
951
903
  if (finalNormTensor) {
952
904
  releaseBuffer(finalNormTensor.buffer);
953
905
  finalNormTensor = null;
@@ -961,6 +913,7 @@ export async function computeLogitsGPU(
961
913
  if (inputBufferOwned && inputBuffer) releaseBuffer(inputBuffer);
962
914
  if (normInputOwned && normInputTensor) releaseBuffer(normInputTensor.buffer);
963
915
  if (lmHeadInputOwned && lmHeadInputTensor) releaseBuffer(lmHeadInputTensor.buffer);
916
+ if (logitInputOwned && logitInputTensor) releaseBuffer(logitInputTensor.buffer);
964
917
  if (finalNormTensor) releaseBuffer(finalNormTensor.buffer);
965
918
  if (normedTensor) releaseBuffer(normedTensor.buffer);
966
919
  if (normWeightBufferOwned && normWeightBuffer) releaseBuffer(normWeightBuffer);
@@ -999,13 +952,9 @@ async function recordLogitsTailGPU(
999
952
 
1000
953
  let normWeightBuffer;
1001
954
  let normWeightOwned = false;
1002
- if (isGpuBufferInstance(finalNorm)) {
1003
- normWeightBuffer = finalNorm;
1004
- } else {
1005
- normWeightBuffer = acquireBuffer( (finalNorm).byteLength, undefined, 'final_norm_w');
1006
- recorder.device.queue.writeBuffer(normWeightBuffer, 0, (finalNorm));
1007
- normWeightOwned = true;
1008
- }
955
+ const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, recorder.device.queue, 'final_norm_w');
956
+ normWeightBuffer = resolvedFinalNorm.buffer;
957
+ normWeightOwned = resolvedFinalNorm.owned;
1009
958
 
1010
959
 
1011
960
  const inputDtype = activationDtype;
@@ -1080,20 +1029,29 @@ async function recordLogitsTailGPU(
1080
1029
  operatorDiagnostics,
1081
1030
  dtype: finalNormTensor.dtype,
1082
1031
  });
1032
+ const logitInputScale = resolveLogitInputScale(config);
1033
+ let logitInputTensor = finalNormTensor;
1034
+ let logitInputOwned = false;
1035
+ if (logitInputScale !== 1) {
1036
+ logitInputTensor = await recordScale(recorder, finalNormTensor, logitInputScale, {
1037
+ count: numTokens * hiddenSize,
1038
+ });
1039
+ logitInputOwned = true;
1040
+ }
1083
1041
  const lmHeadRole = resolveLmHeadMatmulRole(phase);
1084
1042
  const lmHeadInputDtype = forceStableF32Logits
1085
- ? finalNormTensor.dtype
1086
- : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
1043
+ ? logitInputTensor.dtype
1044
+ : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'inputDtype');
1087
1045
  const lmHeadOutputDtype = forceStableF32Logits
1088
- ? finalNormTensor.dtype
1089
- : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
1090
- const lmHeadInputTensor = lmHeadInputDtype !== finalNormTensor.dtype
1091
- ? await coerceTensorDtype(finalNormTensor, lmHeadInputDtype, recorder, {
1046
+ ? logitInputTensor.dtype
1047
+ : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'outputDtype');
1048
+ const lmHeadInputTensor = lmHeadInputDtype !== logitInputTensor.dtype
1049
+ ? await coerceTensorDtype(logitInputTensor, lmHeadInputDtype, recorder, {
1092
1050
  executionPolicies: config.executionPolicies ?? null,
1093
1051
  op: 'lm_head',
1094
1052
  transitionDeclaredBy: 'step_precision',
1095
1053
  })
1096
- : finalNormTensor;
1054
+ : logitInputTensor;
1097
1055
 
1098
1056
  // Get LM head buffer
1099
1057
 
@@ -1119,6 +1077,8 @@ async function recordLogitsTailGPU(
1119
1077
  lmHeadOutputDtype,
1120
1078
  normedTensor,
1121
1079
  finalNormTensor,
1080
+ logitInputTensor,
1081
+ logitInputOwned,
1122
1082
  normInputTensor,
1123
1083
  normInputOwned,
1124
1084
  normWeightBuffer,
@@ -1142,7 +1102,10 @@ function trackRecordedLogitsTail(recorder, tail) {
1142
1102
  trackTempBufferOnce(tail.normedTensor.buffer);
1143
1103
  }
1144
1104
  trackTempBufferOnce(tail.finalNormTensor.buffer);
1145
- if (tail.lmHeadInputTensor !== tail.finalNormTensor) {
1105
+ if (tail.logitInputOwned) {
1106
+ trackTempBufferOnce(tail.logitInputTensor.buffer);
1107
+ }
1108
+ if (tail.lmHeadInputTensor !== tail.logitInputTensor) {
1146
1109
  trackTempBufferOnce(tail.lmHeadInputTensor.buffer);
1147
1110
  }
1148
1111
  if (tail.normWeightOwned) {
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
12
12
  // Imports for computeLogits orchestrator
13
13
  import { getDevice } from '../../../../gpu/device.js';
14
14
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
15
- import { runMatmul, runRMSNorm, castF16ToF32, castF32ToF16 } from '../../../../gpu/kernel-selector.js';
15
+ import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16 } from '../../../../gpu/kernel-selector.js';
16
16
  import { createTensor } from '../../../../gpu/tensor.js';
17
17
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
18
18
  import { kernelTrace, traceStep } from '../kernel-trace.js';
@@ -20,29 +20,13 @@ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
20
20
  import { runProbes } from '../probes.js';
21
21
  import { rmsNormCPU, matmulCPU, f16BufferToF32 } from './cpu.js';
22
22
  import { resolveCpuWeightDims, computeChunkedLogitsGPU, computeSplitLogitsGPU } from './gpu.js';
23
- import { finalizeLogits, readBufferWithCleanup } from './utils.js';
23
+ import { finalizeLogits, readBufferWithCleanup, resolveLogitInputScale } from './utils.js';
24
24
  import { getLogitsHealth } from '../debug-utils/index.js';
25
25
  import { getRuntimeConfig } from '../../../../config/runtime.js';
26
26
  import { getKernelPathMatmulPrecision, getKernelPathStepPrecision } from '../../../../config/kernel-path-loader.js';
27
27
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
28
28
  import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
29
-
30
- function shouldForceStableF32Logits(config, inputDtype) {
31
- if (inputDtype !== 'f16') {
32
- return false;
33
- }
34
- // Softcapped output heads are numerically sensitive in pure F16 on the
35
- // final RMSNorm + LM-head path. Widen only the logits tail so the main
36
- // layer stack and KV cache can stay on the faster F16 lane.
37
- if (Number.isFinite(config.finalLogitSoftcapping) && config.finalLogitSoftcapping > 0) {
38
- return true;
39
- }
40
- // Small Gemma-family checkpoints can also overflow in pure F16 logits path
41
- // after RMSNorm offset even without output softcapping.
42
- return config.rmsNormWeightOffset === true
43
- && Number.isFinite(config.hiddenSize)
44
- && config.hiddenSize <= 768;
45
- }
29
+ import { shouldForceStableF32Logits, createStableF32LogitsKernelPath } from './precision-policy.js';
46
30
 
47
31
  function resolvePrecisionFieldDtype(precision, fallback, field) {
48
32
  const requested = precision?.[field] ?? fallback;
@@ -87,68 +71,6 @@ async function coerceTensorDtype(tensor, targetDtype, options = {}) {
87
71
  throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
88
72
  }
89
73
 
90
- const STABLE_F32_LOGITS_KERNEL_MAP = new Map([
91
- ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
92
- ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
93
- ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
94
- ]);
95
-
96
- function createStableF32LogitsKernelPath(kernelPath) {
97
- if (!kernelPath?.postLayer) {
98
- return kernelPath;
99
- }
100
- let changed = false;
101
- const postLayer = kernelPath.postLayer.map((step) => {
102
- if (step?.op === 'final_norm') {
103
- const precision = {
104
- ...(step.precision ?? {}),
105
- inputDtype: 'f32',
106
- outputDtype: 'f32',
107
- };
108
- if (
109
- step.precision?.inputDtype === precision.inputDtype
110
- && step.precision?.outputDtype === precision.outputDtype
111
- ) {
112
- return step;
113
- }
114
- changed = true;
115
- return {
116
- ...step,
117
- precision,
118
- };
119
- }
120
- if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
121
- return step;
122
- }
123
- const replacement = STABLE_F32_LOGITS_KERNEL_MAP.get(step.kernel) ?? step.kernel;
124
- const precision = {
125
- ...(step.precision ?? {}),
126
- inputDtype: 'f32',
127
- outputDtype: 'f32',
128
- };
129
- if (
130
- replacement === step.kernel
131
- && step.precision?.inputDtype === precision.inputDtype
132
- && step.precision?.outputDtype === precision.outputDtype
133
- ) {
134
- return step;
135
- }
136
- changed = true;
137
- return {
138
- ...step,
139
- kernel: replacement,
140
- precision,
141
- };
142
- });
143
- if (!changed) {
144
- return kernelPath;
145
- }
146
- return {
147
- ...kernelPath,
148
- postLayer,
149
- };
150
- }
151
-
152
74
  async function traceTensorHealth(label, tensor, elementCount) {
153
75
  if (!isTraceEnabled('logits')) {
154
76
  return;
@@ -173,6 +95,26 @@ export function resolveLmHeadMatmulConfig(numTokens, options = null) {
173
95
  };
174
96
  }
175
97
 
98
+ function resolveFinalNormGpuBuffer(finalNorm, device, label) {
99
+ if (isWeightBuffer(finalNorm)) {
100
+ return { buffer: finalNorm.buffer, owned: false };
101
+ }
102
+ if (isGpuBufferInstance(finalNorm)) {
103
+ return { buffer: finalNorm, owned: false };
104
+ }
105
+ if (!ArrayBuffer.isView(finalNorm)) {
106
+ throw new Error('[Logits] final_norm must be a GPU buffer, typed array, or WeightBuffer.');
107
+ }
108
+ const buffer = acquireBuffer(finalNorm.byteLength, undefined, label);
109
+ try {
110
+ device.queue.writeBuffer(buffer, 0, finalNorm);
111
+ return { buffer, owned: true };
112
+ } catch (error) {
113
+ releaseBuffer(buffer);
114
+ throw error;
115
+ }
116
+ }
117
+
176
118
 
177
119
  export async function computeLogits(
178
120
  hiddenStates,
@@ -281,6 +223,12 @@ export async function computeLogits(
281
223
  rmsNormEps,
282
224
  config.rmsNormWeightOffset
283
225
  );
226
+ const logitInputScale = resolveLogitInputScale(config);
227
+ if (logitInputScale !== 1) {
228
+ for (let i = 0; i < normed.length; i += 1) {
229
+ normed[i] *= logitInputScale;
230
+ }
231
+ }
284
232
  const rawLogits = isCpuWeightBuffer(lmHead)
285
233
  ? matmulCPU(
286
234
  normed,
@@ -319,13 +267,13 @@ export async function computeLogits(
319
267
  // 2. Apply final RMSNorm
320
268
 
321
269
  let normWeightBuffer;
270
+ let normWeightBufferOwned = false;
322
271
  if (getNormWeightBuffer) {
323
272
  normWeightBuffer = getNormWeightBuffer(finalNorm, 'final_norm_w');
324
- } else if (isGpuBufferInstance(finalNorm)) {
325
- normWeightBuffer = finalNorm;
326
273
  } else {
327
- normWeightBuffer = acquireBuffer((finalNorm).byteLength, undefined, 'final_norm_w');
328
- device.queue.writeBuffer(normWeightBuffer, 0, (finalNorm));
274
+ const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, device, 'final_norm_w');
275
+ normWeightBuffer = resolvedFinalNorm.buffer;
276
+ normWeightBufferOwned = resolvedFinalNorm.owned;
329
277
  }
330
278
 
331
279
  // Debug: Check hidden state before final norm
@@ -418,6 +366,15 @@ export async function computeLogits(
418
366
  await debugCheckBuffer(finalNormTensor.buffer, 'After final norm', numTokens, hiddenSize);
419
367
  }
420
368
 
369
+ const logitInputScale = resolveLogitInputScale(config);
370
+ if (logitInputScale !== 1) {
371
+ const unscaledFinalNormTensor = finalNormTensor;
372
+ finalNormTensor = await runScale(finalNormTensor, logitInputScale, {
373
+ count: numTokens * hiddenSize,
374
+ });
375
+ releaseBuffer(unscaledFinalNormTensor.buffer);
376
+ }
377
+
421
378
  const lastTokenMatmul = resolveLmHeadMatmulConfig(numTokens, options);
422
379
  const { lastPositionOnly, matmulRows } = lastTokenMatmul;
423
380
  const matmulPhaseOverride = lastTokenMatmul.phaseOverride;
@@ -473,7 +430,7 @@ export async function computeLogits(
473
430
  if (inputBufferOwned) releaseBuffer(inputBuffer);
474
431
  releaseBuffer(finalNormTensor.buffer);
475
432
  if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
476
- if (!getNormWeightBuffer && !isGpuBufferInstance(finalNorm)) releaseBuffer(normWeightBuffer);
433
+ if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
477
434
 
478
435
  return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
479
436
  }
@@ -554,7 +511,7 @@ export async function computeLogits(
554
511
  releaseBuffer(finalNormTensor.buffer);
555
512
  if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
556
513
  releaseBuffer(logitsTensor.buffer);
557
- if (!getNormWeightBuffer && !isGpuBufferInstance(finalNorm)) releaseBuffer(normWeightBuffer);
514
+ if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
558
515
  if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
559
516
  });
560
517
 
@@ -0,0 +1,11 @@
1
+ import type { KernelPathSchema } from '../../../../config/schema/index.js';
2
+ import type { LogitsConfig } from './types.js';
3
+
4
+ export function shouldForceStableF32Logits(
5
+ config: Pick<LogitsConfig, 'finalLogitSoftcapping' | 'rmsNormWeightOffset' | 'hiddenSize'>,
6
+ inputDtype: 'f16' | 'f32'
7
+ ): boolean;
8
+
9
+ export function createStableF32LogitsKernelPath(
10
+ kernelPath: KernelPathSchema | null | undefined
11
+ ): KernelPathSchema | null | undefined;
@@ -0,0 +1,103 @@
1
+ import {
2
+ getKernelFilePrecisionPatch,
3
+ resolveF16ToF32ActivationKernel,
4
+ } from '../../../../config/transforms/execution-graph-transforms.js';
5
+ import { selectRuleValue } from '../../../../rules/rule-registry.js';
6
+
7
+ function requireLogitsConfigValue(condition, label, value) {
8
+ if (!condition) {
9
+ throw new Error(`[Logits] stable F32 policy requires ${label}, got ${String(value)}.`);
10
+ }
11
+ }
12
+
13
+ export function shouldForceStableF32Logits(config, inputDtype) {
14
+ requireLogitsConfigValue(inputDtype === 'f16' || inputDtype === 'f32', 'input dtype "f16" or "f32"', inputDtype);
15
+ requireLogitsConfigValue(config && typeof config === 'object', 'a logits config object', config);
16
+ requireLogitsConfigValue(
17
+ config.finalLogitSoftcapping === null || Number.isFinite(config.finalLogitSoftcapping),
18
+ 'config.finalLogitSoftcapping to be null or finite',
19
+ config.finalLogitSoftcapping
20
+ );
21
+ requireLogitsConfigValue(
22
+ typeof config.rmsNormWeightOffset === 'boolean',
23
+ 'config.rmsNormWeightOffset to be boolean',
24
+ config.rmsNormWeightOffset
25
+ );
26
+ requireLogitsConfigValue(
27
+ Number.isFinite(config.hiddenSize) && config.hiddenSize > 0,
28
+ 'config.hiddenSize to be a positive finite number',
29
+ config.hiddenSize
30
+ );
31
+
32
+ return selectRuleValue('inference', 'config', 'stableF32Logits', {
33
+ inputDtype,
34
+ finalLogitSoftcapping: config.finalLogitSoftcapping,
35
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
36
+ hiddenSize: config.hiddenSize,
37
+ });
38
+ }
39
+
40
+ export function createStableF32LogitsKernelPath(kernelPath) {
41
+ if (!kernelPath?.postLayer) {
42
+ return kernelPath;
43
+ }
44
+ let changed = false;
45
+ const postLayer = kernelPath.postLayer.map((step) => {
46
+ if (step?.op === 'final_norm') {
47
+ const precision = {
48
+ ...(step.precision ?? {}),
49
+ inputDtype: 'f32',
50
+ outputDtype: 'f32',
51
+ };
52
+ if (
53
+ step.precision?.inputDtype === precision.inputDtype
54
+ && step.precision?.outputDtype === precision.outputDtype
55
+ ) {
56
+ return step;
57
+ }
58
+ changed = true;
59
+ return {
60
+ ...step,
61
+ precision,
62
+ };
63
+ }
64
+ if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
65
+ return step;
66
+ }
67
+ const replacement = resolveF16ToF32ActivationKernel(step.kernel);
68
+ const existingPrecision = getKernelFilePrecisionPatch(step.kernel);
69
+ const kernelAlreadyF32 = existingPrecision?.inputDtype === 'f32'
70
+ && existingPrecision?.outputDtype === 'f32';
71
+ if (!replacement && !kernelAlreadyF32) {
72
+ throw new Error(
73
+ `[Logits] stable F32 policy cannot map LM-head kernel "${step.kernel}" to an F32 activation kernel.`
74
+ );
75
+ }
76
+ const kernel = replacement ?? step.kernel;
77
+ const precision = {
78
+ ...(step.precision ?? {}),
79
+ inputDtype: 'f32',
80
+ outputDtype: 'f32',
81
+ };
82
+ if (
83
+ kernel === step.kernel
84
+ && step.precision?.inputDtype === precision.inputDtype
85
+ && step.precision?.outputDtype === precision.outputDtype
86
+ ) {
87
+ return step;
88
+ }
89
+ changed = true;
90
+ return {
91
+ ...step,
92
+ kernel,
93
+ precision,
94
+ };
95
+ });
96
+ if (!changed) {
97
+ return kernelPath;
98
+ }
99
+ return {
100
+ ...kernelPath,
101
+ postLayer,
102
+ };
103
+ }
@@ -21,6 +21,7 @@ export interface LogitsConfig {
21
21
  useTiedEmbeddings: boolean;
22
22
  embeddingVocabSize: number | null;
23
23
  finalLogitSoftcapping: number | null; // Gemma 2: 30.0 - applies tanh(x/cap)*cap
24
+ logitInputScale: number;
24
25
  largeWeights?: LargeWeightConfigSchema;
25
26
  /** Dtype for hidden state activations */
26
27
  activationDtype?: 'f16' | 'f32';
@@ -9,6 +9,8 @@
9
9
  import type { ProbeConfigSchema } from '../../../../config/schema/index.js';
10
10
  import type { LogitsConfig } from './types.js';
11
11
 
12
+ export function resolveLogitInputScale(config: LogitsConfig): number;
13
+
12
14
  /**
13
15
  * Extract logits for only the last position.
14
16
  *
@@ -4,6 +4,17 @@ import { readBuffer } from '../../../../memory/buffer-pool.js';
4
4
  import { runProbes } from '../probes.js';
5
5
  import { applySoftcapping } from './cpu.js';
6
6
 
7
+ export function resolveLogitInputScale(config) {
8
+ const value = config?.logitInputScale;
9
+ if (value == null) {
10
+ throw new Error('[Logits] logitInputScale is required.');
11
+ }
12
+ const scale = Number(value);
13
+ if (!Number.isFinite(scale) || scale <= 0) {
14
+ throw new Error(`[Logits] logitInputScale must be a positive finite number; got "${String(value)}".`);
15
+ }
16
+ return scale;
17
+ }
7
18
 
8
19
  export function extractLastPositionLogits(
9
20
  logits,
@@ -39,17 +39,19 @@ export async function applyLoRA(input, baseOutput, lora, dims, getWeightBuffer,
39
39
  // Extract underlying GPUBuffer for WeightBuffers
40
40
  const aBufGPU = isWeightBuffer(aBuf) ? aBuf.buffer : aBuf;
41
41
  const bBufGPU = isWeightBuffer(bBuf) ? bBuf.buffer : bBuf;
42
+ const intermediateDtype = input.dtype;
43
+ const outputDtype = baseOutput.dtype;
42
44
  let loraIntermediate = null;
43
45
  let loraOutput = null;
44
46
  let scaled = null;
45
47
  try {
46
48
  loraIntermediate = recorder
47
- ? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath })
48
- : await runMatmul(input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath });
49
+ ? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
50
+ : await runMatmul(input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
49
51
 
50
52
  loraOutput = recorder
51
- ? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath })
52
- : await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath });
53
+ ? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype })
54
+ : await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype });
53
55
 
54
56
  scaled = recorder
55
57
  ? await recordScale(recorder, loraOutput, lora.scale, { outputBuffer: null })