doppler-gpu 0.4.10 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +27 -0
  3. package/models/catalog.json +612 -7
  4. package/package.json +13 -1
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +126 -4
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +64 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/kernels/registry.json +52 -9
  18. package/src/config/merge.js +24 -0
  19. package/src/config/required-inference-fields-contract-check.js +2 -0
  20. package/src/config/schema/converter.schema.d.ts +7 -0
  21. package/src/config/schema/converter.schema.js +15 -0
  22. package/src/config/schema/inference.schema.d.ts +8 -0
  23. package/src/config/schema/manifest.schema.d.ts +32 -1
  24. package/src/config/schema/manifest.schema.js +26 -0
  25. package/src/converter/conversion-plan.js +15 -1
  26. package/src/converter/core.js +5 -4
  27. package/src/converter/source-tensor-rules.d.ts +50 -0
  28. package/src/converter/source-tensor-rules.js +180 -0
  29. package/src/converter/source-tokenizer.d.ts +45 -0
  30. package/src/converter/source-tokenizer.js +155 -0
  31. package/src/experimental/adapters/lora-loader.js +15 -0
  32. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  33. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  34. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  35. package/src/inference/pipelines/text/attention/projections.js +161 -24
  36. package/src/inference/pipelines/text/attention/record.js +67 -9
  37. package/src/inference/pipelines/text/attention/run.js +60 -7
  38. package/src/inference/pipelines/text/attention/types.js +18 -9
  39. package/src/inference/pipelines/text/config.d.ts +5 -0
  40. package/src/inference/pipelines/text/config.js +91 -0
  41. package/src/inference/pipelines/text/ffn/dense.js +189 -2
  42. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  43. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  44. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  45. package/src/inference/pipelines/text/generator-steps.js +3 -1
  46. package/src/inference/pipelines/text/generator.js +90 -3
  47. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  48. package/src/inference/pipelines/text/init.d.ts +2 -0
  49. package/src/inference/pipelines/text/init.js +2 -0
  50. package/src/inference/pipelines/text/layer.js +6 -0
  51. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  52. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  53. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  54. package/src/inference/pipelines/text/logits/index.js +85 -11
  55. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  56. package/src/inference/pipelines/text/ops.js +32 -0
  57. package/src/inference/pipelines/text/types.d.ts +38 -0
  58. package/src/inference/pipelines/text/weights.d.ts +10 -0
  59. package/src/inference/pipelines/text/weights.js +33 -1
  60. package/src/inference/pipelines/text.d.ts +4 -2
  61. package/src/inference/pipelines/text.js +112 -0
  62. package/src/inference/tokenizers/behavior-flags.js +5 -1
  63. package/src/inference/tokenizers/bundled.js +99 -8
  64. package/src/loader/doppler-loader.d.ts +2 -0
  65. package/src/loader/doppler-loader.js +9 -0
  66. package/src/loader/final-weights-loader.d.ts +8 -0
  67. package/src/loader/final-weights-loader.js +42 -0
  68. package/src/loader/layer-loader.d.ts +6 -0
  69. package/src/loader/layer-loader.js +102 -3
  70. package/src/loader/loader-types.d.ts +15 -2
  71. package/src/tooling/node-converter.js +33 -1
  72. package/src/tooling/node-webgpu.js +85 -2
  73. package/src/version.js +1 -1
@@ -2,7 +2,13 @@ import { getDevice } from '../../../gpu/device.js';
2
2
  import { Q4K_BLOCK_BYTES, q4kBlockCount } from '../../../config/schema/index.js';
3
3
  import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
4
4
  import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
5
- import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
5
+ import {
6
+ createWeightBuffer,
7
+ getWeightDtype,
8
+ isGpuBufferInstance,
9
+ isWeightBuffer,
10
+ tagBufferDtype,
11
+ } from '../../../gpu/weight-buffer.js';
6
12
  import { selectRuleValue } from '../../../rules/rule-registry.js';
7
13
  import { log } from '../../../debug/index.js';
8
14
 
@@ -112,6 +118,26 @@ function formatsMatch(a, b) {
112
118
  && a?.bytesPerRow === b?.bytesPerRow;
113
119
  }
114
120
 
121
+ function resolveBiasStorageFormat(weight, expectedElements) {
122
+ if (!weight || !Number.isFinite(expectedElements) || expectedElements <= 0) {
123
+ return null;
124
+ }
125
+ const buffer = isWeightBuffer(weight) ? weight.buffer : weight;
126
+ if (!isGpuBufferInstance(buffer)) {
127
+ return null;
128
+ }
129
+ const dtype = String(getWeightDtype(weight) ?? getWeightDtype(buffer) ?? '').toLowerCase();
130
+ if (dtype !== 'f16' && dtype !== 'f32') {
131
+ return null;
132
+ }
133
+ const bytesPerElement = dtype === 'f16' ? 2 : 4;
134
+ const byteLength = expectedElements * bytesPerElement;
135
+ if (buffer.size < byteLength) {
136
+ return null;
137
+ }
138
+ return { buffer, dtype, bytesPerElement, byteLength };
139
+ }
140
+
115
141
  export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, options = {}) {
116
142
  const device = getDevice();
117
143
  if (!device) {
@@ -174,6 +200,17 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
174
200
  continue;
175
201
  }
176
202
 
203
+ const projectionBiases = [weights.qProjBias, weights.kProjBias, weights.vProjBias];
204
+ const presentProjectionBiases = projectionBiases.filter(Boolean).length;
205
+ if (presentProjectionBiases !== 0 && presentProjectionBiases !== projectionBiases.length) {
206
+ log.debug('QKV Fusion', `Layer ${l}: partial Q/K/V projection biases require split projections`);
207
+ continue;
208
+ }
209
+ if (hasAttentionOutputGate && presentProjectionBiases > 0) {
210
+ log.debug('QKV Fusion', `Layer ${l}: gated Q projection biases require split projections`);
211
+ continue;
212
+ }
213
+
177
214
  const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
178
215
  const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
179
216
  const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
@@ -228,6 +265,32 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
228
265
  })
229
266
  : null;
230
267
 
268
+ let qBiasFormat = null;
269
+ let kBiasFormat = null;
270
+ let vBiasFormat = null;
271
+ let qkvBiasBuffer = null;
272
+ if (presentProjectionBiases === projectionBiases.length) {
273
+ qBiasFormat = resolveBiasStorageFormat(weights.qProjBias, qSize);
274
+ kBiasFormat = resolveBiasStorageFormat(weights.kProjBias, kSize);
275
+ vBiasFormat = resolveBiasStorageFormat(weights.vProjBias, vSize);
276
+ const biasFormatsMatch = qBiasFormat
277
+ && kBiasFormat
278
+ && vBiasFormat
279
+ && qBiasFormat.dtype === kBiasFormat.dtype
280
+ && qBiasFormat.dtype === vBiasFormat.dtype;
281
+ if (!biasFormatsMatch) {
282
+ log.debug('QKV Fusion', `Layer ${l}: inconsistent Q/K/V projection bias formats, skipping`);
283
+ qkvBuffer.destroy();
284
+ qGateBuffer?.destroy();
285
+ continue;
286
+ }
287
+ qkvBiasBuffer = device.createBuffer({
288
+ label: `layer_${l}_qkv_proj_bias`,
289
+ size: qBiasFormat.byteLength + kBiasFormat.byteLength + vBiasFormat.byteLength,
290
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
291
+ });
292
+ }
293
+
231
294
  const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
232
295
  if (hasAttentionOutputGate) {
233
296
  for (let head = 0; head < numHeads; head++) {
@@ -266,6 +329,23 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
266
329
  qkvBuffer, qBytes + kFormat.byteLength,
267
330
  vFormat.byteLength
268
331
  );
332
+ if (qkvBiasBuffer) {
333
+ encoder.copyBufferToBuffer(qBiasFormat.buffer, 0, qkvBiasBuffer, 0, qBiasFormat.byteLength);
334
+ encoder.copyBufferToBuffer(
335
+ kBiasFormat.buffer,
336
+ 0,
337
+ qkvBiasBuffer,
338
+ qBiasFormat.byteLength,
339
+ kBiasFormat.byteLength
340
+ );
341
+ encoder.copyBufferToBuffer(
342
+ vBiasFormat.buffer,
343
+ 0,
344
+ qkvBiasBuffer,
345
+ qBiasFormat.byteLength + kBiasFormat.byteLength,
346
+ vBiasFormat.byteLength
347
+ );
348
+ }
269
349
  device.queue.submit([encoder.finish()]);
270
350
 
271
351
  // Store fused buffer, sizes, and dtype
@@ -278,6 +358,10 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
278
358
  );
279
359
  weights.qkvSizes = [qSize, kSize, vSize];
280
360
  weights.qkvDtype = dtype;
361
+ if (qkvBiasBuffer) {
362
+ tagBufferDtype(qkvBiasBuffer, qBiasFormat.dtype);
363
+ weights.qkvProjBias = qkvBiasBuffer;
364
+ }
281
365
  if (qGateBuffer) {
282
366
  weights.qGateProj = createWeightBuffer(
283
367
  qGateBuffer,
@@ -237,7 +237,9 @@ export interface WeightLoadResult {
237
237
  layerWeights: Map<string, LayerWeights>;
238
238
  embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
239
239
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
240
+ lmHeadBias: Float32Array | null;
240
241
  finalNorm: GPUBuffer | Float32Array | null;
242
+ finalNormBias: Float32Array | null;
241
243
  embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
242
244
  diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
243
245
  perLayerInputWeights: PerLayerInputWeights | null;
@@ -1020,7 +1020,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1020
1020
  layerWeights,
1021
1021
  embeddings: dopplerLoader.embeddings,
1022
1022
  lmHead: dopplerLoader.lmHead,
1023
+ lmHeadBias: dopplerLoader.lmHeadBias,
1023
1024
  finalNorm: dopplerLoader.finalNorm,
1025
+ finalNormBias: dopplerLoader.finalNormBias,
1024
1026
  embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
1025
1027
  diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
1026
1028
  perLayerInputWeights: dopplerLoader.perLayerInputWeights,
@@ -57,6 +57,7 @@ function shouldUseSandwichRMSNormPairFusion({
57
57
  attnOutput,
58
58
  inputTensor,
59
59
  }) {
60
+ if (context.config?.normalizationType === 'layernorm') return false;
60
61
  const mergedSession = getRuntimeConfig()?.inference?.session;
61
62
  if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
62
63
  return false;
@@ -123,6 +124,7 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
123
124
  activationDtype,
124
125
  layerScalar,
125
126
  }) {
127
+ if (config.normalizationType === 'layernorm') return false;
126
128
  const mergedSession = getRuntimeConfig()?.inference?.session;
127
129
  if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
128
130
  return false;
@@ -176,6 +178,7 @@ function shouldUseStandardPostFfnNextInputRMSNormPairFusion({
176
178
  layerScalar,
177
179
  residualBranchScale,
178
180
  }) {
181
+ if (config.normalizationType === 'layernorm') return false;
179
182
  const mergedSession = getRuntimeConfig()?.inference?.session;
180
183
  if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
181
184
  return false;
@@ -851,6 +854,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
851
854
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
852
855
  queryPreAttnScalar: config.queryPreAttnScalar,
853
856
  queryKeyNorm,
857
+ queryKeyNormType: config.queryKeyNormType,
858
+ queryKeyNormAxis: config.queryKeyNormAxis,
854
859
  queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
855
860
  valueNorm: config.valueNorm,
856
861
  attentionOutputGate: config.attentionOutputGate,
@@ -860,6 +865,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
860
865
  ? (context.multimodalBidirectionalSpan ?? null)
861
866
  : null,
862
867
  rmsNormWeightOffset: config.rmsNormWeightOffset,
868
+ normalizationType: config.normalizationType,
863
869
  ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
864
870
  ropeFrequencyBaseDim: resolveAttentionFrequencyBaseDim(config, layerType),
865
871
  ropeInterleaved: config.ropeInterleaved,
@@ -25,6 +25,12 @@ export function rmsNormCPU(
25
25
  eps?: number,
26
26
  rmsNormWeightOffset?: boolean
27
27
  ): Float32Array;
28
+ export function layerNormCPU(
29
+ x: Float32Array,
30
+ weight: Float32Array,
31
+ bias: Float32Array,
32
+ eps?: number
33
+ ): Float32Array;
28
34
 
29
35
  /**
30
36
  * Convert a single float16 value to float32.
@@ -25,6 +25,28 @@ export function rmsNormCPU(
25
25
  return result;
26
26
  }
27
27
 
28
+ export function layerNormCPU(x, weight, bias, eps = 1e-5) {
29
+ if (weight.length !== x.length || bias.length !== x.length) {
30
+ throw new Error(
31
+ `LayerNorm parameter length mismatch: input=${x.length}, weight=${weight.length}, bias=${bias.length}.`
32
+ );
33
+ }
34
+ let sum = 0;
35
+ for (let i = 0; i < x.length; i++) sum += x[i];
36
+ const mean = sum / x.length;
37
+ let squaredDeviation = 0;
38
+ for (let i = 0; i < x.length; i++) {
39
+ const centered = x[i] - mean;
40
+ squaredDeviation += centered * centered;
41
+ }
42
+ const invStd = 1 / Math.sqrt(squaredDeviation / x.length + eps);
43
+ const result = new Float32Array(x.length);
44
+ for (let i = 0; i < x.length; i++) {
45
+ result[i] = (x[i] - mean) * invStd * weight[i] + bias[i];
46
+ }
47
+ return result;
48
+ }
49
+
28
50
 
29
51
  export function f16BufferToF32(data) {
30
52
  const u16 = new Uint16Array(data);
@@ -17,7 +17,7 @@ import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
17
17
  export type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
18
18
 
19
19
  // Re-export CPU functions
20
- export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
20
+ export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
21
21
 
22
22
  // Re-export GPU functions
23
23
  export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
@@ -1,7 +1,7 @@
1
1
 
2
2
 
3
3
  // Re-export CPU functions
4
- export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
4
+ export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
5
5
 
6
6
  // Re-export GPU functions
7
7
  export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
12
12
  // Imports for computeLogits orchestrator
13
13
  import { getDevice } from '../../../../gpu/device.js';
14
14
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
15
- import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
15
+ import { runLayerNorm, runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
16
16
  import { createTensor } from '../../../../gpu/tensor.js';
17
17
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
18
18
  import { kernelTrace, traceStep } from '../kernel-trace.js';
@@ -72,6 +72,45 @@ function normalizeSelectedLogitTokenIds(value, vocabSize) {
72
72
  return tokenIds;
73
73
  }
74
74
 
75
+ function applyLmHeadBias(logits, bias, rowCount, rowSize, selectedTokenIds = null) {
76
+ if (bias == null) {
77
+ return logits;
78
+ }
79
+ if (!(bias instanceof Float32Array)) {
80
+ throw new Error('[Logits] LM head bias must be a Float32Array.');
81
+ }
82
+ if (selectedTokenIds !== null) {
83
+ if (logits.length !== selectedTokenIds.length) {
84
+ throw new Error('[Logits] Selected-logit bias application length mismatch.');
85
+ }
86
+ for (let index = 0; index < selectedTokenIds.length; index += 1) {
87
+ const tokenId = selectedTokenIds[index];
88
+ if (tokenId >= bias.length) {
89
+ throw new Error(`[Logits] LM head bias is missing token ${tokenId}.`);
90
+ }
91
+ logits[index] += bias[tokenId];
92
+ }
93
+ return logits;
94
+ }
95
+ if (bias.length < rowSize) {
96
+ throw new Error(
97
+ `[Logits] LM head bias length mismatch: expected at least ${rowSize}, got ${bias.length}.`
98
+ );
99
+ }
100
+ if (logits.length !== rowCount * rowSize) {
101
+ throw new Error(
102
+ `[Logits] LM head bias matrix mismatch: expected ${rowCount * rowSize} logits, got ${logits.length}.`
103
+ );
104
+ }
105
+ for (let row = 0; row < rowCount; row += 1) {
106
+ const offset = row * rowSize;
107
+ for (let column = 0; column < rowSize; column += 1) {
108
+ logits[offset + column] += bias[column];
109
+ }
110
+ }
111
+ return logits;
112
+ }
113
+
75
114
  async function coerceTensorDtype(tensor, targetDtype, options = {}) {
76
115
  if (!targetDtype || tensor.dtype === targetDtype) {
77
116
  return tensor;
@@ -164,7 +203,7 @@ export async function computeLogits(
164
203
  activationDtype: activationDtypeOverride,
165
204
  } = config;
166
205
  const activationDtype = activationDtypeOverride ?? getRuntimeConfig().inference.compute.activationDtype;
167
- const { finalNorm, lmHead } = weights;
206
+ const { finalNorm, finalNormBias = null, lmHead, lmHeadBias = null } = weights;
168
207
  const device = getDevice();
169
208
 
170
209
  // Consistency check: warn if lmHead weight dtype disagrees with layer activation dtype.
@@ -269,6 +308,7 @@ export async function computeLogits(
269
308
  cpuWeightLayout === 'column' ? cpuWeightVocabSize : null
270
309
  )
271
310
  : matmulCPU(normed, (lmHead), numTokens, matmulVocabSize, hiddenSize);
311
+ applyLmHeadBias(rawLogits, lmHeadBias, numTokens, matmulVocabSize);
272
312
  return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
273
313
  }
274
314
 
@@ -293,7 +333,7 @@ export async function computeLogits(
293
333
  dtype: inputDtype,
294
334
  });
295
335
 
296
- // 2. Apply final RMSNorm
336
+ // 2. Apply the manifest-owned final normalization.
297
337
 
298
338
  let normWeightBuffer;
299
339
  let normWeightBufferOwned = false;
@@ -319,7 +359,10 @@ export async function computeLogits(
319
359
  const kernelPath = config.kernelPath ?? null;
320
360
  const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
321
361
  const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
322
- const forceStableF32Logits = !hasExplicitFinalNormPrecision && shouldForceStableF32Logits(config, inputDtype);
362
+ const forceStableF32Logits = !hasExplicitFinalNormPrecision && (
363
+ config.normalizationType === 'layernorm'
364
+ || shouldForceStableF32Logits(config, inputDtype)
365
+ );
323
366
  const stableKernelPath = forceStableF32Logits
324
367
  ? createStableF32LogitsKernelPath(kernelPath)
325
368
  : kernelPath;
@@ -346,11 +389,39 @@ export async function computeLogits(
346
389
  : inputTensor;
347
390
  normInputBufferOwned = normInputTensor !== inputTensor;
348
391
  }
349
- let normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
350
- batchSize: numTokens,
351
- hiddenSize,
352
- rmsNormWeightOffset: config.rmsNormWeightOffset,
353
- });
392
+ let finalNormBiasBuffer = null;
393
+ let normedTensor;
394
+ if (config.normalizationType === 'layernorm') {
395
+ if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
396
+ throw new Error(
397
+ `[Logits] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
398
+ );
399
+ }
400
+ const effectiveBias = finalNormBias instanceof Float32Array
401
+ ? finalNormBias
402
+ : new Float32Array(hiddenSize);
403
+ if (effectiveBias.length !== hiddenSize) {
404
+ throw new Error(
405
+ `[Logits] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveBias.length}.`
406
+ );
407
+ }
408
+ finalNormBiasBuffer = acquireBuffer(effectiveBias.byteLength, undefined, 'final_norm_bias');
409
+ device.queue.writeBuffer(finalNormBiasBuffer, 0, effectiveBias);
410
+ normedTensor = await runLayerNorm(
411
+ normInputTensor,
412
+ normWeightBuffer,
413
+ finalNormBiasBuffer,
414
+ rmsNormEps,
415
+ { batchSize: numTokens, hiddenSize }
416
+ );
417
+ } else {
418
+ normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
419
+ batchSize: numTokens,
420
+ hiddenSize,
421
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
422
+ });
423
+ }
424
+ if (finalNormBiasBuffer) releaseBuffer(finalNormBiasBuffer);
354
425
  if (normInputBufferOwned) {
355
426
  releaseBuffer(normInputTensor.buffer);
356
427
  }
@@ -386,7 +457,7 @@ export async function computeLogits(
386
457
 
387
458
  // Trace final norm output
388
459
  if (kernelTrace.enabled) {
389
- await traceStep('rmsnorm', 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
460
+ await traceStep(config.normalizationType, 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
390
461
  }
391
462
 
392
463
  // Debug: Check hidden state after final norm
@@ -461,6 +532,7 @@ export async function computeLogits(
461
532
  if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
462
533
  if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
463
534
 
535
+ applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
464
536
  return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
465
537
  }
466
538
 
@@ -533,6 +605,7 @@ export async function computeLogits(
533
605
  if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
534
606
  });
535
607
  const selectedLogits = new Float32Array(selectedData);
608
+ applyLmHeadBias(selectedLogits, lmHeadBias, 1, selectedTokenIds.length, selectedTokenIds);
536
609
  await runProbes('logits_final', selectedLogits, {
537
610
  numTokens: 1,
538
611
  hiddenSize: selectedTokenIds.length,
@@ -579,6 +652,7 @@ export async function computeLogits(
579
652
  const rawLogits = logitsTensor.dtype === 'f16'
580
653
  ? f16BufferToF32(logitsData)
581
654
  : new Float32Array(logitsData);
655
+ applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
582
656
  if (isTraceEnabled('logits')) {
583
657
  trace.logits('LM_HEAD_RAW_LOGITS_HEALTH', getLogitsHealth(rawLogits));
584
658
  }
@@ -17,6 +17,8 @@ import type { WeightBuffer, CpuWeightBuffer, SplitWeightBuffer } from '../../../
17
17
  export interface LogitsConfig {
18
18
  hiddenSize: number;
19
19
  vocabSize: number;
20
+ normalizationType: 'rmsnorm' | 'layernorm';
21
+ finalNormBiasTensor: string | null;
20
22
  rmsNormEps: number;
21
23
  useTiedEmbeddings: boolean;
22
24
  embeddingVocabSize: number | null;
@@ -38,7 +40,9 @@ export interface LogitsConfig {
38
40
  */
39
41
  export interface LogitsWeights {
40
42
  finalNorm: GPUBuffer | Float32Array;
43
+ finalNormBias?: Float32Array | null;
41
44
  lmHead: GPUBuffer | Float32Array | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer;
45
+ lmHeadBias?: Float32Array | null;
42
46
  }
43
47
 
44
48
  /**
@@ -2,7 +2,9 @@
2
2
 
3
3
  import {
4
4
  runRMSNorm, runResidualAdd, runMatmul, runSiLU, runGeLU,
5
+ runLayerNorm, runBiasAdd,
5
6
  recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
7
+ recordLayerNorm, recordBiasAdd,
6
8
  runSiLURowSplit, recordSiLURowSplit,
7
9
  runMatmulRMSNormFused, recordMatmulRMSNormFused,
8
10
  runRMSNormStats, recordRMSNormStats,
@@ -60,6 +62,36 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
60
62
  return result;
61
63
  }
62
64
 
65
+ export async function doLayerNorm(input, weight, bias, eps, options, recorder) {
66
+ const result = recorder
67
+ ? await recordLayerNorm(recorder, input, weight, bias, eps, options)
68
+ : await runLayerNorm(input, weight, bias, eps, options);
69
+
70
+ if (kernelTrace.enabled && !recorder) {
71
+ const layer = options.layerIdx ?? -1;
72
+ const label = options.label ?? 'layernorm';
73
+ await traceStep('layernorm', label, layer, result.buffer, [options.batchSize, options.hiddenSize]);
74
+ }
75
+ return result;
76
+ }
77
+
78
+ export async function doBiasAdd(data, bias, numTokens, dim, options, recorder) {
79
+ const result = recorder
80
+ ? await recordBiasAdd(recorder, data, bias, numTokens, dim, options)
81
+ : await runBiasAdd(data, bias, numTokens, dim, options);
82
+
83
+ if (kernelTrace.enabled && !recorder) {
84
+ await traceStep(
85
+ 'bias_add',
86
+ options?.label ?? 'bias_add',
87
+ options?.layerIdx ?? -1,
88
+ result.buffer,
89
+ [numTokens, dim]
90
+ );
91
+ }
92
+ return result;
93
+ }
94
+
63
95
  export async function doRMSNormStats(input, residual, eps, options, recorder) {
64
96
  return recorder
65
97
  ? await recordRMSNormStats(recorder, input, residual, eps, options)
@@ -390,6 +390,12 @@ export interface GenerateOptions {
390
390
  offset: number;
391
391
  length: number;
392
392
  } | null;
393
+
394
+ /** @internal Return final-normalized embeddings for every input token. */
395
+ __returnTokenEmbeddings?: boolean;
396
+
397
+ /** @internal Return full per-token vocabulary logits. */
398
+ __returnSequenceLogits?: boolean;
393
399
  }
394
400
 
395
401
  /**
@@ -438,6 +444,30 @@ export interface PrefillEmbeddingResult extends KVCacheSnapshot {
438
444
  /** Pooling mode used to construct embedding */
439
445
  embeddingMode: 'last' | 'mean';
440
446
 
447
+ /** Final-normalized token matrix when explicitly requested. */
448
+ tokenEmbeddings?: Float32Array | null;
449
+
450
+ /** Full token-by-vocabulary logits when explicitly requested. */
451
+ logits?: Float32Array | null;
452
+
453
+ phase?: WorkloadPhaseTiming | null;
454
+ }
455
+
456
+ export interface SequenceEncodeOptions extends GenerateOptions {
457
+ includeTokenEmbeddings?: boolean;
458
+ includeLogits?: boolean;
459
+ }
460
+
461
+ export interface SequenceEncodeResult {
462
+ alphabet: 'amino_acid' | 'nucleotide';
463
+ tokens: number[];
464
+ tokenMask: Uint8Array;
465
+ includedTokenCount: number;
466
+ tokenEmbeddings: Float32Array | null;
467
+ pooledEmbedding: Float32Array | null;
468
+ logits: Float32Array | null;
469
+ embeddingDim: number;
470
+ vocabSize: number;
441
471
  phase?: WorkloadPhaseTiming | null;
442
472
  }
443
473
 
@@ -495,16 +525,23 @@ export type LayerWeightBuffer = GPUBuffer | WeightBuffer | Float32Array | CpuWei
495
525
  export interface LayerWeights {
496
526
  // Attention
497
527
  inputNorm: GPUBuffer | Float32Array;
528
+ inputNormBias?: GPUBuffer | Float32Array | null;
498
529
  qProj: LayerWeightBuffer;
530
+ qProjBias?: GPUBuffer | Float32Array | null;
499
531
  kProj: LayerWeightBuffer;
532
+ kProjBias?: GPUBuffer | Float32Array | null;
500
533
  vProj?: LayerWeightBuffer;
534
+ vProjBias?: GPUBuffer | Float32Array | null;
501
535
  oProj: LayerWeightBuffer;
536
+ oProjBias?: GPUBuffer | Float32Array | null;
502
537
  qGateProj?: LayerWeightBuffer | null;
503
538
  convInProj?: LayerWeightBuffer;
504
539
  convKernel?: LayerWeightBuffer;
505
540
  convOutProj?: LayerWeightBuffer;
506
541
  /** Fused Q/K/V projection (runtime-generated for 3->1 matmul optimization) */
507
542
  qkvProj?: GPUBuffer | WeightBuffer | null;
543
+ /** Concatenated Q/K/V projection bias matching qkvSizes. */
544
+ qkvProjBias?: GPUBuffer | Float32Array | null;
508
545
  /** Sizes for splitting fused QKV output: [qSize, kSize, vSize] in elements */
509
546
  qkvSizes?: [number, number, number];
510
547
  /** Data type of fused QKV weights (f16, f32, or q4k) */
@@ -516,6 +553,7 @@ export interface LayerWeights {
516
553
 
517
554
  // FFN (dense layers)
518
555
  postAttentionNorm?: GPUBuffer | Float32Array;
556
+ postAttentionNormBias?: GPUBuffer | Float32Array | null;
519
557
  postAttnNorm?: GPUBuffer | Float32Array; // LLaMA-style pre-FFN norm
520
558
  gate?: LayerWeightBuffer;
521
559
  up?: LayerWeightBuffer;
@@ -12,6 +12,7 @@
12
12
 
13
13
  import type { LayerWeights } from './types.js';
14
14
  import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
15
+ import type { Tensor } from '../../../gpu/tensor.js';
15
16
 
16
17
  // ============================================================================
17
18
  // Types
@@ -60,3 +61,12 @@ export function getNormWeightBuffer(
60
61
  debugFlags?: WeightDebugFlags,
61
62
  deviceOverride?: GPUDevice | null
62
63
  ): GPUBuffer;
64
+
65
+ export function getVectorTensor(
66
+ weight: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | { buffer: ArrayBuffer; byteOffset: number; byteLength: number } | CpuWeightBuffer,
67
+ label: string,
68
+ length: number,
69
+ config: WeightBufferConfig,
70
+ debugFlags?: WeightDebugFlags,
71
+ deviceOverride?: GPUDevice | null
72
+ ): { tensor: Tensor; owned: boolean };
@@ -3,7 +3,14 @@
3
3
  import { getDevice } from '../../../gpu/device.js';
4
4
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
5
5
  import { log } from '../../../debug/index.js';
6
- import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, tagBufferDtype } from '../../../gpu/weight-buffer.js';
6
+ import {
7
+ getBufferDtype,
8
+ isWeightBuffer,
9
+ isCpuWeightBuffer,
10
+ isGpuBufferInstance,
11
+ tagBufferDtype,
12
+ } from '../../../gpu/weight-buffer.js';
13
+ import { createTensor } from '../../../gpu/tensor.js';
7
14
 
8
15
  // ============================================================================
9
16
  // Type Guards
@@ -116,6 +123,31 @@ export function getNormWeightBuffer(weight, label, config, debugFlags, deviceOve
116
123
  }
117
124
  }
118
125
 
126
+ export function getVectorTensor(weight, label, length, config, debugFlags, deviceOverride = null) {
127
+ if (!Number.isInteger(length) || length <= 0) {
128
+ throw new Error(`Vector tensor "${label}" requires a positive integer length.`);
129
+ }
130
+ const buffer = getNormWeightBuffer(weight, label, config, debugFlags, deviceOverride);
131
+ let dtype = getBufferDtype(buffer);
132
+ if (dtype == null) {
133
+ const bytesPerElement = buffer.size / length;
134
+ if (bytesPerElement === 2) dtype = 'f16';
135
+ else if (bytesPerElement === 4) dtype = 'f32';
136
+ else {
137
+ throw new Error(
138
+ `Vector tensor "${label}" cannot infer dtype from ${buffer.size} bytes for ${length} elements.`
139
+ );
140
+ }
141
+ }
142
+ if (dtype !== 'f16' && dtype !== 'f32') {
143
+ throw new Error(`Vector tensor "${label}" has unsupported dtype "${String(dtype)}".`);
144
+ }
145
+ return {
146
+ tensor: createTensor(buffer, dtype, [length], label),
147
+ owned: !isGpuBufferInstance(weight) && !isWeightBuffer(weight),
148
+ };
149
+ }
150
+
119
151
 
120
152
  function getGPUWeightBuffer(weight, label) {
121
153
  // Handle WeightBuffer by extracting underlying GPUBuffer
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
15
15
  import { PipelineGenerator } from './text/generator.js';
16
16
  import type { Manifest } from './text/config.js';
17
17
  import type { WeightLoadResult, PipelineContexts } from './text/init.js';
18
- import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
18
+ import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
19
19
  import type { ChatMessage } from './text/chat-format.js';
20
20
  import type { LoRAAdapter } from './text/lora.js';
21
21
  import type { DiffusionPipeline } from './diffusion/pipeline.js';
@@ -31,7 +31,7 @@ import type {
31
31
  } from './text/generator.js';
32
32
 
33
33
  // Re-export types for external use
34
- export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
34
+ export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
35
35
  export type { PipelineContexts };
36
36
 
37
37
  export interface ChatRequestInput {
@@ -130,6 +130,8 @@ export declare class InferencePipeline extends PipelineState {
130
130
  embeddingMode: string;
131
131
  }>>;
132
132
 
133
+ encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise<SequenceEncodeResult>;
134
+
133
135
  prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
134
136
 
135
137
  prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{