doppler-gpu 0.4.9 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/README.md +89 -12
  3. package/models/catalog.json +612 -7
  4. package/package.json +30 -1
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +126 -4
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +64 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/kernels/registry.json +52 -9
  18. package/src/config/merge.js +24 -0
  19. package/src/config/required-inference-fields-contract-check.js +2 -0
  20. package/src/config/schema/converter.schema.d.ts +7 -0
  21. package/src/config/schema/converter.schema.js +15 -0
  22. package/src/config/schema/inference.schema.d.ts +8 -0
  23. package/src/config/schema/manifest.schema.d.ts +32 -1
  24. package/src/config/schema/manifest.schema.js +26 -0
  25. package/src/converter/conversion-plan.js +15 -1
  26. package/src/converter/core.js +5 -4
  27. package/src/converter/source-tensor-rules.d.ts +50 -0
  28. package/src/converter/source-tensor-rules.js +180 -0
  29. package/src/converter/source-tokenizer.d.ts +45 -0
  30. package/src/converter/source-tokenizer.js +155 -0
  31. package/src/experimental/adapters/lora-loader.js +15 -0
  32. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  33. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  34. package/src/gpu/device.js +100 -32
  35. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  36. package/src/inference/pipelines/text/attention/projections.js +161 -24
  37. package/src/inference/pipelines/text/attention/record.js +67 -9
  38. package/src/inference/pipelines/text/attention/run.js +60 -7
  39. package/src/inference/pipelines/text/attention/types.js +18 -9
  40. package/src/inference/pipelines/text/config.d.ts +5 -0
  41. package/src/inference/pipelines/text/config.js +91 -0
  42. package/src/inference/pipelines/text/ffn/dense.js +189 -2
  43. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  44. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  45. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  46. package/src/inference/pipelines/text/generator-steps.js +3 -1
  47. package/src/inference/pipelines/text/generator.js +90 -3
  48. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  49. package/src/inference/pipelines/text/init.d.ts +2 -0
  50. package/src/inference/pipelines/text/init.js +2 -0
  51. package/src/inference/pipelines/text/layer.js +6 -0
  52. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  53. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  54. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  55. package/src/inference/pipelines/text/logits/index.js +85 -11
  56. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  57. package/src/inference/pipelines/text/ops.js +32 -0
  58. package/src/inference/pipelines/text/types.d.ts +38 -0
  59. package/src/inference/pipelines/text/weights.d.ts +10 -0
  60. package/src/inference/pipelines/text/weights.js +33 -1
  61. package/src/inference/pipelines/text.d.ts +4 -2
  62. package/src/inference/pipelines/text.js +112 -0
  63. package/src/inference/tokenizers/behavior-flags.js +5 -1
  64. package/src/inference/tokenizers/bundled.js +99 -8
  65. package/src/loader/doppler-loader.d.ts +2 -0
  66. package/src/loader/doppler-loader.js +9 -0
  67. package/src/loader/final-weights-loader.d.ts +8 -0
  68. package/src/loader/final-weights-loader.js +42 -0
  69. package/src/loader/layer-loader.d.ts +6 -0
  70. package/src/loader/layer-loader.js +102 -3
  71. package/src/loader/loader-types.d.ts +15 -2
  72. package/src/tooling/node-converter.js +33 -1
  73. package/src/tooling/node-webgpu.js +85 -2
  74. package/src/tooling/runtime-optimization.d.ts +109 -0
  75. package/src/tooling/runtime-optimization.js +872 -0
  76. package/src/tooling-exports.shared.d.ts +19 -0
  77. package/src/tooling-exports.shared.js +11 -0
  78. package/src/version.js +1 -1
@@ -63,6 +63,7 @@ import {
63
63
  resolveAdvanceEmbeddingMode,
64
64
  getFinalNormWeights,
65
65
  extractEmbeddingFromHidden,
66
+ extractTokenEmbeddingsFromHidden,
66
67
  } from './generator-runtime.js';
67
68
 
68
69
  import { resolveSamplingConfig } from './sampling-config.js';
@@ -642,7 +643,20 @@ export class PipelineGenerator {
642
643
  embeddingMode,
643
644
  finalNormWeights,
644
645
  config,
645
- this.#state.embeddingPostprocessor
646
+ this.#state.embeddingPostprocessor,
647
+ null,
648
+ this.#state.weights.get('final_norm_bias') ?? null
649
+ );
650
+ }
651
+
652
+ _extractTokenEmbeddingsFromHidden(hiddenStates, numTokens, hiddenSize, finalNormWeights, config) {
653
+ return extractTokenEmbeddingsFromHidden(
654
+ hiddenStates,
655
+ numTokens,
656
+ hiddenSize,
657
+ finalNormWeights,
658
+ config,
659
+ this.#state.weights.get('final_norm_bias') ?? null
646
660
  );
647
661
  }
648
662
 
@@ -2112,6 +2126,34 @@ export class PipelineGenerator {
2112
2126
  this.#state.stats.gpuTimePrefillMs = undefined;
2113
2127
  this.#state.stats.prefillProfileSteps = [];
2114
2128
  const opts = resolvePrefillEmbeddingOptions(this.#state, options);
2129
+ const diagnosticsEnabled = options?.diagnostics?.enabled === true
2130
+ || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
2131
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
2132
+ if (diagnosticsEnabled || tsirFixtureCfg) {
2133
+ const captureConfig = {
2134
+ ...createDefaultCaptureConfig(),
2135
+ enabled: true,
2136
+ defaultLevel: CAPTURE_LEVELS.SLICE,
2137
+ ...(options?.diagnostics?.captureConfig ?? {}),
2138
+ };
2139
+ validateCaptureConfig(captureConfig);
2140
+ this.#state.operatorDiagnostics = {
2141
+ enabled: diagnosticsEnabled === true,
2142
+ captureConfig,
2143
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
2144
+ modelHash: this.#state.manifest?.modelId ?? null,
2145
+ runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
2146
+ executionPlanHash: opts.executionPlan?.id ?? null,
2147
+ }) : null,
2148
+ tsirFixture: tsirFixtureCfg ? {
2149
+ dir: tsirFixtureCfg.dir,
2150
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
2151
+ ? tsirFixtureCfg.layerFilter
2152
+ : null,
2153
+ records: [],
2154
+ } : null,
2155
+ };
2156
+ }
2115
2157
  const prefillStartSeqLen = this.#state.currentSeqLen;
2116
2158
  const inputStart = performance.now();
2117
2159
  const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
@@ -2178,12 +2220,32 @@ export class PipelineGenerator {
2178
2220
  }
2179
2221
 
2180
2222
  let embedding;
2223
+ let tokenEmbeddings = null;
2224
+ let logits = null;
2181
2225
  let hiddenBytes = 0;
2182
2226
  let readbackMs = 0;
2183
2227
  let decodeHiddenMs = 0;
2184
2228
  let finalNormMs = 0;
2185
2229
  let extractMs = 0;
2230
+ let logitsMs = 0;
2186
2231
  try {
2232
+ if (options.__returnSequenceLogits === true) {
2233
+ const logitsStart = performance.now();
2234
+ logits = await computeLogits(
2235
+ currentHiddenBuffer,
2236
+ numTokens,
2237
+ getLogitsWeights(this.#state),
2238
+ getLogitsConfig(this.#state),
2239
+ this.#state.useGPU,
2240
+ this.#state.debugFlags,
2241
+ undefined,
2242
+ undefined,
2243
+ this.#state.runtimeConfig.shared.debug.probes,
2244
+ { lastPositionOnly: false },
2245
+ this.#state.operatorDiagnostics
2246
+ );
2247
+ logitsMs = performance.now() - logitsStart;
2248
+ }
2187
2249
  const hiddenSize = config.hiddenSize;
2188
2250
  hiddenBytes = numTokens * hiddenSize * activationBytes;
2189
2251
  const readbackStart = performance.now();
@@ -2199,13 +2261,25 @@ export class PipelineGenerator {
2199
2261
  const finalNormWeights = await this._getFinalNormWeights();
2200
2262
  finalNormMs = performance.now() - finalNormStart;
2201
2263
  const extractStart = performance.now();
2202
- embedding = this._extractEmbeddingFromHidden(
2264
+ if (options.__returnTokenEmbeddings === true) {
2265
+ tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
2266
+ hiddenStates,
2267
+ numTokens,
2268
+ hiddenSize,
2269
+ finalNormWeights,
2270
+ config
2271
+ );
2272
+ }
2273
+ embedding = extractEmbeddingFromHidden(
2203
2274
  hiddenStates,
2204
2275
  numTokens,
2205
2276
  hiddenSize,
2206
2277
  opts.embeddingMode,
2207
2278
  finalNormWeights,
2208
- config
2279
+ config,
2280
+ this.#state.embeddingPostprocessor,
2281
+ tokenEmbeddings,
2282
+ this.#state.weights.get('final_norm_bias') ?? null
2209
2283
  );
2210
2284
  extractMs = performance.now() - extractStart;
2211
2285
  } finally {
@@ -2222,6 +2296,7 @@ export class PipelineGenerator {
2222
2296
  decodeHiddenMs,
2223
2297
  finalNormMs,
2224
2298
  extractMs,
2299
+ logitsMs,
2225
2300
  hiddenBytes,
2226
2301
  tokens: numTokens,
2227
2302
  activationDtype,
@@ -2241,6 +2316,8 @@ export class PipelineGenerator {
2241
2316
  seqLen: this.#state.currentSeqLen,
2242
2317
  tokens: inputIds,
2243
2318
  embedding,
2319
+ tokenEmbeddings,
2320
+ logits,
2244
2321
  embeddingMode: opts.embeddingMode,
2245
2322
  phase,
2246
2323
  linearAttention: null,
@@ -2257,12 +2334,22 @@ export class PipelineGenerator {
2257
2334
  seqLen: this.#state.currentSeqLen,
2258
2335
  tokens: inputIds,
2259
2336
  embedding,
2337
+ tokenEmbeddings,
2338
+ logits,
2260
2339
  embeddingMode: opts.embeddingMode,
2261
2340
  phase,
2262
2341
  linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2263
2342
  };
2264
2343
  } finally {
2265
2344
  this._closeFinitenessFallbackWindow(opts);
2345
+ this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2346
+ ? {
2347
+ enabled: true,
2348
+ timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2349
+ recordCount: this.#state.operatorDiagnostics.emitter.length,
2350
+ }
2351
+ : null;
2352
+ this.#state.operatorDiagnostics = null;
2266
2353
  }
2267
2354
  }
2268
2355
 
@@ -2,7 +2,13 @@ import { getDevice } from '../../../gpu/device.js';
2
2
  import { Q4K_BLOCK_BYTES, q4kBlockCount } from '../../../config/schema/index.js';
3
3
  import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
4
4
  import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
5
- import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
5
+ import {
6
+ createWeightBuffer,
7
+ getWeightDtype,
8
+ isGpuBufferInstance,
9
+ isWeightBuffer,
10
+ tagBufferDtype,
11
+ } from '../../../gpu/weight-buffer.js';
6
12
  import { selectRuleValue } from '../../../rules/rule-registry.js';
7
13
  import { log } from '../../../debug/index.js';
8
14
 
@@ -112,6 +118,26 @@ function formatsMatch(a, b) {
112
118
  && a?.bytesPerRow === b?.bytesPerRow;
113
119
  }
114
120
 
121
+ function resolveBiasStorageFormat(weight, expectedElements) {
122
+ if (!weight || !Number.isFinite(expectedElements) || expectedElements <= 0) {
123
+ return null;
124
+ }
125
+ const buffer = isWeightBuffer(weight) ? weight.buffer : weight;
126
+ if (!isGpuBufferInstance(buffer)) {
127
+ return null;
128
+ }
129
+ const dtype = String(getWeightDtype(weight) ?? getWeightDtype(buffer) ?? '').toLowerCase();
130
+ if (dtype !== 'f16' && dtype !== 'f32') {
131
+ return null;
132
+ }
133
+ const bytesPerElement = dtype === 'f16' ? 2 : 4;
134
+ const byteLength = expectedElements * bytesPerElement;
135
+ if (buffer.size < byteLength) {
136
+ return null;
137
+ }
138
+ return { buffer, dtype, bytesPerElement, byteLength };
139
+ }
140
+
115
141
  export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, options = {}) {
116
142
  const device = getDevice();
117
143
  if (!device) {
@@ -174,6 +200,17 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
174
200
  continue;
175
201
  }
176
202
 
203
+ const projectionBiases = [weights.qProjBias, weights.kProjBias, weights.vProjBias];
204
+ const presentProjectionBiases = projectionBiases.filter(Boolean).length;
205
+ if (presentProjectionBiases !== 0 && presentProjectionBiases !== projectionBiases.length) {
206
+ log.debug('QKV Fusion', `Layer ${l}: partial Q/K/V projection biases require split projections`);
207
+ continue;
208
+ }
209
+ if (hasAttentionOutputGate && presentProjectionBiases > 0) {
210
+ log.debug('QKV Fusion', `Layer ${l}: gated Q projection biases require split projections`);
211
+ continue;
212
+ }
213
+
177
214
  const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
178
215
  const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
179
216
  const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
@@ -228,6 +265,32 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
228
265
  })
229
266
  : null;
230
267
 
268
+ let qBiasFormat = null;
269
+ let kBiasFormat = null;
270
+ let vBiasFormat = null;
271
+ let qkvBiasBuffer = null;
272
+ if (presentProjectionBiases === projectionBiases.length) {
273
+ qBiasFormat = resolveBiasStorageFormat(weights.qProjBias, qSize);
274
+ kBiasFormat = resolveBiasStorageFormat(weights.kProjBias, kSize);
275
+ vBiasFormat = resolveBiasStorageFormat(weights.vProjBias, vSize);
276
+ const biasFormatsMatch = qBiasFormat
277
+ && kBiasFormat
278
+ && vBiasFormat
279
+ && qBiasFormat.dtype === kBiasFormat.dtype
280
+ && qBiasFormat.dtype === vBiasFormat.dtype;
281
+ if (!biasFormatsMatch) {
282
+ log.debug('QKV Fusion', `Layer ${l}: inconsistent Q/K/V projection bias formats, skipping`);
283
+ qkvBuffer.destroy();
284
+ qGateBuffer?.destroy();
285
+ continue;
286
+ }
287
+ qkvBiasBuffer = device.createBuffer({
288
+ label: `layer_${l}_qkv_proj_bias`,
289
+ size: qBiasFormat.byteLength + kBiasFormat.byteLength + vBiasFormat.byteLength,
290
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
291
+ });
292
+ }
293
+
231
294
  const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
232
295
  if (hasAttentionOutputGate) {
233
296
  for (let head = 0; head < numHeads; head++) {
@@ -266,6 +329,23 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
266
329
  qkvBuffer, qBytes + kFormat.byteLength,
267
330
  vFormat.byteLength
268
331
  );
332
+ if (qkvBiasBuffer) {
333
+ encoder.copyBufferToBuffer(qBiasFormat.buffer, 0, qkvBiasBuffer, 0, qBiasFormat.byteLength);
334
+ encoder.copyBufferToBuffer(
335
+ kBiasFormat.buffer,
336
+ 0,
337
+ qkvBiasBuffer,
338
+ qBiasFormat.byteLength,
339
+ kBiasFormat.byteLength
340
+ );
341
+ encoder.copyBufferToBuffer(
342
+ vBiasFormat.buffer,
343
+ 0,
344
+ qkvBiasBuffer,
345
+ qBiasFormat.byteLength + kBiasFormat.byteLength,
346
+ vBiasFormat.byteLength
347
+ );
348
+ }
269
349
  device.queue.submit([encoder.finish()]);
270
350
 
271
351
  // Store fused buffer, sizes, and dtype
@@ -278,6 +358,10 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
278
358
  );
279
359
  weights.qkvSizes = [qSize, kSize, vSize];
280
360
  weights.qkvDtype = dtype;
361
+ if (qkvBiasBuffer) {
362
+ tagBufferDtype(qkvBiasBuffer, qBiasFormat.dtype);
363
+ weights.qkvProjBias = qkvBiasBuffer;
364
+ }
281
365
  if (qGateBuffer) {
282
366
  weights.qGateProj = createWeightBuffer(
283
367
  qGateBuffer,
@@ -237,7 +237,9 @@ export interface WeightLoadResult {
237
237
  layerWeights: Map<string, LayerWeights>;
238
238
  embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
239
239
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
240
+ lmHeadBias: Float32Array | null;
240
241
  finalNorm: GPUBuffer | Float32Array | null;
242
+ finalNormBias: Float32Array | null;
241
243
  embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
242
244
  diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
243
245
  perLayerInputWeights: PerLayerInputWeights | null;
@@ -1020,7 +1020,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1020
1020
  layerWeights,
1021
1021
  embeddings: dopplerLoader.embeddings,
1022
1022
  lmHead: dopplerLoader.lmHead,
1023
+ lmHeadBias: dopplerLoader.lmHeadBias,
1023
1024
  finalNorm: dopplerLoader.finalNorm,
1025
+ finalNormBias: dopplerLoader.finalNormBias,
1024
1026
  embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
1025
1027
  diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
1026
1028
  perLayerInputWeights: dopplerLoader.perLayerInputWeights,
@@ -57,6 +57,7 @@ function shouldUseSandwichRMSNormPairFusion({
57
57
  attnOutput,
58
58
  inputTensor,
59
59
  }) {
60
+ if (context.config?.normalizationType === 'layernorm') return false;
60
61
  const mergedSession = getRuntimeConfig()?.inference?.session;
61
62
  if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
62
63
  return false;
@@ -123,6 +124,7 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
123
124
  activationDtype,
124
125
  layerScalar,
125
126
  }) {
127
+ if (config.normalizationType === 'layernorm') return false;
126
128
  const mergedSession = getRuntimeConfig()?.inference?.session;
127
129
  if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
128
130
  return false;
@@ -176,6 +178,7 @@ function shouldUseStandardPostFfnNextInputRMSNormPairFusion({
176
178
  layerScalar,
177
179
  residualBranchScale,
178
180
  }) {
181
+ if (config.normalizationType === 'layernorm') return false;
179
182
  const mergedSession = getRuntimeConfig()?.inference?.session;
180
183
  if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
181
184
  return false;
@@ -851,6 +854,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
851
854
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
852
855
  queryPreAttnScalar: config.queryPreAttnScalar,
853
856
  queryKeyNorm,
857
+ queryKeyNormType: config.queryKeyNormType,
858
+ queryKeyNormAxis: config.queryKeyNormAxis,
854
859
  queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
855
860
  valueNorm: config.valueNorm,
856
861
  attentionOutputGate: config.attentionOutputGate,
@@ -860,6 +865,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
860
865
  ? (context.multimodalBidirectionalSpan ?? null)
861
866
  : null,
862
867
  rmsNormWeightOffset: config.rmsNormWeightOffset,
868
+ normalizationType: config.normalizationType,
863
869
  ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
864
870
  ropeFrequencyBaseDim: resolveAttentionFrequencyBaseDim(config, layerType),
865
871
  ropeInterleaved: config.ropeInterleaved,
@@ -25,6 +25,12 @@ export function rmsNormCPU(
25
25
  eps?: number,
26
26
  rmsNormWeightOffset?: boolean
27
27
  ): Float32Array;
28
+ export function layerNormCPU(
29
+ x: Float32Array,
30
+ weight: Float32Array,
31
+ bias: Float32Array,
32
+ eps?: number
33
+ ): Float32Array;
28
34
 
29
35
  /**
30
36
  * Convert a single float16 value to float32.
@@ -25,6 +25,28 @@ export function rmsNormCPU(
25
25
  return result;
26
26
  }
27
27
 
28
+ export function layerNormCPU(x, weight, bias, eps = 1e-5) {
29
+ if (weight.length !== x.length || bias.length !== x.length) {
30
+ throw new Error(
31
+ `LayerNorm parameter length mismatch: input=${x.length}, weight=${weight.length}, bias=${bias.length}.`
32
+ );
33
+ }
34
+ let sum = 0;
35
+ for (let i = 0; i < x.length; i++) sum += x[i];
36
+ const mean = sum / x.length;
37
+ let squaredDeviation = 0;
38
+ for (let i = 0; i < x.length; i++) {
39
+ const centered = x[i] - mean;
40
+ squaredDeviation += centered * centered;
41
+ }
42
+ const invStd = 1 / Math.sqrt(squaredDeviation / x.length + eps);
43
+ const result = new Float32Array(x.length);
44
+ for (let i = 0; i < x.length; i++) {
45
+ result[i] = (x[i] - mean) * invStd * weight[i] + bias[i];
46
+ }
47
+ return result;
48
+ }
49
+
28
50
 
29
51
  export function f16BufferToF32(data) {
30
52
  const u16 = new Uint16Array(data);
@@ -17,7 +17,7 @@ import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
17
17
  export type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
18
18
 
19
19
  // Re-export CPU functions
20
- export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
20
+ export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
21
21
 
22
22
  // Re-export GPU functions
23
23
  export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
@@ -1,7 +1,7 @@
1
1
 
2
2
 
3
3
  // Re-export CPU functions
4
- export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
4
+ export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
5
5
 
6
6
  // Re-export GPU functions
7
7
  export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
12
12
  // Imports for computeLogits orchestrator
13
13
  import { getDevice } from '../../../../gpu/device.js';
14
14
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
15
- import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
15
+ import { runLayerNorm, runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
16
16
  import { createTensor } from '../../../../gpu/tensor.js';
17
17
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
18
18
  import { kernelTrace, traceStep } from '../kernel-trace.js';
@@ -72,6 +72,45 @@ function normalizeSelectedLogitTokenIds(value, vocabSize) {
72
72
  return tokenIds;
73
73
  }
74
74
 
75
+ function applyLmHeadBias(logits, bias, rowCount, rowSize, selectedTokenIds = null) {
76
+ if (bias == null) {
77
+ return logits;
78
+ }
79
+ if (!(bias instanceof Float32Array)) {
80
+ throw new Error('[Logits] LM head bias must be a Float32Array.');
81
+ }
82
+ if (selectedTokenIds !== null) {
83
+ if (logits.length !== selectedTokenIds.length) {
84
+ throw new Error('[Logits] Selected-logit bias application length mismatch.');
85
+ }
86
+ for (let index = 0; index < selectedTokenIds.length; index += 1) {
87
+ const tokenId = selectedTokenIds[index];
88
+ if (tokenId >= bias.length) {
89
+ throw new Error(`[Logits] LM head bias is missing token ${tokenId}.`);
90
+ }
91
+ logits[index] += bias[tokenId];
92
+ }
93
+ return logits;
94
+ }
95
+ if (bias.length < rowSize) {
96
+ throw new Error(
97
+ `[Logits] LM head bias length mismatch: expected at least ${rowSize}, got ${bias.length}.`
98
+ );
99
+ }
100
+ if (logits.length !== rowCount * rowSize) {
101
+ throw new Error(
102
+ `[Logits] LM head bias matrix mismatch: expected ${rowCount * rowSize} logits, got ${logits.length}.`
103
+ );
104
+ }
105
+ for (let row = 0; row < rowCount; row += 1) {
106
+ const offset = row * rowSize;
107
+ for (let column = 0; column < rowSize; column += 1) {
108
+ logits[offset + column] += bias[column];
109
+ }
110
+ }
111
+ return logits;
112
+ }
113
+
75
114
  async function coerceTensorDtype(tensor, targetDtype, options = {}) {
76
115
  if (!targetDtype || tensor.dtype === targetDtype) {
77
116
  return tensor;
@@ -164,7 +203,7 @@ export async function computeLogits(
164
203
  activationDtype: activationDtypeOverride,
165
204
  } = config;
166
205
  const activationDtype = activationDtypeOverride ?? getRuntimeConfig().inference.compute.activationDtype;
167
- const { finalNorm, lmHead } = weights;
206
+ const { finalNorm, finalNormBias = null, lmHead, lmHeadBias = null } = weights;
168
207
  const device = getDevice();
169
208
 
170
209
  // Consistency check: warn if lmHead weight dtype disagrees with layer activation dtype.
@@ -269,6 +308,7 @@ export async function computeLogits(
269
308
  cpuWeightLayout === 'column' ? cpuWeightVocabSize : null
270
309
  )
271
310
  : matmulCPU(normed, (lmHead), numTokens, matmulVocabSize, hiddenSize);
311
+ applyLmHeadBias(rawLogits, lmHeadBias, numTokens, matmulVocabSize);
272
312
  return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
273
313
  }
274
314
 
@@ -293,7 +333,7 @@ export async function computeLogits(
293
333
  dtype: inputDtype,
294
334
  });
295
335
 
296
- // 2. Apply final RMSNorm
336
+ // 2. Apply the manifest-owned final normalization.
297
337
 
298
338
  let normWeightBuffer;
299
339
  let normWeightBufferOwned = false;
@@ -319,7 +359,10 @@ export async function computeLogits(
319
359
  const kernelPath = config.kernelPath ?? null;
320
360
  const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
321
361
  const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
322
- const forceStableF32Logits = !hasExplicitFinalNormPrecision && shouldForceStableF32Logits(config, inputDtype);
362
+ const forceStableF32Logits = !hasExplicitFinalNormPrecision && (
363
+ config.normalizationType === 'layernorm'
364
+ || shouldForceStableF32Logits(config, inputDtype)
365
+ );
323
366
  const stableKernelPath = forceStableF32Logits
324
367
  ? createStableF32LogitsKernelPath(kernelPath)
325
368
  : kernelPath;
@@ -346,11 +389,39 @@ export async function computeLogits(
346
389
  : inputTensor;
347
390
  normInputBufferOwned = normInputTensor !== inputTensor;
348
391
  }
349
- let normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
350
- batchSize: numTokens,
351
- hiddenSize,
352
- rmsNormWeightOffset: config.rmsNormWeightOffset,
353
- });
392
+ let finalNormBiasBuffer = null;
393
+ let normedTensor;
394
+ if (config.normalizationType === 'layernorm') {
395
+ if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
396
+ throw new Error(
397
+ `[Logits] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
398
+ );
399
+ }
400
+ const effectiveBias = finalNormBias instanceof Float32Array
401
+ ? finalNormBias
402
+ : new Float32Array(hiddenSize);
403
+ if (effectiveBias.length !== hiddenSize) {
404
+ throw new Error(
405
+ `[Logits] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveBias.length}.`
406
+ );
407
+ }
408
+ finalNormBiasBuffer = acquireBuffer(effectiveBias.byteLength, undefined, 'final_norm_bias');
409
+ device.queue.writeBuffer(finalNormBiasBuffer, 0, effectiveBias);
410
+ normedTensor = await runLayerNorm(
411
+ normInputTensor,
412
+ normWeightBuffer,
413
+ finalNormBiasBuffer,
414
+ rmsNormEps,
415
+ { batchSize: numTokens, hiddenSize }
416
+ );
417
+ } else {
418
+ normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
419
+ batchSize: numTokens,
420
+ hiddenSize,
421
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
422
+ });
423
+ }
424
+ if (finalNormBiasBuffer) releaseBuffer(finalNormBiasBuffer);
354
425
  if (normInputBufferOwned) {
355
426
  releaseBuffer(normInputTensor.buffer);
356
427
  }
@@ -386,7 +457,7 @@ export async function computeLogits(
386
457
 
387
458
  // Trace final norm output
388
459
  if (kernelTrace.enabled) {
389
- await traceStep('rmsnorm', 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
460
+ await traceStep(config.normalizationType, 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
390
461
  }
391
462
 
392
463
  // Debug: Check hidden state after final norm
@@ -461,6 +532,7 @@ export async function computeLogits(
461
532
  if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
462
533
  if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
463
534
 
535
+ applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
464
536
  return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
465
537
  }
466
538
 
@@ -533,6 +605,7 @@ export async function computeLogits(
533
605
  if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
534
606
  });
535
607
  const selectedLogits = new Float32Array(selectedData);
608
+ applyLmHeadBias(selectedLogits, lmHeadBias, 1, selectedTokenIds.length, selectedTokenIds);
536
609
  await runProbes('logits_final', selectedLogits, {
537
610
  numTokens: 1,
538
611
  hiddenSize: selectedTokenIds.length,
@@ -579,6 +652,7 @@ export async function computeLogits(
579
652
  const rawLogits = logitsTensor.dtype === 'f16'
580
653
  ? f16BufferToF32(logitsData)
581
654
  : new Float32Array(logitsData);
655
+ applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
582
656
  if (isTraceEnabled('logits')) {
583
657
  trace.logits('LM_HEAD_RAW_LOGITS_HEALTH', getLogitsHealth(rawLogits));
584
658
  }
@@ -17,6 +17,8 @@ import type { WeightBuffer, CpuWeightBuffer, SplitWeightBuffer } from '../../../
17
17
  export interface LogitsConfig {
18
18
  hiddenSize: number;
19
19
  vocabSize: number;
20
+ normalizationType: 'rmsnorm' | 'layernorm';
21
+ finalNormBiasTensor: string | null;
20
22
  rmsNormEps: number;
21
23
  useTiedEmbeddings: boolean;
22
24
  embeddingVocabSize: number | null;
@@ -38,7 +40,9 @@ export interface LogitsConfig {
38
40
  */
39
41
  export interface LogitsWeights {
40
42
  finalNorm: GPUBuffer | Float32Array;
43
+ finalNormBias?: Float32Array | null;
41
44
  lmHead: GPUBuffer | Float32Array | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer;
45
+ lmHeadBias?: Float32Array | null;
42
46
  }
43
47
 
44
48
  /**
@@ -2,7 +2,9 @@
2
2
 
3
3
  import {
4
4
  runRMSNorm, runResidualAdd, runMatmul, runSiLU, runGeLU,
5
+ runLayerNorm, runBiasAdd,
5
6
  recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
7
+ recordLayerNorm, recordBiasAdd,
6
8
  runSiLURowSplit, recordSiLURowSplit,
7
9
  runMatmulRMSNormFused, recordMatmulRMSNormFused,
8
10
  runRMSNormStats, recordRMSNormStats,
@@ -60,6 +62,36 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
60
62
  return result;
61
63
  }
62
64
 
65
+ export async function doLayerNorm(input, weight, bias, eps, options, recorder) {
66
+ const result = recorder
67
+ ? await recordLayerNorm(recorder, input, weight, bias, eps, options)
68
+ : await runLayerNorm(input, weight, bias, eps, options);
69
+
70
+ if (kernelTrace.enabled && !recorder) {
71
+ const layer = options.layerIdx ?? -1;
72
+ const label = options.label ?? 'layernorm';
73
+ await traceStep('layernorm', label, layer, result.buffer, [options.batchSize, options.hiddenSize]);
74
+ }
75
+ return result;
76
+ }
77
+
78
+ export async function doBiasAdd(data, bias, numTokens, dim, options, recorder) {
79
+ const result = recorder
80
+ ? await recordBiasAdd(recorder, data, bias, numTokens, dim, options)
81
+ : await runBiasAdd(data, bias, numTokens, dim, options);
82
+
83
+ if (kernelTrace.enabled && !recorder) {
84
+ await traceStep(
85
+ 'bias_add',
86
+ options?.label ?? 'bias_add',
87
+ options?.layerIdx ?? -1,
88
+ result.buffer,
89
+ [numTokens, dim]
90
+ );
91
+ }
92
+ return result;
93
+ }
94
+
63
95
  export async function doRMSNormStats(input, residual, eps, options, recorder) {
64
96
  return recorder
65
97
  ? await recordRMSNormStats(recorder, input, residual, eps, options)