doppler-gpu 0.4.10 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +95 -0
  2. package/README.md +40 -1
  3. package/models/catalog.json +637 -32
  4. package/package.json +24 -3
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +135 -13
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +69 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
  18. package/src/config/kernels/kernel-ref-digests.js +2 -1
  19. package/src/config/kernels/registry.json +73 -11
  20. package/src/config/merge.js +24 -0
  21. package/src/config/required-inference-fields-contract-check.js +2 -0
  22. package/src/config/schema/adapter.schema.d.ts +6 -1
  23. package/src/config/schema/adapter.schema.js +5 -0
  24. package/src/config/schema/converter.schema.d.ts +7 -0
  25. package/src/config/schema/converter.schema.js +15 -0
  26. package/src/config/schema/inference.schema.d.ts +8 -0
  27. package/src/config/schema/manifest.schema.d.ts +32 -1
  28. package/src/config/schema/manifest.schema.js +26 -0
  29. package/src/converter/conversion-plan.js +15 -1
  30. package/src/converter/core.js +5 -4
  31. package/src/converter/source-tensor-rules.d.ts +50 -0
  32. package/src/converter/source-tensor-rules.js +180 -0
  33. package/src/converter/source-tokenizer.d.ts +45 -0
  34. package/src/converter/source-tokenizer.js +155 -0
  35. package/src/experimental/adapters/artifact-contract.d.ts +83 -0
  36. package/src/experimental/adapters/artifact-contract.js +254 -0
  37. package/src/experimental/adapters/index.d.ts +25 -0
  38. package/src/experimental/adapters/index.js +12 -0
  39. package/src/experimental/adapters/lora-loader.js +24 -4
  40. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  41. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  42. package/src/experimental/training/artifacts.d.ts +160 -0
  43. package/src/experimental/training/attention-backward.d.ts +64 -0
  44. package/src/experimental/training/attention-forward.d.ts +22 -0
  45. package/src/experimental/training/attention-forward.js +82 -0
  46. package/src/experimental/training/autograd.d.ts +76 -0
  47. package/src/experimental/training/checkpoint.d.ts +36 -0
  48. package/src/experimental/training/clip.d.ts +9 -0
  49. package/src/experimental/training/dataloader.d.ts +8 -0
  50. package/src/experimental/training/distillation/student-fixture.js +1 -0
  51. package/src/experimental/training/export.d.ts +41 -0
  52. package/src/experimental/training/lora-pipeline.d.ts +186 -0
  53. package/src/experimental/training/lora-pipeline.js +5 -0
  54. package/src/experimental/training/lora.d.ts +20 -0
  55. package/src/experimental/training/lora.js +4 -3
  56. package/src/experimental/training/loss-scaling.d.ts +21 -0
  57. package/src/experimental/training/loss.d.ts +10 -0
  58. package/src/experimental/training/native-qwen-lora.d.ts +106 -0
  59. package/src/experimental/training/native-qwen-lora.js +449 -0
  60. package/src/experimental/training/objectives/base.d.ts +58 -0
  61. package/src/experimental/training/optimizer.d.ts +22 -0
  62. package/src/experimental/training/optimizer.js +1 -0
  63. package/src/experimental/training/runner.d.ts +249 -0
  64. package/src/experimental/training/trainer.d.ts +89 -0
  65. package/src/experimental/training/workloads.d.ts +205 -0
  66. package/src/gpu/kernels/backward/adam.d.ts +1 -0
  67. package/src/gpu/kernels/backward/adam.js +6 -2
  68. package/src/gpu/kernels/backward/adam.wgsl +3 -2
  69. package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
  70. package/src/gpu/kernels/backward/utils.js +18 -4
  71. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  72. package/src/inference/pipelines/text/attention/projections.js +161 -24
  73. package/src/inference/pipelines/text/attention/record.js +67 -9
  74. package/src/inference/pipelines/text/attention/run.js +60 -7
  75. package/src/inference/pipelines/text/attention/types.js +18 -9
  76. package/src/inference/pipelines/text/config.d.ts +5 -0
  77. package/src/inference/pipelines/text/config.js +91 -0
  78. package/src/inference/pipelines/text/ffn/dense.js +205 -2
  79. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  80. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  81. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  82. package/src/inference/pipelines/text/generator-steps.js +3 -1
  83. package/src/inference/pipelines/text/generator.d.ts +5 -1
  84. package/src/inference/pipelines/text/generator.js +196 -3
  85. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  86. package/src/inference/pipelines/text/init.d.ts +2 -0
  87. package/src/inference/pipelines/text/init.js +2 -0
  88. package/src/inference/pipelines/text/layer.js +6 -0
  89. package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
  90. package/src/inference/pipelines/text/linear-attention.js +77 -6
  91. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  92. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  93. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  94. package/src/inference/pipelines/text/logits/index.js +85 -11
  95. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  96. package/src/inference/pipelines/text/lora-types.d.ts +6 -1
  97. package/src/inference/pipelines/text/lora-types.js +5 -0
  98. package/src/inference/pipelines/text/ops.js +33 -0
  99. package/src/inference/pipelines/text/types.d.ts +66 -0
  100. package/src/inference/pipelines/text/weights.d.ts +10 -0
  101. package/src/inference/pipelines/text/weights.js +33 -1
  102. package/src/inference/pipelines/text.d.ts +9 -2
  103. package/src/inference/pipelines/text.js +116 -0
  104. package/src/inference/tokenizers/behavior-flags.js +5 -1
  105. package/src/inference/tokenizers/bundled.js +99 -8
  106. package/src/loader/doppler-loader.d.ts +2 -0
  107. package/src/loader/doppler-loader.js +9 -0
  108. package/src/loader/final-weights-loader.d.ts +8 -0
  109. package/src/loader/final-weights-loader.js +42 -0
  110. package/src/loader/layer-loader.d.ts +6 -0
  111. package/src/loader/layer-loader.js +102 -3
  112. package/src/loader/loader-types.d.ts +15 -2
  113. package/src/models/diffusiongemma.js +1 -1
  114. package/src/models/embeddinggemma.js +1 -1
  115. package/src/models/gemma3.js +1 -1
  116. package/src/models/gemma4.js +1 -1
  117. package/src/models/qwen3.js +1 -1
  118. package/src/tooling/hf-registry-utils.js +1 -1
  119. package/src/tooling/node-browser-command-runner.d.ts +16 -0
  120. package/src/tooling/node-browser-command-runner.js +1 -1
  121. package/src/tooling/node-converter.js +33 -1
  122. package/src/tooling/node-webgpu.js +85 -2
  123. package/src/training-browser.d.ts +19 -0
  124. package/src/training-browser.js +13 -0
  125. package/src/training.d.ts +137 -0
  126. package/src/training.js +219 -0
  127. package/src/version.js +1 -1
  128. package/tests/kernels/browser/kernel-suite.js +9 -1
  129. package/tests/kernels/browser/test-page.js +4 -2
  130. package/tests/kernels/reference/adam.js +4 -2
  131. package/NOTICE +0 -5
@@ -63,6 +63,7 @@ import {
63
63
  resolveAdvanceEmbeddingMode,
64
64
  getFinalNormWeights,
65
65
  extractEmbeddingFromHidden,
66
+ extractTokenEmbeddingsFromHidden,
66
67
  } from './generator-runtime.js';
67
68
 
68
69
  import { resolveSamplingConfig } from './sampling-config.js';
@@ -642,7 +643,20 @@ export class PipelineGenerator {
642
643
  embeddingMode,
643
644
  finalNormWeights,
644
645
  config,
645
- this.#state.embeddingPostprocessor
646
+ this.#state.embeddingPostprocessor,
647
+ null,
648
+ this.#state.weights.get('final_norm_bias') ?? null
649
+ );
650
+ }
651
+
652
+ _extractTokenEmbeddingsFromHidden(hiddenStates, numTokens, hiddenSize, finalNormWeights, config) {
653
+ return extractTokenEmbeddingsFromHidden(
654
+ hiddenStates,
655
+ numTokens,
656
+ hiddenSize,
657
+ finalNormWeights,
658
+ config,
659
+ this.#state.weights.get('final_norm_bias') ?? null
646
660
  );
647
661
  }
648
662
 
@@ -2101,6 +2115,111 @@ export class PipelineGenerator {
2101
2115
  }
2102
2116
  }
2103
2117
 
2118
+ async prefillForLoRATraining(inputIds, options = {}) {
2119
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
2120
+ if (this.#state.isGenerating) throw new Error('Generation already in progress');
2121
+ if (!Array.isArray(inputIds) && !(inputIds instanceof Int32Array) && !(inputIds instanceof Uint32Array)) {
2122
+ throw new Error('prefillForLoRATraining requires token IDs.');
2123
+ }
2124
+ const tokens = Array.from(inputIds, (value) => Number(value));
2125
+ if (tokens.length === 0) throw new Error('prefillForLoRATraining requires at least one token.');
2126
+ this._assertTokenIdsInRange(tokens, 'prefillForLoRATraining.inputIds');
2127
+
2128
+ const config = this.#state.modelConfig;
2129
+ const layerIdx = Number(options.layerIdx);
2130
+ if (!Number.isInteger(layerIdx) || layerIdx !== config.numLayers - 1) {
2131
+ throw new Error(`native_lora_target_not_supported: layerIdx must be the final layer (${config.numLayers - 1}).`);
2132
+ }
2133
+ if (options.module !== 'down_proj') {
2134
+ throw new Error('native_lora_target_not_supported: module must be down_proj.');
2135
+ }
2136
+ if (config.layerTypes?.[layerIdx] !== 'full_attention') {
2137
+ throw new Error('native_lora_target_not_supported: final layer must be full_attention.');
2138
+ }
2139
+ if (config.postFeedforwardNorm === true) {
2140
+ throw new Error('native_lora_target_not_supported: final-layer post-feedforward normalization is not supported.');
2141
+ }
2142
+ if (this.#state.lora) {
2143
+ throw new Error('native_lora_base_capture_requires_inactive_adapter.');
2144
+ }
2145
+
2146
+ this.resetToSeqLen(0);
2147
+ this._resetDecodeRuntimeState();
2148
+ const opts = resolvePrefillOptions(this.#state, {
2149
+ ...options,
2150
+ inputIds: tokens,
2151
+ useChatTemplate: false,
2152
+ });
2153
+ let activationTensor = null;
2154
+ let hiddenTensor = null;
2155
+ let disposed = false;
2156
+ const releaseOwned = () => {
2157
+ if (disposed) return;
2158
+ disposed = true;
2159
+ if (activationTensor?.buffer) releaseBuffer(activationTensor.buffer);
2160
+ if (hiddenTensor?.buffer) releaseBuffer(hiddenTensor.buffer);
2161
+ };
2162
+ const capture = async ({ tensor, numTokens, hiddenSize, recorder }) => {
2163
+ if (activationTensor) {
2164
+ throw new Error('native_lora_training_capture_duplicated.');
2165
+ }
2166
+ const bytesPerElement = tensor.dtype === 'f16' ? 2 : 4;
2167
+ const byteLength = Math.ceil((numTokens * hiddenSize * bytesPerElement) / 4) * 4;
2168
+ const buffer = acquireBuffer(byteLength, undefined, `L${layerIdx}.native_lora_ffn_act`);
2169
+ if (recorder) {
2170
+ recorder.getEncoder().copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
2171
+ } else {
2172
+ const device = getDevice();
2173
+ const encoder = device.createCommandEncoder({ label: `L${layerIdx}.native_lora_capture` });
2174
+ encoder.copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
2175
+ device.queue.submit([encoder.finish()]);
2176
+ }
2177
+ activationTensor = createTensor(
2178
+ buffer,
2179
+ tensor.dtype,
2180
+ [numTokens, hiddenSize],
2181
+ `L${layerIdx}.native_lora_ffn_act`
2182
+ );
2183
+ };
2184
+
2185
+ try {
2186
+ const prefill = await this._prefillToHidden(tokens, {
2187
+ ...opts,
2188
+ _trainingCapture: { layerIdx, stage: 'ffn_act', capture },
2189
+ });
2190
+ if (prefill.currentRecorder) {
2191
+ recordPrefillRecorderStats(this.#state, prefill.currentRecorder);
2192
+ await prefill.currentRecorder.submitAndWait();
2193
+ await prefill.recordProfile(prefill.currentRecorder);
2194
+ } else {
2195
+ await getDevice().queue.onSubmittedWorkDone();
2196
+ }
2197
+ if (!activationTensor) {
2198
+ releaseBuffer(prefill.currentHiddenBuffer);
2199
+ throw new Error('native_lora_training_activation_not_captured.');
2200
+ }
2201
+ hiddenTensor = createTensor(
2202
+ prefill.currentHiddenBuffer,
2203
+ prefill.activationDtype,
2204
+ [prefill.numTokens, config.hiddenSize],
2205
+ `L${layerIdx}.native_lora_base_hidden`
2206
+ );
2207
+ this.resetToSeqLen(0);
2208
+ return {
2209
+ inputIds: tokens,
2210
+ layerIdx,
2211
+ module: 'down_proj',
2212
+ activation: activationTensor,
2213
+ baseHidden: hiddenTensor,
2214
+ dispose: releaseOwned,
2215
+ };
2216
+ } catch (error) {
2217
+ releaseOwned();
2218
+ this.resetToSeqLen(0);
2219
+ throw error;
2220
+ }
2221
+ }
2222
+
2104
2223
  async prefillWithEmbedding(prompt, options = {}) {
2105
2224
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
2106
2225
  if (this.#state.isGenerating && options.__internalGenerate !== true) {
@@ -2112,6 +2231,34 @@ export class PipelineGenerator {
2112
2231
  this.#state.stats.gpuTimePrefillMs = undefined;
2113
2232
  this.#state.stats.prefillProfileSteps = [];
2114
2233
  const opts = resolvePrefillEmbeddingOptions(this.#state, options);
2234
+ const diagnosticsEnabled = options?.diagnostics?.enabled === true
2235
+ || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
2236
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
2237
+ if (diagnosticsEnabled || tsirFixtureCfg) {
2238
+ const captureConfig = {
2239
+ ...createDefaultCaptureConfig(),
2240
+ enabled: true,
2241
+ defaultLevel: CAPTURE_LEVELS.SLICE,
2242
+ ...(options?.diagnostics?.captureConfig ?? {}),
2243
+ };
2244
+ validateCaptureConfig(captureConfig);
2245
+ this.#state.operatorDiagnostics = {
2246
+ enabled: diagnosticsEnabled === true,
2247
+ captureConfig,
2248
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
2249
+ modelHash: this.#state.manifest?.modelId ?? null,
2250
+ runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
2251
+ executionPlanHash: opts.executionPlan?.id ?? null,
2252
+ }) : null,
2253
+ tsirFixture: tsirFixtureCfg ? {
2254
+ dir: tsirFixtureCfg.dir,
2255
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
2256
+ ? tsirFixtureCfg.layerFilter
2257
+ : null,
2258
+ records: [],
2259
+ } : null,
2260
+ };
2261
+ }
2115
2262
  const prefillStartSeqLen = this.#state.currentSeqLen;
2116
2263
  const inputStart = performance.now();
2117
2264
  const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
@@ -2178,12 +2325,32 @@ export class PipelineGenerator {
2178
2325
  }
2179
2326
 
2180
2327
  let embedding;
2328
+ let tokenEmbeddings = null;
2329
+ let logits = null;
2181
2330
  let hiddenBytes = 0;
2182
2331
  let readbackMs = 0;
2183
2332
  let decodeHiddenMs = 0;
2184
2333
  let finalNormMs = 0;
2185
2334
  let extractMs = 0;
2335
+ let logitsMs = 0;
2186
2336
  try {
2337
+ if (options.__returnSequenceLogits === true) {
2338
+ const logitsStart = performance.now();
2339
+ logits = await computeLogits(
2340
+ currentHiddenBuffer,
2341
+ numTokens,
2342
+ getLogitsWeights(this.#state),
2343
+ getLogitsConfig(this.#state),
2344
+ this.#state.useGPU,
2345
+ this.#state.debugFlags,
2346
+ undefined,
2347
+ undefined,
2348
+ this.#state.runtimeConfig.shared.debug.probes,
2349
+ { lastPositionOnly: false },
2350
+ this.#state.operatorDiagnostics
2351
+ );
2352
+ logitsMs = performance.now() - logitsStart;
2353
+ }
2187
2354
  const hiddenSize = config.hiddenSize;
2188
2355
  hiddenBytes = numTokens * hiddenSize * activationBytes;
2189
2356
  const readbackStart = performance.now();
@@ -2199,13 +2366,25 @@ export class PipelineGenerator {
2199
2366
  const finalNormWeights = await this._getFinalNormWeights();
2200
2367
  finalNormMs = performance.now() - finalNormStart;
2201
2368
  const extractStart = performance.now();
2202
- embedding = this._extractEmbeddingFromHidden(
2369
+ if (options.__returnTokenEmbeddings === true) {
2370
+ tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
2371
+ hiddenStates,
2372
+ numTokens,
2373
+ hiddenSize,
2374
+ finalNormWeights,
2375
+ config
2376
+ );
2377
+ }
2378
+ embedding = extractEmbeddingFromHidden(
2203
2379
  hiddenStates,
2204
2380
  numTokens,
2205
2381
  hiddenSize,
2206
2382
  opts.embeddingMode,
2207
2383
  finalNormWeights,
2208
- config
2384
+ config,
2385
+ this.#state.embeddingPostprocessor,
2386
+ tokenEmbeddings,
2387
+ this.#state.weights.get('final_norm_bias') ?? null
2209
2388
  );
2210
2389
  extractMs = performance.now() - extractStart;
2211
2390
  } finally {
@@ -2222,6 +2401,7 @@ export class PipelineGenerator {
2222
2401
  decodeHiddenMs,
2223
2402
  finalNormMs,
2224
2403
  extractMs,
2404
+ logitsMs,
2225
2405
  hiddenBytes,
2226
2406
  tokens: numTokens,
2227
2407
  activationDtype,
@@ -2241,6 +2421,8 @@ export class PipelineGenerator {
2241
2421
  seqLen: this.#state.currentSeqLen,
2242
2422
  tokens: inputIds,
2243
2423
  embedding,
2424
+ tokenEmbeddings,
2425
+ logits,
2244
2426
  embeddingMode: opts.embeddingMode,
2245
2427
  phase,
2246
2428
  linearAttention: null,
@@ -2257,12 +2439,22 @@ export class PipelineGenerator {
2257
2439
  seqLen: this.#state.currentSeqLen,
2258
2440
  tokens: inputIds,
2259
2441
  embedding,
2442
+ tokenEmbeddings,
2443
+ logits,
2260
2444
  embeddingMode: opts.embeddingMode,
2261
2445
  phase,
2262
2446
  linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2263
2447
  };
2264
2448
  } finally {
2265
2449
  this._closeFinitenessFallbackWindow(opts);
2450
+ this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2451
+ ? {
2452
+ enabled: true,
2453
+ timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2454
+ recordCount: this.#state.operatorDiagnostics.emitter.length,
2455
+ }
2456
+ : null;
2457
+ this.#state.operatorDiagnostics = null;
2266
2458
  }
2267
2459
  }
2268
2460
 
@@ -3092,6 +3284,7 @@ export class PipelineGenerator {
3092
3284
  debugCheckBuffer,
3093
3285
  opts.executionPlan
3094
3286
  );
3287
+ context.trainingCapture = opts?._trainingCapture ?? null;
3095
3288
  context.currentTokenIds = inputIds;
3096
3289
  context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
3097
3290
  context.skipKVCacheWrites = returnHidden
@@ -2,7 +2,13 @@ import { getDevice } from '../../../gpu/device.js';
2
2
  import { Q4K_BLOCK_BYTES, q4kBlockCount } from '../../../config/schema/index.js';
3
3
  import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
4
4
  import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
5
- import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
5
+ import {
6
+ createWeightBuffer,
7
+ getWeightDtype,
8
+ isGpuBufferInstance,
9
+ isWeightBuffer,
10
+ tagBufferDtype,
11
+ } from '../../../gpu/weight-buffer.js';
6
12
  import { selectRuleValue } from '../../../rules/rule-registry.js';
7
13
  import { log } from '../../../debug/index.js';
8
14
 
@@ -112,6 +118,26 @@ function formatsMatch(a, b) {
112
118
  && a?.bytesPerRow === b?.bytesPerRow;
113
119
  }
114
120
 
121
+ function resolveBiasStorageFormat(weight, expectedElements) {
122
+ if (!weight || !Number.isFinite(expectedElements) || expectedElements <= 0) {
123
+ return null;
124
+ }
125
+ const buffer = isWeightBuffer(weight) ? weight.buffer : weight;
126
+ if (!isGpuBufferInstance(buffer)) {
127
+ return null;
128
+ }
129
+ const dtype = String(getWeightDtype(weight) ?? getWeightDtype(buffer) ?? '').toLowerCase();
130
+ if (dtype !== 'f16' && dtype !== 'f32') {
131
+ return null;
132
+ }
133
+ const bytesPerElement = dtype === 'f16' ? 2 : 4;
134
+ const byteLength = expectedElements * bytesPerElement;
135
+ if (buffer.size < byteLength) {
136
+ return null;
137
+ }
138
+ return { buffer, dtype, bytesPerElement, byteLength };
139
+ }
140
+
115
141
  export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, options = {}) {
116
142
  const device = getDevice();
117
143
  if (!device) {
@@ -174,6 +200,17 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
174
200
  continue;
175
201
  }
176
202
 
203
+ const projectionBiases = [weights.qProjBias, weights.kProjBias, weights.vProjBias];
204
+ const presentProjectionBiases = projectionBiases.filter(Boolean).length;
205
+ if (presentProjectionBiases !== 0 && presentProjectionBiases !== projectionBiases.length) {
206
+ log.debug('QKV Fusion', `Layer ${l}: partial Q/K/V projection biases require split projections`);
207
+ continue;
208
+ }
209
+ if (hasAttentionOutputGate && presentProjectionBiases > 0) {
210
+ log.debug('QKV Fusion', `Layer ${l}: gated Q projection biases require split projections`);
211
+ continue;
212
+ }
213
+
177
214
  const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
178
215
  const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
179
216
  const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
@@ -228,6 +265,32 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
228
265
  })
229
266
  : null;
230
267
 
268
+ let qBiasFormat = null;
269
+ let kBiasFormat = null;
270
+ let vBiasFormat = null;
271
+ let qkvBiasBuffer = null;
272
+ if (presentProjectionBiases === projectionBiases.length) {
273
+ qBiasFormat = resolveBiasStorageFormat(weights.qProjBias, qSize);
274
+ kBiasFormat = resolveBiasStorageFormat(weights.kProjBias, kSize);
275
+ vBiasFormat = resolveBiasStorageFormat(weights.vProjBias, vSize);
276
+ const biasFormatsMatch = qBiasFormat
277
+ && kBiasFormat
278
+ && vBiasFormat
279
+ && qBiasFormat.dtype === kBiasFormat.dtype
280
+ && qBiasFormat.dtype === vBiasFormat.dtype;
281
+ if (!biasFormatsMatch) {
282
+ log.debug('QKV Fusion', `Layer ${l}: inconsistent Q/K/V projection bias formats, skipping`);
283
+ qkvBuffer.destroy();
284
+ qGateBuffer?.destroy();
285
+ continue;
286
+ }
287
+ qkvBiasBuffer = device.createBuffer({
288
+ label: `layer_${l}_qkv_proj_bias`,
289
+ size: qBiasFormat.byteLength + kBiasFormat.byteLength + vBiasFormat.byteLength,
290
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
291
+ });
292
+ }
293
+
231
294
  const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
232
295
  if (hasAttentionOutputGate) {
233
296
  for (let head = 0; head < numHeads; head++) {
@@ -266,6 +329,23 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
266
329
  qkvBuffer, qBytes + kFormat.byteLength,
267
330
  vFormat.byteLength
268
331
  );
332
+ if (qkvBiasBuffer) {
333
+ encoder.copyBufferToBuffer(qBiasFormat.buffer, 0, qkvBiasBuffer, 0, qBiasFormat.byteLength);
334
+ encoder.copyBufferToBuffer(
335
+ kBiasFormat.buffer,
336
+ 0,
337
+ qkvBiasBuffer,
338
+ qBiasFormat.byteLength,
339
+ kBiasFormat.byteLength
340
+ );
341
+ encoder.copyBufferToBuffer(
342
+ vBiasFormat.buffer,
343
+ 0,
344
+ qkvBiasBuffer,
345
+ qBiasFormat.byteLength + kBiasFormat.byteLength,
346
+ vBiasFormat.byteLength
347
+ );
348
+ }
269
349
  device.queue.submit([encoder.finish()]);
270
350
 
271
351
  // Store fused buffer, sizes, and dtype
@@ -278,6 +358,10 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
278
358
  );
279
359
  weights.qkvSizes = [qSize, kSize, vSize];
280
360
  weights.qkvDtype = dtype;
361
+ if (qkvBiasBuffer) {
362
+ tagBufferDtype(qkvBiasBuffer, qBiasFormat.dtype);
363
+ weights.qkvProjBias = qkvBiasBuffer;
364
+ }
281
365
  if (qGateBuffer) {
282
366
  weights.qGateProj = createWeightBuffer(
283
367
  qGateBuffer,
@@ -237,7 +237,9 @@ export interface WeightLoadResult {
237
237
  layerWeights: Map<string, LayerWeights>;
238
238
  embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
239
239
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
240
+ lmHeadBias: Float32Array | null;
240
241
  finalNorm: GPUBuffer | Float32Array | null;
242
+ finalNormBias: Float32Array | null;
241
243
  embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
242
244
  diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
243
245
  perLayerInputWeights: PerLayerInputWeights | null;
@@ -1020,7 +1020,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1020
1020
  layerWeights,
1021
1021
  embeddings: dopplerLoader.embeddings,
1022
1022
  lmHead: dopplerLoader.lmHead,
1023
+ lmHeadBias: dopplerLoader.lmHeadBias,
1023
1024
  finalNorm: dopplerLoader.finalNorm,
1025
+ finalNormBias: dopplerLoader.finalNormBias,
1024
1026
  embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
1025
1027
  diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
1026
1028
  perLayerInputWeights: dopplerLoader.perLayerInputWeights,
@@ -57,6 +57,7 @@ function shouldUseSandwichRMSNormPairFusion({
57
57
  attnOutput,
58
58
  inputTensor,
59
59
  }) {
60
+ if (context.config?.normalizationType === 'layernorm') return false;
60
61
  const mergedSession = getRuntimeConfig()?.inference?.session;
61
62
  if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
62
63
  return false;
@@ -123,6 +124,7 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
123
124
  activationDtype,
124
125
  layerScalar,
125
126
  }) {
127
+ if (config.normalizationType === 'layernorm') return false;
126
128
  const mergedSession = getRuntimeConfig()?.inference?.session;
127
129
  if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
128
130
  return false;
@@ -176,6 +178,7 @@ function shouldUseStandardPostFfnNextInputRMSNormPairFusion({
176
178
  layerScalar,
177
179
  residualBranchScale,
178
180
  }) {
181
+ if (config.normalizationType === 'layernorm') return false;
179
182
  const mergedSession = getRuntimeConfig()?.inference?.session;
180
183
  if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
181
184
  return false;
@@ -851,6 +854,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
851
854
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
852
855
  queryPreAttnScalar: config.queryPreAttnScalar,
853
856
  queryKeyNorm,
857
+ queryKeyNormType: config.queryKeyNormType,
858
+ queryKeyNormAxis: config.queryKeyNormAxis,
854
859
  queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
855
860
  valueNorm: config.valueNorm,
856
861
  attentionOutputGate: config.attentionOutputGate,
@@ -860,6 +865,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
860
865
  ? (context.multimodalBidirectionalSpan ?? null)
861
866
  : null,
862
867
  rmsNormWeightOffset: config.rmsNormWeightOffset,
868
+ normalizationType: config.normalizationType,
863
869
  ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
864
870
  ropeFrequencyBaseDim: resolveAttentionFrequencyBaseDim(config, layerType),
865
871
  ropeInterleaved: config.ropeInterleaved,
@@ -6,6 +6,7 @@ import type { LinearNormMode } from '../../../config/schema/index.js';
6
6
  import type { ProbeConfigSchema } from '../../../config/schema/index.js';
7
7
  import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
8
8
  import type { KernelPathSchema } from '../../../config/schema/kernel-path.schema.js';
9
+ import type { LoRAAdapter } from './lora.js';
9
10
 
10
11
  export interface LinearLayerRuntimeState {
11
12
  layerIdx: number;
@@ -76,6 +77,7 @@ export interface RunLinearAttentionLayerOptions {
76
77
  precomputedInputNorm?: Tensor | null;
77
78
  debugProbes?: ProbeConfigSchema[] | null;
78
79
  recorder?: CommandRecorder | null;
80
+ lora?: LoRAAdapter | null;
79
81
  }
80
82
 
81
83
  export declare function hasLinearAttentionLayers(layerTypes: unknown): boolean;
@@ -23,6 +23,8 @@ import { dequantizeQ4KM } from '../../../converter/quantizer.js';
23
23
  import { getKernelPathMatmulPrecision } from '../../../config/kernel-path-loader.js';
24
24
  import { selectRuleValue } from '../../../rules/rule-registry.js';
25
25
  import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
26
+ import { applyLoRA } from './lora-apply.js';
27
+ import { getLoRAModule } from './lora.js';
26
28
 
27
29
  const LINEAR_RUNTIME_SCHEMA_VERSION = 1;
28
30
  const QK_L2NORM_EPS = 1e-6;
@@ -172,6 +174,38 @@ function releaseOrTrackBuffer(recorder, buffer) {
172
174
  }
173
175
  }
174
176
 
177
+ async function applyProjectionLoRA({
178
+ inputTensor,
179
+ baseOutput,
180
+ loraModule,
181
+ dims,
182
+ getWeightBuffer,
183
+ recorder,
184
+ kernelPath,
185
+ }) {
186
+ if (!loraModule) {
187
+ return baseOutput;
188
+ }
189
+ try {
190
+ const combined = await applyLoRA(
191
+ inputTensor,
192
+ baseOutput,
193
+ loraModule,
194
+ dims,
195
+ getWeightBuffer,
196
+ recorder ?? undefined,
197
+ { kernelPath }
198
+ );
199
+ if (combined.buffer !== baseOutput.buffer) {
200
+ releaseOrTrackBuffer(recorder, baseOutput.buffer);
201
+ }
202
+ return combined;
203
+ } catch (error) {
204
+ releaseOrTrackBuffer(recorder, baseOutput.buffer);
205
+ throw error;
206
+ }
207
+ }
208
+
175
209
  function releaseResolvedWeightBuffer(originalWeight, resolvedWeight, recorder) {
176
210
  if (isResolvedWeightShared(originalWeight)) {
177
211
  return;
@@ -637,6 +671,7 @@ async function projectLinearTensor({
637
671
  getWeightBuffer,
638
672
  recorder,
639
673
  executionPolicies = null,
674
+ loraModule = null,
640
675
  }) {
641
676
  const resolvedWeight = getWeightBuffer(sourceWeight, role);
642
677
  const resolvedInputDtype = resolveMatmulStepDtype(
@@ -702,7 +737,16 @@ async function projectLinearTensor({
702
737
  outputDtype: resolvedOutputDtype,
703
738
  executionPolicies,
704
739
  });
705
- return await settleProjectionOutputDtype(result);
740
+ const projected = await settleProjectionOutputDtype(result);
741
+ return await applyProjectionLoRA({
742
+ inputTensor: matmulInput,
743
+ baseOutput: projected,
744
+ loraModule,
745
+ dims: { M: numTokens, N: outDim, K: hiddenSize },
746
+ getWeightBuffer,
747
+ recorder,
748
+ kernelPath,
749
+ });
706
750
  }
707
751
  const result = await runMatmul(matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
708
752
  transposeB: 'auto',
@@ -712,7 +756,16 @@ async function projectLinearTensor({
712
756
  outputDtype: resolvedOutputDtype,
713
757
  executionPolicies,
714
758
  });
715
- return await settleProjectionOutputDtype(result);
759
+ const projected = await settleProjectionOutputDtype(result);
760
+ return await applyProjectionLoRA({
761
+ inputTensor: matmulInput,
762
+ baseOutput: projected,
763
+ loraModule,
764
+ dims: { M: numTokens, N: outDim, K: hiddenSize },
765
+ getWeightBuffer,
766
+ recorder: null,
767
+ kernelPath,
768
+ });
716
769
  } finally {
717
770
  if (matmulInput !== inputTensor) {
718
771
  releaseOrTrackBuffer(recorder, matmulInput.buffer);
@@ -835,6 +888,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
835
888
  recorder,
836
889
  executionPolicies = null,
837
890
  precomputedInputNorm = null,
891
+ lora = null,
838
892
  } = options;
839
893
 
840
894
  if (!layerWeights) {
@@ -909,7 +963,12 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
909
963
  }
910
964
  }
911
965
 
912
- const qkvzProjection = resolveLinearAttentionQKVZProjection(layerWeights, {
966
+ const qkvLoRA = getLoRAModule(lora, layerIdx, 'in_proj_qkv');
967
+ const zLoRA = getLoRAModule(lora, layerIdx, 'in_proj_z');
968
+ const aLoRA = getLoRAModule(lora, layerIdx, 'in_proj_a');
969
+ const bLoRA = getLoRAModule(lora, layerIdx, 'in_proj_b');
970
+ const outLoRA = getLoRAModule(lora, layerIdx, 'out_proj');
971
+ const qkvzProjection = qkvLoRA || zLoRA ? null : resolveLinearAttentionQKVZProjection(layerWeights, {
913
972
  phase,
914
973
  numTokens,
915
974
  hiddenSize,
@@ -951,6 +1010,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
951
1010
  getWeightBuffer,
952
1011
  recorder,
953
1012
  executionPolicies,
1013
+ loraModule: qkvLoRA,
954
1014
  });
955
1015
  const zTensor = qkvzTensor ?? await projectLinearTensor({
956
1016
  inputTensor: normedTensor,
@@ -966,8 +1026,9 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
966
1026
  getWeightBuffer,
967
1027
  recorder,
968
1028
  executionPolicies,
1029
+ loraModule: zLoRA,
969
1030
  });
970
- const abProjection = resolveLinearAttentionABProjection(layerWeights, {
1031
+ const abProjection = aLoRA || bLoRA ? null : resolveLinearAttentionABProjection(layerWeights, {
971
1032
  phase,
972
1033
  numTokens,
973
1034
  hiddenSize,
@@ -1007,6 +1068,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
1007
1068
  getWeightBuffer,
1008
1069
  recorder,
1009
1070
  executionPolicies,
1071
+ loraModule: aLoRA,
1010
1072
  });
1011
1073
  const bTensor = abTensor ?? await projectLinearTensor({
1012
1074
  inputTensor: normedTensor,
@@ -1022,6 +1084,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
1022
1084
  getWeightBuffer,
1023
1085
  recorder,
1024
1086
  executionPolicies,
1087
+ loraModule: bLoRA,
1025
1088
  });
1026
1089
 
1027
1090
  const outProjInputDtype = resolveMatmulStepDtype(
@@ -1160,9 +1223,17 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
1160
1223
  ? (recorder ? await recordCastF32ToF16(recorder, result) : await castF32ToF16(result))
1161
1224
  : (recorder ? await recordCastF16ToF32(recorder, result) : await castF16ToF32(result));
1162
1225
  releaseOrTrackBuffer(recorder, result.buffer);
1163
- return casted;
1226
+ result = casted;
1164
1227
  }
1165
- return result;
1228
+ return await applyProjectionLoRA({
1229
+ inputTensor: coreTensor,
1230
+ baseOutput: result,
1231
+ loraModule: outLoRA,
1232
+ dims: { M: numTokens, N: hiddenSize, K: projectionLayout.valueDim },
1233
+ getWeightBuffer,
1234
+ recorder,
1235
+ kernelPath,
1236
+ });
1166
1237
  } finally {
1167
1238
  releaseOrTrackBuffer(recorder, coreTensor.buffer);
1168
1239
  releaseResolvedWeightBuffer(layerWeights.oProj, outProjWeight, recorder);
@@ -25,6 +25,12 @@ export function rmsNormCPU(
25
25
  eps?: number,
26
26
  rmsNormWeightOffset?: boolean
27
27
  ): Float32Array;
28
+ export function layerNormCPU(
29
+ x: Float32Array,
30
+ weight: Float32Array,
31
+ bias: Float32Array,
32
+ eps?: number
33
+ ): Float32Array;
28
34
 
29
35
  /**
30
36
  * Convert a single float16 value to float32.