doppler-gpu 0.4.10 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +95 -0
  2. package/README.md +40 -1
  3. package/models/catalog.json +637 -32
  4. package/package.json +24 -3
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +135 -13
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +69 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +3 -2
  18. package/src/config/kernels/kernel-ref-digests.js +2 -1
  19. package/src/config/kernels/registry.json +73 -11
  20. package/src/config/merge.js +24 -0
  21. package/src/config/required-inference-fields-contract-check.js +2 -0
  22. package/src/config/schema/adapter.schema.d.ts +6 -1
  23. package/src/config/schema/adapter.schema.js +5 -0
  24. package/src/config/schema/converter.schema.d.ts +7 -0
  25. package/src/config/schema/converter.schema.js +15 -0
  26. package/src/config/schema/inference.schema.d.ts +8 -0
  27. package/src/config/schema/manifest.schema.d.ts +32 -1
  28. package/src/config/schema/manifest.schema.js +26 -0
  29. package/src/converter/conversion-plan.js +15 -1
  30. package/src/converter/core.js +5 -4
  31. package/src/converter/source-tensor-rules.d.ts +50 -0
  32. package/src/converter/source-tensor-rules.js +180 -0
  33. package/src/converter/source-tokenizer.d.ts +45 -0
  34. package/src/converter/source-tokenizer.js +155 -0
  35. package/src/experimental/adapters/artifact-contract.d.ts +83 -0
  36. package/src/experimental/adapters/artifact-contract.js +254 -0
  37. package/src/experimental/adapters/index.d.ts +25 -0
  38. package/src/experimental/adapters/index.js +12 -0
  39. package/src/experimental/adapters/lora-loader.js +24 -4
  40. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  41. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  42. package/src/experimental/training/artifacts.d.ts +160 -0
  43. package/src/experimental/training/attention-backward.d.ts +64 -0
  44. package/src/experimental/training/attention-forward.d.ts +22 -0
  45. package/src/experimental/training/attention-forward.js +82 -0
  46. package/src/experimental/training/autograd.d.ts +76 -0
  47. package/src/experimental/training/checkpoint.d.ts +36 -0
  48. package/src/experimental/training/clip.d.ts +9 -0
  49. package/src/experimental/training/dataloader.d.ts +8 -0
  50. package/src/experimental/training/distillation/student-fixture.js +1 -0
  51. package/src/experimental/training/export.d.ts +41 -0
  52. package/src/experimental/training/lora-pipeline.d.ts +186 -0
  53. package/src/experimental/training/lora-pipeline.js +5 -0
  54. package/src/experimental/training/lora.d.ts +20 -0
  55. package/src/experimental/training/lora.js +4 -3
  56. package/src/experimental/training/loss-scaling.d.ts +21 -0
  57. package/src/experimental/training/loss.d.ts +10 -0
  58. package/src/experimental/training/native-qwen-lora.d.ts +106 -0
  59. package/src/experimental/training/native-qwen-lora.js +449 -0
  60. package/src/experimental/training/objectives/base.d.ts +58 -0
  61. package/src/experimental/training/optimizer.d.ts +22 -0
  62. package/src/experimental/training/optimizer.js +1 -0
  63. package/src/experimental/training/runner.d.ts +249 -0
  64. package/src/experimental/training/trainer.d.ts +89 -0
  65. package/src/experimental/training/workloads.d.ts +205 -0
  66. package/src/gpu/kernels/backward/adam.d.ts +1 -0
  67. package/src/gpu/kernels/backward/adam.js +6 -2
  68. package/src/gpu/kernels/backward/adam.wgsl +3 -2
  69. package/src/gpu/kernels/backward/matmul_backward_q4k.wgsl +90 -0
  70. package/src/gpu/kernels/backward/utils.js +18 -4
  71. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  72. package/src/inference/pipelines/text/attention/projections.js +161 -24
  73. package/src/inference/pipelines/text/attention/record.js +67 -9
  74. package/src/inference/pipelines/text/attention/run.js +60 -7
  75. package/src/inference/pipelines/text/attention/types.js +18 -9
  76. package/src/inference/pipelines/text/config.d.ts +5 -0
  77. package/src/inference/pipelines/text/config.js +91 -0
  78. package/src/inference/pipelines/text/ffn/dense.js +205 -2
  79. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  80. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  81. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  82. package/src/inference/pipelines/text/generator-steps.js +3 -1
  83. package/src/inference/pipelines/text/generator.d.ts +5 -1
  84. package/src/inference/pipelines/text/generator.js +196 -3
  85. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  86. package/src/inference/pipelines/text/init.d.ts +2 -0
  87. package/src/inference/pipelines/text/init.js +2 -0
  88. package/src/inference/pipelines/text/layer.js +6 -0
  89. package/src/inference/pipelines/text/linear-attention.d.ts +2 -0
  90. package/src/inference/pipelines/text/linear-attention.js +77 -6
  91. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  92. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  93. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  94. package/src/inference/pipelines/text/logits/index.js +85 -11
  95. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  96. package/src/inference/pipelines/text/lora-types.d.ts +6 -1
  97. package/src/inference/pipelines/text/lora-types.js +5 -0
  98. package/src/inference/pipelines/text/ops.js +33 -0
  99. package/src/inference/pipelines/text/types.d.ts +66 -0
  100. package/src/inference/pipelines/text/weights.d.ts +10 -0
  101. package/src/inference/pipelines/text/weights.js +33 -1
  102. package/src/inference/pipelines/text.d.ts +9 -2
  103. package/src/inference/pipelines/text.js +116 -0
  104. package/src/inference/tokenizers/behavior-flags.js +5 -1
  105. package/src/inference/tokenizers/bundled.js +99 -8
  106. package/src/loader/doppler-loader.d.ts +2 -0
  107. package/src/loader/doppler-loader.js +9 -0
  108. package/src/loader/final-weights-loader.d.ts +8 -0
  109. package/src/loader/final-weights-loader.js +42 -0
  110. package/src/loader/layer-loader.d.ts +6 -0
  111. package/src/loader/layer-loader.js +102 -3
  112. package/src/loader/loader-types.d.ts +15 -2
  113. package/src/models/diffusiongemma.js +1 -1
  114. package/src/models/embeddinggemma.js +1 -1
  115. package/src/models/gemma3.js +1 -1
  116. package/src/models/gemma4.js +1 -1
  117. package/src/models/qwen3.js +1 -1
  118. package/src/tooling/hf-registry-utils.js +1 -1
  119. package/src/tooling/node-browser-command-runner.d.ts +16 -0
  120. package/src/tooling/node-browser-command-runner.js +1 -1
  121. package/src/tooling/node-converter.js +33 -1
  122. package/src/tooling/node-webgpu.js +85 -2
  123. package/src/training-browser.d.ts +19 -0
  124. package/src/training-browser.js +13 -0
  125. package/src/training.d.ts +137 -0
  126. package/src/training.js +219 -0
  127. package/src/version.js +1 -1
  128. package/tests/kernels/browser/kernel-suite.js +9 -1
  129. package/tests/kernels/browser/test-page.js +4 -2
  130. package/tests/kernels/reference/adam.js +4 -2
  131. package/NOTICE +0 -5
@@ -25,6 +25,28 @@ export function rmsNormCPU(
25
25
  return result;
26
26
  }
27
27
 
28
+ export function layerNormCPU(x, weight, bias, eps = 1e-5) {
29
+ if (weight.length !== x.length || bias.length !== x.length) {
30
+ throw new Error(
31
+ `LayerNorm parameter length mismatch: input=${x.length}, weight=${weight.length}, bias=${bias.length}.`
32
+ );
33
+ }
34
+ let sum = 0;
35
+ for (let i = 0; i < x.length; i++) sum += x[i];
36
+ const mean = sum / x.length;
37
+ let squaredDeviation = 0;
38
+ for (let i = 0; i < x.length; i++) {
39
+ const centered = x[i] - mean;
40
+ squaredDeviation += centered * centered;
41
+ }
42
+ const invStd = 1 / Math.sqrt(squaredDeviation / x.length + eps);
43
+ const result = new Float32Array(x.length);
44
+ for (let i = 0; i < x.length; i++) {
45
+ result[i] = (x[i] - mean) * invStd * weight[i] + bias[i];
46
+ }
47
+ return result;
48
+ }
49
+
28
50
 
29
51
  export function f16BufferToF32(data) {
30
52
  const u16 = new Uint16Array(data);
@@ -17,7 +17,7 @@ import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
17
17
  export type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
18
18
 
19
19
  // Re-export CPU functions
20
- export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
20
+ export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
21
21
 
22
22
  // Re-export GPU functions
23
23
  export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
@@ -1,7 +1,7 @@
1
1
 
2
2
 
3
3
  // Re-export CPU functions
4
- export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
4
+ export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
5
5
 
6
6
  // Re-export GPU functions
7
7
  export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
12
12
  // Imports for computeLogits orchestrator
13
13
  import { getDevice } from '../../../../gpu/device.js';
14
14
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
15
- import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
15
+ import { runLayerNorm, runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
16
16
  import { createTensor } from '../../../../gpu/tensor.js';
17
17
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
18
18
  import { kernelTrace, traceStep } from '../kernel-trace.js';
@@ -72,6 +72,45 @@ function normalizeSelectedLogitTokenIds(value, vocabSize) {
72
72
  return tokenIds;
73
73
  }
74
74
 
75
+ function applyLmHeadBias(logits, bias, rowCount, rowSize, selectedTokenIds = null) {
76
+ if (bias == null) {
77
+ return logits;
78
+ }
79
+ if (!(bias instanceof Float32Array)) {
80
+ throw new Error('[Logits] LM head bias must be a Float32Array.');
81
+ }
82
+ if (selectedTokenIds !== null) {
83
+ if (logits.length !== selectedTokenIds.length) {
84
+ throw new Error('[Logits] Selected-logit bias application length mismatch.');
85
+ }
86
+ for (let index = 0; index < selectedTokenIds.length; index += 1) {
87
+ const tokenId = selectedTokenIds[index];
88
+ if (tokenId >= bias.length) {
89
+ throw new Error(`[Logits] LM head bias is missing token ${tokenId}.`);
90
+ }
91
+ logits[index] += bias[tokenId];
92
+ }
93
+ return logits;
94
+ }
95
+ if (bias.length < rowSize) {
96
+ throw new Error(
97
+ `[Logits] LM head bias length mismatch: expected at least ${rowSize}, got ${bias.length}.`
98
+ );
99
+ }
100
+ if (logits.length !== rowCount * rowSize) {
101
+ throw new Error(
102
+ `[Logits] LM head bias matrix mismatch: expected ${rowCount * rowSize} logits, got ${logits.length}.`
103
+ );
104
+ }
105
+ for (let row = 0; row < rowCount; row += 1) {
106
+ const offset = row * rowSize;
107
+ for (let column = 0; column < rowSize; column += 1) {
108
+ logits[offset + column] += bias[column];
109
+ }
110
+ }
111
+ return logits;
112
+ }
113
+
75
114
  async function coerceTensorDtype(tensor, targetDtype, options = {}) {
76
115
  if (!targetDtype || tensor.dtype === targetDtype) {
77
116
  return tensor;
@@ -164,7 +203,7 @@ export async function computeLogits(
164
203
  activationDtype: activationDtypeOverride,
165
204
  } = config;
166
205
  const activationDtype = activationDtypeOverride ?? getRuntimeConfig().inference.compute.activationDtype;
167
- const { finalNorm, lmHead } = weights;
206
+ const { finalNorm, finalNormBias = null, lmHead, lmHeadBias = null } = weights;
168
207
  const device = getDevice();
169
208
 
170
209
  // Consistency check: warn if lmHead weight dtype disagrees with layer activation dtype.
@@ -269,6 +308,7 @@ export async function computeLogits(
269
308
  cpuWeightLayout === 'column' ? cpuWeightVocabSize : null
270
309
  )
271
310
  : matmulCPU(normed, (lmHead), numTokens, matmulVocabSize, hiddenSize);
311
+ applyLmHeadBias(rawLogits, lmHeadBias, numTokens, matmulVocabSize);
272
312
  return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
273
313
  }
274
314
 
@@ -293,7 +333,7 @@ export async function computeLogits(
293
333
  dtype: inputDtype,
294
334
  });
295
335
 
296
- // 2. Apply final RMSNorm
336
+ // 2. Apply the manifest-owned final normalization.
297
337
 
298
338
  let normWeightBuffer;
299
339
  let normWeightBufferOwned = false;
@@ -319,7 +359,10 @@ export async function computeLogits(
319
359
  const kernelPath = config.kernelPath ?? null;
320
360
  const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
321
361
  const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
322
- const forceStableF32Logits = !hasExplicitFinalNormPrecision && shouldForceStableF32Logits(config, inputDtype);
362
+ const forceStableF32Logits = !hasExplicitFinalNormPrecision && (
363
+ config.normalizationType === 'layernorm'
364
+ || shouldForceStableF32Logits(config, inputDtype)
365
+ );
323
366
  const stableKernelPath = forceStableF32Logits
324
367
  ? createStableF32LogitsKernelPath(kernelPath)
325
368
  : kernelPath;
@@ -346,11 +389,39 @@ export async function computeLogits(
346
389
  : inputTensor;
347
390
  normInputBufferOwned = normInputTensor !== inputTensor;
348
391
  }
349
- let normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
350
- batchSize: numTokens,
351
- hiddenSize,
352
- rmsNormWeightOffset: config.rmsNormWeightOffset,
353
- });
392
+ let finalNormBiasBuffer = null;
393
+ let normedTensor;
394
+ if (config.normalizationType === 'layernorm') {
395
+ if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
396
+ throw new Error(
397
+ `[Logits] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
398
+ );
399
+ }
400
+ const effectiveBias = finalNormBias instanceof Float32Array
401
+ ? finalNormBias
402
+ : new Float32Array(hiddenSize);
403
+ if (effectiveBias.length !== hiddenSize) {
404
+ throw new Error(
405
+ `[Logits] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveBias.length}.`
406
+ );
407
+ }
408
+ finalNormBiasBuffer = acquireBuffer(effectiveBias.byteLength, undefined, 'final_norm_bias');
409
+ device.queue.writeBuffer(finalNormBiasBuffer, 0, effectiveBias);
410
+ normedTensor = await runLayerNorm(
411
+ normInputTensor,
412
+ normWeightBuffer,
413
+ finalNormBiasBuffer,
414
+ rmsNormEps,
415
+ { batchSize: numTokens, hiddenSize }
416
+ );
417
+ } else {
418
+ normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
419
+ batchSize: numTokens,
420
+ hiddenSize,
421
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
422
+ });
423
+ }
424
+ if (finalNormBiasBuffer) releaseBuffer(finalNormBiasBuffer);
354
425
  if (normInputBufferOwned) {
355
426
  releaseBuffer(normInputTensor.buffer);
356
427
  }
@@ -386,7 +457,7 @@ export async function computeLogits(
386
457
 
387
458
  // Trace final norm output
388
459
  if (kernelTrace.enabled) {
389
- await traceStep('rmsnorm', 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
460
+ await traceStep(config.normalizationType, 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
390
461
  }
391
462
 
392
463
  // Debug: Check hidden state after final norm
@@ -461,6 +532,7 @@ export async function computeLogits(
461
532
  if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
462
533
  if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
463
534
 
535
+ applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
464
536
  return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
465
537
  }
466
538
 
@@ -533,6 +605,7 @@ export async function computeLogits(
533
605
  if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
534
606
  });
535
607
  const selectedLogits = new Float32Array(selectedData);
608
+ applyLmHeadBias(selectedLogits, lmHeadBias, 1, selectedTokenIds.length, selectedTokenIds);
536
609
  await runProbes('logits_final', selectedLogits, {
537
610
  numTokens: 1,
538
611
  hiddenSize: selectedTokenIds.length,
@@ -579,6 +652,7 @@ export async function computeLogits(
579
652
  const rawLogits = logitsTensor.dtype === 'f16'
580
653
  ? f16BufferToF32(logitsData)
581
654
  : new Float32Array(logitsData);
655
+ applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
582
656
  if (isTraceEnabled('logits')) {
583
657
  trace.logits('LM_HEAD_RAW_LOGITS_HEALTH', getLogitsHealth(rawLogits));
584
658
  }
@@ -17,6 +17,8 @@ import type { WeightBuffer, CpuWeightBuffer, SplitWeightBuffer } from '../../../
17
17
  export interface LogitsConfig {
18
18
  hiddenSize: number;
19
19
  vocabSize: number;
20
+ normalizationType: 'rmsnorm' | 'layernorm';
21
+ finalNormBiasTensor: string | null;
20
22
  rmsNormEps: number;
21
23
  useTiedEmbeddings: boolean;
22
24
  embeddingVocabSize: number | null;
@@ -38,7 +40,9 @@ export interface LogitsConfig {
38
40
  */
39
41
  export interface LogitsWeights {
40
42
  finalNorm: GPUBuffer | Float32Array;
43
+ finalNormBias?: Float32Array | null;
41
44
  lmHead: GPUBuffer | Float32Array | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer;
45
+ lmHeadBias?: Float32Array | null;
42
46
  }
43
47
 
44
48
  /**
@@ -14,7 +14,12 @@ export type LoRAModuleName =
14
14
  | 'gate_proj'
15
15
  | 'up_proj'
16
16
  | 'down_proj'
17
- | 'gate_up_proj';
17
+ | 'gate_up_proj'
18
+ | 'in_proj_a'
19
+ | 'in_proj_b'
20
+ | 'in_proj_qkv'
21
+ | 'in_proj_z'
22
+ | 'out_proj';
18
23
 
19
24
  export interface LoRAModuleWeights {
20
25
  a: MaybeGPUBuffer;
@@ -15,4 +15,9 @@ export const LORA_MODULE_ALIASES = {
15
15
  'gate': 'gate_proj',
16
16
  'up': 'up_proj',
17
17
  'down': 'down_proj',
18
+ 'in_proj_a': 'in_proj_a',
19
+ 'in_proj_b': 'in_proj_b',
20
+ 'in_proj_qkv': 'in_proj_qkv',
21
+ 'in_proj_z': 'in_proj_z',
22
+ 'out_proj': 'out_proj',
18
23
  };
@@ -2,7 +2,9 @@
2
2
 
3
3
  import {
4
4
  runRMSNorm, runResidualAdd, runMatmul, runSiLU, runGeLU,
5
+ runLayerNorm, runBiasAdd,
5
6
  recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
7
+ recordLayerNorm, recordBiasAdd,
6
8
  runSiLURowSplit, recordSiLURowSplit,
7
9
  runMatmulRMSNormFused, recordMatmulRMSNormFused,
8
10
  runRMSNormStats, recordRMSNormStats,
@@ -60,6 +62,36 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
60
62
  return result;
61
63
  }
62
64
 
65
+ export async function doLayerNorm(input, weight, bias, eps, options, recorder) {
66
+ const result = recorder
67
+ ? await recordLayerNorm(recorder, input, weight, bias, eps, options)
68
+ : await runLayerNorm(input, weight, bias, eps, options);
69
+
70
+ if (kernelTrace.enabled && !recorder) {
71
+ const layer = options.layerIdx ?? -1;
72
+ const label = options.label ?? 'layernorm';
73
+ await traceStep('layernorm', label, layer, result.buffer, [options.batchSize, options.hiddenSize]);
74
+ }
75
+ return result;
76
+ }
77
+
78
+ export async function doBiasAdd(data, bias, numTokens, dim, options, recorder) {
79
+ const result = recorder
80
+ ? await recordBiasAdd(recorder, data, bias, numTokens, dim, options)
81
+ : await runBiasAdd(data, bias, numTokens, dim, options);
82
+
83
+ if (kernelTrace.enabled && !recorder) {
84
+ await traceStep(
85
+ 'bias_add',
86
+ options?.label ?? 'bias_add',
87
+ options?.layerIdx ?? -1,
88
+ result.buffer,
89
+ [numTokens, dim]
90
+ );
91
+ }
92
+ return result;
93
+ }
94
+
63
95
  export async function doRMSNormStats(input, residual, eps, options, recorder) {
64
96
  return recorder
65
97
  ? await recordRMSNormStats(recorder, input, residual, eps, options)
@@ -472,6 +504,7 @@ export async function doAttention(
472
504
  getWeightBuffer: getWeightBufferFn,
473
505
  getNormWeightBuffer: getNormWeightBufferFn,
474
506
  recorder: recorder ?? null,
507
+ lora: lora ?? null,
475
508
  }),
476
509
  residualFused: false,
477
510
  };
@@ -34,6 +34,7 @@ import type { LinearAttentionRuntime } from './linear-attention.js';
34
34
  import type { TokenizerLoadTiming } from '../../tokenizer.js';
35
35
  import type { LoaderLoadTiming, PerLayerInputWeights } from '../../../loader/loader-types.js';
36
36
  import type { UniformCacheStats } from '../../../gpu/uniform-cache.js';
37
+ import type { Tensor } from '../../../gpu/tensor.js';
37
38
 
38
39
  // ============================================================================
39
40
  // Core Context Types
@@ -85,6 +86,19 @@ export interface LayerContext {
85
86
  debug: boolean;
86
87
  /** Config-driven probes */
87
88
  debugProbes?: ProbeConfigSchema[];
89
+ /** Internal native-LoRA activation capture owned by the training operator. */
90
+ trainingCapture?: {
91
+ layerIdx: number;
92
+ stage: 'ffn_act';
93
+ capture: (capture: {
94
+ layerIdx: number;
95
+ stage: 'ffn_act';
96
+ tensor: Tensor;
97
+ numTokens: number;
98
+ hiddenSize: number;
99
+ recorder: CommandRecorder | null;
100
+ }) => Promise<void>;
101
+ } | null;
88
102
  /** Layers to debug (null = none, undefined/empty = layer 0 only for backward compat) */
89
103
  debugLayers?: number[] | null;
90
104
  /** Optional GPU buffer readback helper for debug checks */
@@ -390,6 +404,26 @@ export interface GenerateOptions {
390
404
  offset: number;
391
405
  length: number;
392
406
  } | null;
407
+
408
+ /** @internal Return final-normalized embeddings for every input token. */
409
+ __returnTokenEmbeddings?: boolean;
410
+
411
+ /** @internal Return full per-token vocabulary logits. */
412
+ __returnSequenceLogits?: boolean;
413
+ }
414
+
415
+ export interface NativeLoRAPrefillOptions extends GenerateOptions {
416
+ layerIdx: number;
417
+ module: 'down_proj';
418
+ }
419
+
420
+ export interface NativeLoRAPrefillResult {
421
+ inputIds: number[];
422
+ layerIdx: number;
423
+ module: 'down_proj';
424
+ activation: Tensor;
425
+ baseHidden: Tensor;
426
+ dispose(): void;
393
427
  }
394
428
 
395
429
  /**
@@ -438,6 +472,30 @@ export interface PrefillEmbeddingResult extends KVCacheSnapshot {
438
472
  /** Pooling mode used to construct embedding */
439
473
  embeddingMode: 'last' | 'mean';
440
474
 
475
+ /** Final-normalized token matrix when explicitly requested. */
476
+ tokenEmbeddings?: Float32Array | null;
477
+
478
+ /** Full token-by-vocabulary logits when explicitly requested. */
479
+ logits?: Float32Array | null;
480
+
481
+ phase?: WorkloadPhaseTiming | null;
482
+ }
483
+
484
+ export interface SequenceEncodeOptions extends GenerateOptions {
485
+ includeTokenEmbeddings?: boolean;
486
+ includeLogits?: boolean;
487
+ }
488
+
489
+ export interface SequenceEncodeResult {
490
+ alphabet: 'amino_acid' | 'nucleotide';
491
+ tokens: number[];
492
+ tokenMask: Uint8Array;
493
+ includedTokenCount: number;
494
+ tokenEmbeddings: Float32Array | null;
495
+ pooledEmbedding: Float32Array | null;
496
+ logits: Float32Array | null;
497
+ embeddingDim: number;
498
+ vocabSize: number;
441
499
  phase?: WorkloadPhaseTiming | null;
442
500
  }
443
501
 
@@ -495,16 +553,23 @@ export type LayerWeightBuffer = GPUBuffer | WeightBuffer | Float32Array | CpuWei
495
553
  export interface LayerWeights {
496
554
  // Attention
497
555
  inputNorm: GPUBuffer | Float32Array;
556
+ inputNormBias?: GPUBuffer | Float32Array | null;
498
557
  qProj: LayerWeightBuffer;
558
+ qProjBias?: GPUBuffer | Float32Array | null;
499
559
  kProj: LayerWeightBuffer;
560
+ kProjBias?: GPUBuffer | Float32Array | null;
500
561
  vProj?: LayerWeightBuffer;
562
+ vProjBias?: GPUBuffer | Float32Array | null;
501
563
  oProj: LayerWeightBuffer;
564
+ oProjBias?: GPUBuffer | Float32Array | null;
502
565
  qGateProj?: LayerWeightBuffer | null;
503
566
  convInProj?: LayerWeightBuffer;
504
567
  convKernel?: LayerWeightBuffer;
505
568
  convOutProj?: LayerWeightBuffer;
506
569
  /** Fused Q/K/V projection (runtime-generated for 3->1 matmul optimization) */
507
570
  qkvProj?: GPUBuffer | WeightBuffer | null;
571
+ /** Concatenated Q/K/V projection bias matching qkvSizes. */
572
+ qkvProjBias?: GPUBuffer | Float32Array | null;
508
573
  /** Sizes for splitting fused QKV output: [qSize, kSize, vSize] in elements */
509
574
  qkvSizes?: [number, number, number];
510
575
  /** Data type of fused QKV weights (f16, f32, or q4k) */
@@ -516,6 +581,7 @@ export interface LayerWeights {
516
581
 
517
582
  // FFN (dense layers)
518
583
  postAttentionNorm?: GPUBuffer | Float32Array;
584
+ postAttentionNormBias?: GPUBuffer | Float32Array | null;
519
585
  postAttnNorm?: GPUBuffer | Float32Array; // LLaMA-style pre-FFN norm
520
586
  gate?: LayerWeightBuffer;
521
587
  up?: LayerWeightBuffer;
@@ -12,6 +12,7 @@
12
12
 
13
13
  import type { LayerWeights } from './types.js';
14
14
  import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
15
+ import type { Tensor } from '../../../gpu/tensor.js';
15
16
 
16
17
  // ============================================================================
17
18
  // Types
@@ -60,3 +61,12 @@ export function getNormWeightBuffer(
60
61
  debugFlags?: WeightDebugFlags,
61
62
  deviceOverride?: GPUDevice | null
62
63
  ): GPUBuffer;
64
+
65
+ export function getVectorTensor(
66
+ weight: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | { buffer: ArrayBuffer; byteOffset: number; byteLength: number } | CpuWeightBuffer,
67
+ label: string,
68
+ length: number,
69
+ config: WeightBufferConfig,
70
+ debugFlags?: WeightDebugFlags,
71
+ deviceOverride?: GPUDevice | null
72
+ ): { tensor: Tensor; owned: boolean };
@@ -3,7 +3,14 @@
3
3
  import { getDevice } from '../../../gpu/device.js';
4
4
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
5
5
  import { log } from '../../../debug/index.js';
6
- import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, tagBufferDtype } from '../../../gpu/weight-buffer.js';
6
+ import {
7
+ getBufferDtype,
8
+ isWeightBuffer,
9
+ isCpuWeightBuffer,
10
+ isGpuBufferInstance,
11
+ tagBufferDtype,
12
+ } from '../../../gpu/weight-buffer.js';
13
+ import { createTensor } from '../../../gpu/tensor.js';
7
14
 
8
15
  // ============================================================================
9
16
  // Type Guards
@@ -116,6 +123,31 @@ export function getNormWeightBuffer(weight, label, config, debugFlags, deviceOve
116
123
  }
117
124
  }
118
125
 
126
+ export function getVectorTensor(weight, label, length, config, debugFlags, deviceOverride = null) {
127
+ if (!Number.isInteger(length) || length <= 0) {
128
+ throw new Error(`Vector tensor "${label}" requires a positive integer length.`);
129
+ }
130
+ const buffer = getNormWeightBuffer(weight, label, config, debugFlags, deviceOverride);
131
+ let dtype = getBufferDtype(buffer);
132
+ if (dtype == null) {
133
+ const bytesPerElement = buffer.size / length;
134
+ if (bytesPerElement === 2) dtype = 'f16';
135
+ else if (bytesPerElement === 4) dtype = 'f32';
136
+ else {
137
+ throw new Error(
138
+ `Vector tensor "${label}" cannot infer dtype from ${buffer.size} bytes for ${length} elements.`
139
+ );
140
+ }
141
+ }
142
+ if (dtype !== 'f16' && dtype !== 'f32') {
143
+ throw new Error(`Vector tensor "${label}" has unsupported dtype "${String(dtype)}".`);
144
+ }
145
+ return {
146
+ tensor: createTensor(buffer, dtype, [length], label),
147
+ owned: !isGpuBufferInstance(weight) && !isWeightBuffer(weight),
148
+ };
149
+ }
150
+
119
151
 
120
152
  function getGPUWeightBuffer(weight, label) {
121
153
  // Handle WeightBuffer by extracting underlying GPUBuffer
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
15
15
  import { PipelineGenerator } from './text/generator.js';
16
16
  import type { Manifest } from './text/config.js';
17
17
  import type { WeightLoadResult, PipelineContexts } from './text/init.js';
18
- import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
18
+ import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming, NativeLoRAPrefillOptions, NativeLoRAPrefillResult } from './text/types.js';
19
19
  import type { ChatMessage } from './text/chat-format.js';
20
20
  import type { LoRAAdapter } from './text/lora.js';
21
21
  import type { DiffusionPipeline } from './diffusion/pipeline.js';
@@ -31,7 +31,7 @@ import type {
31
31
  } from './text/generator.js';
32
32
 
33
33
  // Re-export types for external use
34
- export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
34
+ export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, NativeLoRAPrefillOptions, NativeLoRAPrefillResult };
35
35
  export type { PipelineContexts };
36
36
 
37
37
  export interface ChatRequestInput {
@@ -103,6 +103,11 @@ export declare class InferencePipeline extends PipelineState {
103
103
 
104
104
  prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
105
105
 
106
+ prefillForLoRATraining(
107
+ inputIds: readonly number[] | Int32Array | Uint32Array,
108
+ options: NativeLoRAPrefillOptions
109
+ ): Promise<NativeLoRAPrefillResult>;
110
+
106
111
  computeDiffusionGemmaCanvasLogits(
107
112
  args: DiffusionGemmaCanvasLogitsInput,
108
113
  options?: GenerateOptions & { __internalGenerate?: boolean }
@@ -130,6 +135,8 @@ export declare class InferencePipeline extends PipelineState {
130
135
  embeddingMode: string;
131
136
  }>>;
132
137
 
138
+ encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise<SequenceEncodeResult>;
139
+
133
140
  prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
134
141
 
135
142
  prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{