doppler-gpu 0.4.15 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/README.md +9 -0
  2. package/package.json +25 -2
  3. package/src/cli/doppler-cli.js +253 -94
  4. package/src/cli/runtime-profiles.js +3 -7
  5. package/src/client/doppler-api.d.ts +28 -0
  6. package/src/client/doppler-api.js +2 -0
  7. package/src/client/inspection.d.ts +122 -0
  8. package/src/client/inspection.js +343 -0
  9. package/src/client/runtime/index.d.ts +54 -0
  10. package/src/client/runtime/index.js +190 -18
  11. package/src/client/runtime/model-manager.d.ts +0 -4
  12. package/src/client/runtime/model-manager.js +0 -28
  13. package/src/client/runtime/model-session.d.ts +48 -1
  14. package/src/client/runtime/model-session.js +149 -30
  15. package/src/client/runtime/model-source.d.ts +10 -1
  16. package/src/client/runtime/scoped-session.d.ts +117 -0
  17. package/src/client/runtime/scoped-session.js +257 -0
  18. package/src/client/runtime/types.d.ts +0 -1
  19. package/src/client/runtime/types.js +0 -1
  20. package/src/config/evidence/boundary-evidence-policy.json +54 -0
  21. package/src/config/evidence/source-intake-policy.json +122 -0
  22. package/src/config/inspection/observation-policies.json +80 -0
  23. package/src/config/param-validator.js +21 -36
  24. package/src/config/runtime/compute/f16-activations.json +1 -0
  25. package/src/config/runtime/compute/f16-batched.json +1 -0
  26. package/src/config/runtime/default.json +1 -0
  27. package/src/config/runtime/diagnostics/debug-logits.json +1 -0
  28. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
  29. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
  30. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
  31. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
  32. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
  33. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
  34. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
  35. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
  36. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
  37. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
  38. package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
  39. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
  40. package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
  41. package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
  42. package/src/config/runtime/model/gemma2-debug.json +1 -0
  43. package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
  44. package/src/config/runtime/model/gemma2-pipeline.json +1 -0
  45. package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
  46. package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
  47. package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
  48. package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
  49. package/src/config/runtime/optimization-candidates/registry.json +5 -0
  50. package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
  51. package/src/config/runtime/profiles/default.json +1 -0
  52. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
  53. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
  54. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
  55. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
  56. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
  57. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
  58. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
  59. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
  60. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
  61. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
  62. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
  63. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
  64. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
  65. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
  66. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
  67. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
  68. package/src/config/runtime/profiles/low-memory.json +1 -0
  69. package/src/config/runtime/profiles/production.json +1 -3
  70. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
  71. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
  72. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
  73. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
  74. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
  75. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
  76. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
  77. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
  78. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
  79. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
  80. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
  81. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
  82. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
  83. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
  84. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
  85. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
  86. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
  88. package/src/config/runtime/profiles/rerank-stability.json +1 -3
  89. package/src/config/runtime/profiles/rerank-throughput.json +1 -3
  90. package/src/config/runtime/profiles/simulation.json +1 -3
  91. package/src/config/runtime/profiles/throughput.json +1 -3
  92. package/src/config/runtime/profiles/trace-layers.json +1 -3
  93. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
  94. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
  95. package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
  96. package/src/config/runtime/profiles/turboquant.json +1 -0
  97. package/src/config/runtime/profiles/vector-stability.json +1 -3
  98. package/src/config/runtime/profiles/vector-throughput.json +1 -0
  99. package/src/config/runtime/profiles/verbose-trace.json +1 -3
  100. package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
  101. package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
  102. package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
  103. package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
  104. package/src/config/schema/benchmark.schema.d.ts +7 -0
  105. package/src/config/schema/benchmark.schema.js +5 -0
  106. package/src/config/schema/index.d.ts +8 -16
  107. package/src/config/schema/index.js +5 -8
  108. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
  109. package/src/config/schema/kernel-thresholds.schema.js +0 -12
  110. package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
  111. package/src/config/schema/kernel-warmup.schema.js +0 -1
  112. package/src/config/schema/runtime-profile.schema.d.ts +24 -0
  113. package/src/config/schema/runtime-profile.schema.js +92 -0
  114. package/src/config/schema/shared-runtime.schema.d.ts +1 -4
  115. package/src/config/schema/shared-runtime.schema.js +0 -2
  116. package/src/config/schema/tooling.schema.d.ts +8 -5
  117. package/src/config/schema/tooling.schema.js +3 -2
  118. package/src/debug/tensor.d.ts +2 -0
  119. package/src/debug/tensor.js +21 -2
  120. package/src/gpu/command-recorder.d.ts +5 -0
  121. package/src/gpu/command-recorder.js +17 -0
  122. package/src/gpu/kernel-runtime.js +2 -10
  123. package/src/gpu/kernels/index.d.ts +0 -2
  124. package/src/gpu/kernels/index.js +0 -2
  125. package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
  126. package/src/gpu/kernels/kernel-prewarm.js +60 -0
  127. package/src/gpu/kernels/utils.d.ts +2 -4
  128. package/src/gpu/kernels/utils.js +2 -4
  129. package/src/index.d.ts +13 -1
  130. package/src/index.js +7 -1
  131. package/src/inference/browser-harness-model-helpers.js +4 -0
  132. package/src/inference/browser-harness-runtime-helpers.js +25 -10
  133. package/src/inference/browser-harness-text-helpers.js +4 -1
  134. package/src/inference/browser-harness.d.ts +4 -0
  135. package/src/inference/browser-harness.js +91 -4
  136. package/src/inference/observation-context.d.ts +22 -0
  137. package/src/inference/observation-context.js +41 -0
  138. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  139. package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
  140. package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
  141. package/src/inference/pipelines/text/attention/index.js +7 -0
  142. package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
  143. package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
  144. package/src/inference/pipelines/text/attention/plan.js +328 -0
  145. package/src/inference/pipelines/text/attention/receipt.js +60 -0
  146. package/src/inference/pipelines/text/attention/record.js +8 -1278
  147. package/src/inference/pipelines/text/attention/run.js +8 -1386
  148. package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
  149. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  150. package/src/inference/pipelines/text/generator-steps.js +32 -15
  151. package/src/inference/pipelines/text/generator.js +10 -5
  152. package/src/inference/pipelines/text/init.d.ts +3 -0
  153. package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
  154. package/src/inference/pipelines/text/layer.js +2 -0
  155. package/src/inference/pipelines/text/model-load.d.ts +0 -1
  156. package/src/inference/pipelines/text/model-load.js +2 -11
  157. package/src/inference/pipelines/text/probes.js +10 -0
  158. package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
  159. package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
  160. package/src/inference/pipelines/text/state.d.ts +6 -0
  161. package/src/inference/pipelines/text/state.js +3 -0
  162. package/src/inference/pipelines/text/types.d.ts +6 -0
  163. package/src/inference/pipelines/text.js +16 -1
  164. package/src/inference/refactor-receipt.d.ts +49 -0
  165. package/src/inference/refactor-receipt.js +188 -0
  166. package/src/inference/resource-scope.d.ts +33 -0
  167. package/src/inference/resource-scope.js +168 -0
  168. package/src/inference/test-harness.d.ts +3 -0
  169. package/src/inference/test-harness.js +1 -0
  170. package/src/storage/model-cache.d.ts +53 -0
  171. package/src/storage/model-cache.js +550 -0
  172. package/src/tooling/boundary-evidence.d.ts +51 -0
  173. package/src/tooling/boundary-evidence.js +408 -0
  174. package/src/tooling/browser-command-runner.js +4 -4
  175. package/src/tooling/command-context.d.ts +25 -0
  176. package/src/tooling/command-context.js +53 -0
  177. package/src/tooling/command-runner-shared.js +22 -5
  178. package/src/tooling/demo-receipts.d.ts +56 -0
  179. package/src/tooling/demo-receipts.js +113 -0
  180. package/src/tooling/diagnose-runner.js +2 -2
  181. package/src/tooling/execution-cost-ledger.d.ts +21 -0
  182. package/src/tooling/execution-cost-ledger.js +199 -0
  183. package/src/tooling/node-command-runner.js +2 -2
  184. package/src/tooling/opfs-cache.d.ts +11 -49
  185. package/src/tooling/opfs-cache.js +4 -510
  186. package/src/tooling/precision-replay-math.d.ts +42 -0
  187. package/src/tooling/precision-replay-math.js +174 -0
  188. package/src/tooling/program-bundle-parity.d.ts +3 -0
  189. package/src/tooling/program-bundle-parity.js +8 -0
  190. package/src/tooling/program-bundle.d.ts +1 -52
  191. package/src/tooling/program-bundle.js +0 -139
  192. package/src/tooling/registered-variant-calibration.d.ts +24 -0
  193. package/src/tooling/registered-variant-calibration.js +286 -0
  194. package/src/tooling/runtime-input-composition.js +105 -53
  195. package/src/tooling/runtime-intent.js +41 -0
  196. package/src/tooling/runtime-optimization-index.d.ts +6 -0
  197. package/src/tooling/runtime-optimization-index.js +72 -0
  198. package/src/tooling/runtime-optimization.d.ts +47 -6
  199. package/src/tooling/runtime-optimization.js +559 -40
  200. package/src/tooling/source-intake.d.ts +49 -0
  201. package/src/tooling/source-intake.js +414 -0
  202. package/src/tooling-exports/evidence.d.ts +34 -0
  203. package/src/tooling-exports/evidence.js +34 -0
  204. package/src/tooling-exports/runtime.d.ts +7 -0
  205. package/src/tooling-exports/runtime.js +7 -0
  206. package/src/tooling-exports.d.ts +11 -0
  207. package/src/tooling-exports.js +6 -3
  208. package/src/tooling-exports.shared.d.ts +32 -0
  209. package/src/tooling-exports.shared.js +31 -0
  210. package/src/version.js +1 -1
  211. package/src/config/schema/tuner.schema.d.ts +0 -39
  212. package/src/config/schema/tuner.schema.js +0 -13
  213. package/src/gpu/kernel-tuner/benchmarks.js +0 -902
  214. package/src/gpu/kernel-tuner/cache.js +0 -145
  215. package/src/gpu/kernel-tuner/index.js +0 -38
  216. package/src/gpu/kernel-tuner/tuner.js +0 -247
  217. package/src/gpu/kernel-tuner.js +0 -12
  218. package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
  219. package/src/gpu/kernels/kernel-tuning.js +0 -158
  220. package/src/gpu/profiler.js +0 -382
  221. package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
@@ -1,1386 +1,8 @@
1
-
2
-
3
- import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
- import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
5
- import { getRuntimeConfig } from '../../../../config/runtime.js';
6
- import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
7
- import {
8
- runMatmul,
9
- runRMSNorm,
10
- runLayerNorm,
11
- runBiasAdd,
12
- runRoPE,
13
- canUseRoPEQK,
14
- runRoPEQK,
15
- runAttention,
16
- runAttentionBDPA,
17
- runAttentionTieredQuant,
18
- runAttentionTiered,
19
- runAttentionContiguousQuant,
20
- runSiLU,
21
- castF16ToF32,
22
- castF32ToF16,
23
- runMatmulResidualFused,
24
- shouldUseFusedMatmulResidual,
25
- } from '../../../../gpu/kernel-selector.js';
26
- import { createTensor } from '../../../../gpu/tensor.js';
27
- import { isKernelDebugEnabled, dumpTokenVector, dumpKVCache, logKernelStep } from '../debug-utils.js';
28
- import { applyLoRA } from '../lora-apply.js';
29
- import { getLoRAModule } from '../lora.js';
30
- import { kernelTrace, traceStep } from '../kernel-trace.js';
31
- import { log, trace } from '../../../../debug/index.js';
32
- import { selectRuleValue } from '../../../../rules/rule-registry.js';
33
- import { runProbes } from '../probes.js';
34
- import {
35
- recordAttentionInputs,
36
- shouldForceF32AttentionProjectionForRoPE,
37
- resolveAttentionProjectionOutputDtype,
38
- projectAttentionQKV,
39
- applyAttentionQKNorm,
40
- applyAttentionValueNorm,
41
- hasAttentionProjectionDiagnostics,
42
- hasAttentionStageDiagnostics,
43
- resolveAttentionQKNormState,
44
- } from './projections.js';
45
- import { prepareAttentionProjectionInput } from './output-projection.js';
46
-
47
- import {
48
- shouldDebugLayer,
49
- markStageLogged,
50
- } from './types.js';
51
- import {
52
- getKernelPathMatmulPrecision,
53
- getKernelPathMatmulVariant,
54
- } from '../../../../config/kernel-path-loader.js';
55
- import {
56
- resolveKVCacheState,
57
- createDiffusionGemmaDecoderKVState,
58
- buildAttentionDispatchParams,
59
- buildAttentionInputsData,
60
- } from './dispatch-params.js';
61
- import {
62
- buildTieredQuantAttentionOptions,
63
- buildContiguousQuantAttentionOptions,
64
- } from './quant-options.js';
65
- import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
66
- import {
67
- resolveAttentionPrecisionContract,
68
- isAttentionKvDtypeExplicit,
69
- } from './precision-contract.js';
70
- import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
71
- import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
72
- import { getVectorTensor } from '../weights.js';
73
-
74
- const ATTENTION_DTYPE_LOGGED = new Set();
75
-
76
- function isWideTileQ4KPhaseEnabled(session, phase) {
77
- return phase === 'decode'
78
- ? session?.useWideTileQ4KDecode === true
79
- : session?.useWideTileQ4KPrefill === true;
80
- }
81
-
82
- function canUseQ4KWideTileResidualFusion(options = {}) {
83
- return options.hasResidual === true
84
- && options.oProjDtype === 'q4k'
85
- && options.inputDtype === 'f32'
86
- && options.outputDtype === 'f32'
87
- && options.residualMatches === true
88
- && options.hasLoRA !== true
89
- && getKernelCapabilities().hasF16 === true
90
- && getKernelCapabilities().hasSubgroups === true
91
- && options.session?.useWideTileResidualFusion === true
92
- && options.session?.retainQ4KMaterialization === true
93
- && isWideTileQ4KPhaseEnabled(options.session, options.phase);
94
- }
95
-
96
- function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
97
- assertImplicitDtypeTransitionAllowed({
98
- executionPolicies: state?.executionPolicies ?? null,
99
- fromDtype,
100
- toDtype,
101
- op: 'attention',
102
- detail,
103
- transitionDeclaredBy,
104
- });
105
- }
106
-
107
- export async function runLayerAttentionGPU(
108
- input,
109
- layerWeights,
110
- config,
111
- state,
112
- debug = false,
113
- debugFlags = {},
114
- getWeightBuffer,
115
- getNormWeightBuffer,
116
- debugCheckBuffer,
117
- lora
118
- ) {
119
- const {
120
- layerIdx,
121
- numTokens,
122
- isPrefill,
123
- numHeads,
124
- numKVHeads,
125
- headDim,
126
- hiddenSize,
127
- rmsNormEps,
128
- currentSeqLen,
129
- slidingWindow,
130
- layerType,
131
- residualTensor,
132
- attnSoftcap,
133
- queryPreAttnScalar,
134
- skipInputNorm = false,
135
- tokenIds = null,
136
- kernelPath = null,
137
- disableRoPE = false,
138
- multimodalBidirectionalSpan = null,
139
- sharedKVSourceLayerIdx = null,
140
- storeSharedKV = false,
141
- diffusionGemmaDecoder = false,
142
- precomputedInputNorm = null,
143
- } = config;
144
-
145
- const device = getDevice();
146
- const phase = isPrefill ? 'prefill' : 'decode';
147
- const attentionPrecisionContract = resolveAttentionPrecisionContract(config, state);
148
- const attentionActivationDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
149
- dtype: attentionPrecisionContract.resolvedActivationDtype ?? config.activationDtype,
150
- });
151
- const oProjPrecision = getKernelPathMatmulPrecision('o_proj', phase, layerIdx, kernelPath);
152
- const oProjInputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
153
- dtype: oProjPrecision?.inputDtype ?? attentionActivationDtype,
154
- });
155
- const oProjOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
156
- dtype: oProjPrecision?.outputDtype
157
- ?? attentionPrecisionContract.resolvedOutputDtype
158
- ?? config.activationDtype,
159
- });
160
- const wantsF16Output = oProjOutputDtype === 'f16';
161
- const useF16Activations = attentionActivationDtype === 'f16';
162
- const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
163
- const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
164
- const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
165
- const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
166
- ? 'step_precision'
167
- : null;
168
- let attentionInput = input;
169
- let attentionInputTemp = false;
170
- let normed = attentionInput;
171
- let qTensor = null;
172
- let qGateTensor = null;
173
- let kTensor = null;
174
- let vTensor = null;
175
- let attnOutput = null;
176
- let attnForProjection = null;
177
- let output = null;
178
- let finalOutput = null;
179
- let oProjInputTemp = null;
180
- let valueAliasesKey = false;
181
- let decoderKVState = null;
182
- let decoderKTemp = null;
183
- let decoderVTemp = null;
184
- if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
185
- assertAttentionDtypeTransitionAllowed(
186
- state,
187
- input.dtype,
188
- attentionActivationDtype,
189
- 'The attention kernel selection would widen the input implicitly.',
190
- attentionInputTransitionDeclaredBy
191
- );
192
- attentionInput = attentionActivationDtype === 'f16'
193
- ? await castF32ToF16(input)
194
- : await castF16ToF32(input);
195
- attentionInputTemp = true;
196
- normed = attentionInput;
197
- }
198
-
199
- // Debug: attention input for configured layers
200
- if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'attn_input', debugFlags) && debugCheckBuffer) {
201
- await debugCheckBuffer(attentionInput.buffer, `L${layerIdx} attention input (GPU)`, numTokens, hiddenSize);
202
- }
203
-
204
- // Debug logging moved to debug-utils.ts (enable via setDebugConfig)
205
-
206
- if (!layerWeights) {
207
- const bytesPerElement = wantsF16Output ? 2 : 4;
208
- const outputBuf = acquireBuffer(numTokens * hiddenSize * bytesPerElement, undefined, 'attn_output');
209
- const output = createTensor(outputBuf, oProjOutputDtype, [numTokens, hiddenSize], 'attn_output');
210
- return { output, residualFused: false };
211
- }
212
-
213
- const qSize = numTokens * numHeads * headDim;
214
- const kvSize = numTokens * numKVHeads * headDim;
215
- const sharedKVEntry = sharedKVSourceLayerIdx == null
216
- ? null
217
- : (state.sharedAttentionState?.get(sharedKVSourceLayerIdx) ?? null);
218
- if (sharedKVSourceLayerIdx != null && !sharedKVEntry) {
219
- throw new Error(
220
- `Layer ${layerIdx} requires shared K/V from layer ${sharedKVSourceLayerIdx}, ` +
221
- 'but no shared K/V state was stored for that source layer.'
222
- );
223
- }
224
- if (sharedKVEntry && (
225
- sharedKVEntry.headDim !== headDim
226
- || sharedKVEntry.numKVHeads !== numKVHeads
227
- )) {
228
- throw new Error(
229
- `Layer ${layerIdx} shared K/V geometry mismatch. ` +
230
- `Expected numKVHeads=${numKVHeads}, headDim=${headDim}; ` +
231
- `got numKVHeads=${sharedKVEntry.numKVHeads}, headDim=${sharedKVEntry.headDim}.`
232
- );
233
- }
234
- const reusesSharedKV = sharedKVEntry != null;
235
-
236
- // 1. Input norm
237
- // Opt-in fusion: when useFusedRmsnormWideTile is set AND preconditions match,
238
- // defer the rmsnorm into each q/k/v_proj matmul via a dedicated WideTile
239
- // variant. Saves one standalone rmsnorm dispatch per layer.
240
- let fusedNormWeight = null;
241
- let fusedNormEps = null;
242
- let fusedNormOffset = false;
243
- let fusedNormWeightOwned = false;
244
- // state.runtimeConfig.inference.session is a stale snapshot ({}); the live
245
- // module-level runtime config carries the merged profile/override session.
246
- const rmsNormFusionFlag = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
247
- const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
248
- const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
249
- const canFuseInputNormIntoProjections = rmsNormFusionFlag
250
- && config.normalizationType === 'rmsnorm'
251
- && hasInputNormWeight
252
- && canSelectFusedRmsNormProjection
253
- && !skipInputNorm
254
- && isPrefill
255
- && numTokens > 1
256
- && attentionInput.dtype === 'f32';
257
-
258
- try {
259
- if (precomputedInputNorm) {
260
- if (
261
- isPrefill
262
- || numTokens !== 1
263
- || skipInputNorm
264
- || !layerWeights.inputNorm
265
- || !getNormWeightBuffer
266
- || precomputedInputNorm.dtype !== attentionInput.dtype
267
- || precomputedInputNorm.dtype !== 'f32'
268
- ) {
269
- throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
270
- }
271
- normed = precomputedInputNorm;
272
- await runProbes('post_input_norm', normed.buffer, {
273
- layerIdx,
274
- numTokens,
275
- hiddenSize,
276
- probes: state.debugProbes,
277
- operatorDiagnostics: state.operatorDiagnostics,
278
- dtype: normed.dtype,
279
- });
280
- } else if (canFuseInputNormIntoProjections) {
281
- const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
282
- fusedNormWeight = normWeightBuf;
283
- fusedNormEps = rmsNormEps;
284
- fusedNormOffset = config.rmsNormWeightOffset === true;
285
- fusedNormWeightOwned = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
286
- // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs the
287
- // norm internally via the fused kernel variant.
288
- } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
289
- const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
290
- let normBiasTensor = null;
291
- let normBiasOwned = false;
292
- try {
293
- // Debug: norm weights for configured layers
294
- if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'norm_weights', debugFlags) && debugCheckBuffer) {
295
- await debugCheckBuffer(normWeightBuf, `L${layerIdx} input norm weights (GPU)`, 1, hiddenSize);
296
- }
297
-
298
- if (config.normalizationType === 'layernorm') {
299
- if (!layerWeights.inputNormBias) {
300
- throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
301
- }
302
- ({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
303
- layerWeights.inputNormBias,
304
- `L${layerIdx}.input_norm_bias`,
305
- hiddenSize,
306
- {},
307
- debugFlags
308
- ));
309
- normed = await runLayerNorm(
310
- attentionInput,
311
- normWeightBuf,
312
- normBiasTensor.buffer,
313
- rmsNormEps,
314
- { batchSize: numTokens, hiddenSize }
315
- );
316
- } else {
317
- normed = await runRMSNorm(attentionInput, normWeightBuf, rmsNormEps, {
318
- batchSize: numTokens,
319
- hiddenSize,
320
- rmsNormWeightOffset: config.rmsNormWeightOffset,
321
- });
322
- }
323
- } finally {
324
- if (normBiasOwned && normBiasTensor) releaseBuffer(normBiasTensor.buffer);
325
- if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseBuffer(normWeightBuf);
326
- }
327
-
328
- // Trace input norm output
329
- if (kernelTrace.enabled) {
330
- await traceStep('rmsnorm', `L${layerIdx}.input_norm`, layerIdx, normed.buffer, [numTokens, hiddenSize]);
331
- }
332
-
333
- if (isKernelDebugEnabled(layerIdx)) {
334
- logKernelStep('rmsnorm', { layerIdx, label: 'input_norm', size: numTokens * hiddenSize });
335
- await dumpTokenVector(normed.buffer, 'input_norm_out', {
336
- layerIdx,
337
- tokenIdx: Math.max(0, numTokens - 1),
338
- rowSize: hiddenSize,
339
- dtype: normed.dtype,
340
- });
341
- }
342
-
343
- await runProbes('post_input_norm', normed.buffer, {
344
- layerIdx,
345
- numTokens,
346
- hiddenSize,
347
- probes: state.debugProbes,
348
- operatorDiagnostics: state.operatorDiagnostics,
349
- dtype: normed.dtype,
350
- });
351
- }
352
-
353
- // Debug: Check normed input for L0 prefill
354
- // Debug: normed input for configured layers
355
- if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'attn_normed', debugFlags) && debugCheckBuffer) {
356
- await debugCheckBuffer(normed.buffer, `L${layerIdx} normed input (GPU)`, numTokens);
357
- }
358
-
359
- const debugLayers = debugFlags.debugLayers;
360
- const shouldLogLayer = debugLayers === null ? layerIdx === 0 : shouldDebugLayer(layerIdx, debugLayers);
361
- if (shouldLogLayer) {
362
- const phase = selectRuleValue('kernels', 'attention', 'phase', { isDecode: !isPrefill });
363
- const logKey = `L${layerIdx}_${phase}_dtypes`;
364
- if (!ATTENTION_DTYPE_LOGGED.has(logKey)) {
365
- ATTENTION_DTYPE_LOGGED.add(logKey);
366
- trace.attn(layerIdx, `dtypes: activation=${attentionActivationDtype}, input=${input.dtype}, normed=${normed.dtype}`);
367
- }
368
- }
369
-
370
- if (isKernelDebugEnabled(layerIdx)) {
371
- await dumpTokenVector(normed.buffer, 'attn_in', {
372
- layerIdx,
373
- tokenIdx: Math.max(0, numTokens - 1),
374
- rowSize: hiddenSize,
375
- dtype: normed.dtype,
376
- });
377
- }
378
-
379
- // 2. Q/K/V projections
380
- const qProjVariant = getKernelPathMatmulVariant('q_proj', phase, layerIdx, kernelPath);
381
- const kernelPathIsF16 = qProjVariant != null && qProjVariant.includes('f16') && !qProjVariant.includes('f32');
382
- const matmulOutputDtype = resolveAttentionProjectionOutputDtype(attentionActivationDtype, {
383
- forceF32: shouldForceF32AttentionProjectionForRoPE({
384
- attentionInputDtype: attentionActivationDtype,
385
- headDim,
386
- rotaryDim: config.ropeRotaryDim,
387
- interleaved: config.ropeInterleaved,
388
- kernelPathIsF16,
389
- }),
390
- });
391
- let usedFusedQKV = false;
392
- let qkNormApplied = false;
393
- let ropeApplied = false;
394
- const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
395
- const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
396
- || kernelTrace.enabled
397
- || isKernelDebugEnabled(layerIdx)
398
- || (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
399
- const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
400
- state,
401
- ['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
402
- )
403
- || kernelTrace.enabled
404
- || isKernelDebugEnabled(layerIdx)
405
- || (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
406
- const runtimeSession = getRuntimeConfig()?.inference?.session;
407
- const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
408
- const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
409
- ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied } = await projectAttentionQKV({
410
- recorder: null,
411
- normed,
412
- layerWeights,
413
- numTokens,
414
- numHeads,
415
- numKVHeads,
416
- headDim,
417
- hiddenSize,
418
- layerIdx,
419
- kernelPath,
420
- matmulOutputDtype,
421
- getWeightBuffer,
422
- lora,
423
- matmulDebug: state.runtimeConfig?.shared?.debug?.matmul ?? null,
424
- attentionOutputGate: config.attentionOutputGate === true,
425
- sharedKTensor: sharedKVEntry?.kTensor ?? null,
426
- sharedVTensor: sharedKVEntry?.vTensor ?? null,
427
- executionPolicies: state.executionPolicies ?? null,
428
- releaseTemporary: (buffer) => releaseBuffer(buffer),
429
- onFusedQKV: layerIdx === 0 && isPrefill
430
- ? ({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize }) => {
431
- trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
432
- }
433
- : null,
434
- fusedNormWeight,
435
- fusedNormEps,
436
- fusedNormOffset,
437
- qkNormFusion: {
438
- enabled: qkNormFusionFlag
439
- && qkNormState.wantsQKNorm
440
- && config.queryKeyNormType === 'rmsnorm'
441
- && config.queryKeyNormAxis === 'head',
442
- getNormWeightBuffer,
443
- rmsNormEps,
444
- rmsNormWeightOffset: config.rmsNormWeightOffset,
445
- skipKNorm: qkNormState.skipKNorm,
446
- allowUnitQKNorm: qkNormState.allowUnitQKNorm,
447
- projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
448
- },
449
- qkNormRoPEFusion: {
450
- enabled: qkNormRoPEFusionFlag
451
- && qkNormState.wantsQKNorm
452
- && config.queryKeyNormType === 'rmsnorm'
453
- && config.queryKeyNormAxis === 'head'
454
- && !disableRoPE
455
- && !!state.ropeFreqsCos
456
- && !!state.ropeFreqsSin,
457
- getNormWeightBuffer,
458
- rmsNormEps,
459
- rmsNormWeightOffset: config.rmsNormWeightOffset,
460
- skipKNorm: qkNormState.skipKNorm,
461
- allowUnitQKNorm: qkNormState.allowUnitQKNorm,
462
- projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
463
- freqsCos: state.ropeFreqsCos,
464
- freqsSin: state.ropeFreqsSin,
465
- startPos: currentSeqLen,
466
- rotaryDim: config.ropeRotaryDim,
467
- pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
468
- interleaved: config.ropeInterleaved,
469
- reusesSharedKV,
470
- },
471
- }));
472
- // Release the in-flight norm weight buffer used by the fused projections.
473
- if (fusedNormWeight && fusedNormWeightOwned) {
474
- releaseBuffer(fusedNormWeight);
475
- }
476
-
477
- // Trace Q/K/V projections
478
- if (kernelTrace.enabled) {
479
- await traceStep('matmul', `L${layerIdx}.q_proj`, layerIdx, qTensor.buffer, [numTokens, numHeads * headDim]);
480
- await traceStep('matmul', `L${layerIdx}.k_proj`, layerIdx, kTensor.buffer, [numTokens, numKVHeads * headDim]);
481
- await traceStep('matmul', `L${layerIdx}.v_proj`, layerIdx, vTensor.buffer, [numTokens, numKVHeads * headDim]);
482
- }
483
- await runProbes('q_proj', qTensor.buffer, {
484
- layerIdx,
485
- numTokens,
486
- hiddenSize: numHeads * headDim,
487
- probes: state.debugProbes,
488
- operatorDiagnostics: state.operatorDiagnostics,
489
- dtype: qTensor.dtype,
490
- });
491
- if (!reusesSharedKV) {
492
- await runProbes('k_proj', kTensor.buffer, {
493
- layerIdx,
494
- numTokens,
495
- hiddenSize: numKVHeads * headDim,
496
- probes: state.debugProbes,
497
- operatorDiagnostics: state.operatorDiagnostics,
498
- dtype: kTensor.dtype,
499
- });
500
- await runProbes('v_proj', vTensor.buffer, {
501
- layerIdx,
502
- numTokens,
503
- hiddenSize: numKVHeads * headDim,
504
- probes: state.debugProbes,
505
- operatorDiagnostics: state.operatorDiagnostics,
506
- dtype: vTensor.dtype,
507
- });
508
- }
509
-
510
- // Kernel step debug: Q/K/V projections
511
- if (isKernelDebugEnabled(layerIdx)) {
512
- logKernelStep('matmul', { layerIdx, label: 'Q_proj', M: numTokens, N: numHeads * headDim, K: hiddenSize });
513
- await dumpTokenVector(qTensor.buffer, 'Q_proj', {
514
- layerIdx,
515
- tokenIdx: Math.max(0, numTokens - 1),
516
- rowSize: numHeads * headDim,
517
- dtype: qTensor.dtype,
518
- });
519
- logKernelStep('matmul', { layerIdx, label: 'K_proj', M: numTokens, N: numKVHeads * headDim, K: hiddenSize });
520
- await dumpTokenVector(kTensor.buffer, 'K_proj', {
521
- layerIdx,
522
- tokenIdx: Math.max(0, numTokens - 1),
523
- rowSize: numKVHeads * headDim,
524
- dtype: kTensor.dtype,
525
- });
526
- logKernelStep('matmul', { layerIdx, label: 'V_proj', M: numTokens, N: numKVHeads * headDim, K: hiddenSize });
527
- await dumpTokenVector(vTensor.buffer, 'V_proj', {
528
- layerIdx,
529
- tokenIdx: Math.max(0, numTokens - 1),
530
- rowSize: numKVHeads * headDim,
531
- dtype: vTensor.dtype,
532
- });
533
- }
534
-
535
- // Debug: Check Q/K/V after projections for L0 prefill
536
- // Debug: Q/K/V projections for configured layers
537
- if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'qkv_proj', debugFlags) && debugCheckBuffer) {
538
- await debugCheckBuffer(qTensor.buffer, `L${layerIdx} Q after proj (GPU)`, numTokens, numHeads * headDim);
539
- await debugCheckBuffer(kTensor.buffer, `L${layerIdx} K after proj (GPU)`, numTokens, numKVHeads * headDim);
540
- await debugCheckBuffer(vTensor.buffer, `L${layerIdx} V after proj (GPU)`, numTokens, numKVHeads * headDim);
541
- }
542
-
543
- // Optional Q/K normalization; the manifest owns the axis and norm family.
544
- if (isKernelDebugEnabled(layerIdx)) {
545
- logKernelStep(
546
- 'qk_norm',
547
- {
548
- layerIdx,
549
- label: `hasQ=${qkNormState.hasQNorm} hasK=${qkNormState.hasKNorm} ` +
550
- `wants=${qkNormState.wantsQKNorm} unit=${qkNormState.allowUnitQKNorm}`,
551
- }
552
- );
553
- }
554
-
555
- if (qkNormState.wantsQKNorm && !qkNormApplied) {
556
- ({ qTensor, kTensor } = await applyAttentionQKNorm({
557
- recorder: null,
558
- qTensor,
559
- kTensor,
560
- layerWeights,
561
- getNormWeightBuffer,
562
- rmsNormEps,
563
- numTokens,
564
- numHeads,
565
- numKVHeads,
566
- headDim,
567
- rmsNormWeightOffset: config.rmsNormWeightOffset,
568
- normalizationType: config.queryKeyNormType,
569
- normalizationAxis: config.queryKeyNormAxis,
570
- releaseTemporary: (buffer) => releaseBuffer(buffer),
571
- skipKNorm: qkNormState.skipKNorm,
572
- onQNormApplied: isKernelDebugEnabled(layerIdx)
573
- ? async (tensor) => {
574
- await dumpTokenVector(tensor.buffer, 'Q_norm', {
575
- layerIdx,
576
- tokenIdx: Math.max(0, numTokens - 1),
577
- rowSize: numHeads * headDim,
578
- dtype: tensor.dtype,
579
- });
580
- }
581
- : null,
582
- onKNormApplied: isKernelDebugEnabled(layerIdx)
583
- ? async (tensor) => {
584
- await dumpTokenVector(tensor.buffer, 'K_norm', {
585
- layerIdx,
586
- tokenIdx: Math.max(0, numTokens - 1),
587
- rowSize: numKVHeads * headDim,
588
- dtype: tensor.dtype,
589
- });
590
- }
591
- : null,
592
- retainKInput: valueAliasesKey,
593
- allowUnitQKNorm: qkNormState.allowUnitQKNorm,
594
- }));
595
- }
596
-
597
- if (config.valueNorm === true && !reusesSharedKV) {
598
- const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
599
- vTensor = await applyAttentionValueNorm({
600
- recorder: null,
601
- vTensor,
602
- rmsNormEps,
603
- numTokens,
604
- numKVHeads,
605
- headDim,
606
- releaseTemporary: (buffer) => {
607
- if (!valueNormInputAliasesKey) {
608
- releaseBuffer(buffer);
609
- }
610
- },
611
- });
612
- await runProbes('v_norm', vTensor.buffer, {
613
- layerIdx,
614
- numTokens,
615
- hiddenSize: numKVHeads * headDim,
616
- probes: state.debugProbes,
617
- operatorDiagnostics: state.operatorDiagnostics,
618
- dtype: vTensor.dtype,
619
- });
620
- }
621
-
622
- if (qkNormState.wantsQKNorm) {
623
- await runProbes('q_norm', qTensor.buffer, {
624
- layerIdx,
625
- numTokens,
626
- hiddenSize: numHeads * headDim,
627
- probes: state.debugProbes,
628
- operatorDiagnostics: state.operatorDiagnostics,
629
- dtype: qTensor.dtype,
630
- });
631
- if (!reusesSharedKV) {
632
- await runProbes('k_norm', kTensor.buffer, {
633
- layerIdx,
634
- numTokens,
635
- hiddenSize: numKVHeads * headDim,
636
- probes: state.debugProbes,
637
- operatorDiagnostics: state.operatorDiagnostics,
638
- dtype: kTensor.dtype,
639
- });
640
- }
641
- }
642
-
643
- if (normed !== attentionInput) releaseBuffer(normed.buffer);
644
- if (attentionInputTemp) releaseBuffer(attentionInput.buffer);
645
-
646
- // 3. RoPE (modifies tensor in-place)
647
-
648
- if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
649
- const ropeOptions = {
650
- headDim,
651
- rotaryDim: config.ropeRotaryDim,
652
- pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
653
- interleaved: config.ropeInterleaved,
654
- startPos: currentSeqLen,
655
- executionPolicies: state.executionPolicies ?? null,
656
- };
657
- if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
658
- await runRoPEQK(qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
659
- ...ropeOptions,
660
- numQHeads: numHeads,
661
- numKVHeads,
662
- });
663
- } else {
664
- await runRoPE(qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
665
- ...ropeOptions,
666
- numHeads,
667
- });
668
- if (!reusesSharedKV) {
669
- await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
670
- ...ropeOptions,
671
- numHeads: numKVHeads,
672
- });
673
- }
674
- }
675
-
676
- // Trace RoPE outputs
677
- if (kernelTrace.enabled) {
678
- await traceStep('rope', `L${layerIdx}.q_rope`, layerIdx, qTensor.buffer, [numTokens, numHeads * headDim]);
679
- if (!reusesSharedKV) {
680
- await traceStep('rope', `L${layerIdx}.k_rope`, layerIdx, kTensor.buffer, [numTokens, numKVHeads * headDim]);
681
- }
682
- }
683
- }
684
- await runProbes('q_rope', qTensor.buffer, {
685
- layerIdx,
686
- numTokens,
687
- hiddenSize: numHeads * headDim,
688
- probes: state.debugProbes,
689
- operatorDiagnostics: state.operatorDiagnostics,
690
- dtype: qTensor.dtype,
691
- });
692
- await runProbes('k_rope', kTensor.buffer, {
693
- layerIdx,
694
- numTokens,
695
- hiddenSize: numKVHeads * headDim,
696
- probes: state.debugProbes,
697
- operatorDiagnostics: state.operatorDiagnostics,
698
- dtype: kTensor.dtype,
699
- });
700
- if (isKernelDebugEnabled(layerIdx)) {
701
- logKernelStep('rope', { layerIdx, label: `startPos=${currentSeqLen}` });
702
- await dumpTokenVector(qTensor.buffer, 'Q_rope', {
703
- layerIdx,
704
- tokenIdx: Math.max(0, numTokens - 1),
705
- rowSize: numHeads * headDim,
706
- dtype: qTensor.dtype,
707
- });
708
- await dumpTokenVector(kTensor.buffer, 'K_rope', {
709
- layerIdx,
710
- tokenIdx: Math.max(0, numTokens - 1),
711
- rowSize: numKVHeads * headDim,
712
- dtype: kTensor.dtype,
713
- });
714
- }
715
-
716
- if (storeSharedKV && state.sharedAttentionState) {
717
- state.sharedAttentionState.set(layerIdx, {
718
- kTensor,
719
- vTensor,
720
- headDim,
721
- numKVHeads,
722
- });
723
- }
724
-
725
- // Debug: Check Q/K after RoPE for L0 prefill
726
- // Debug: Q/K after RoPE for configured layers
727
- if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'qk_rope', debugFlags) && debugCheckBuffer) {
728
- await debugCheckBuffer(qTensor.buffer, `L${layerIdx} Q after RoPE (GPU)`, numTokens, numHeads * headDim);
729
- await debugCheckBuffer(kTensor.buffer, `L${layerIdx} K after RoPE (GPU)`, numTokens, numKVHeads * headDim);
730
- }
731
-
732
- // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
733
-
734
- if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
735
- if (state.kvCache.kvDtype === 'f16') {
736
- const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
737
- if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
738
- assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
739
- }
740
- if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
741
- assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
742
- }
743
- const kCasted = kTensor.dtype === 'f16' ? kTensor : await castF32ToF16(kTensor);
744
- const vCasted = vTensor.dtype === 'f16' ? vTensor : await castF32ToF16(vTensor);
745
-
746
- await state.kvCache.updateFromGPU(layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
747
-
748
- if (kTensor.dtype !== 'f16') releaseBuffer(kCasted.buffer);
749
- if (vTensor.dtype !== 'f16') releaseBuffer(vCasted.buffer);
750
- } else {
751
- await state.kvCache.updateFromGPU(layerIdx, kTensor.buffer, vTensor.buffer, currentSeqLen, numTokens, tokenIds);
752
- }
753
-
754
- // Kernel step debug: KV cache state after update
755
- if (isKernelDebugEnabled(layerIdx)) {
756
- trace.kv(layerIdx, `KV cache updated: startPos=${currentSeqLen}, numTokens=${numTokens}`);
757
- await dumpKVCache(( (state.kvCache)), layerIdx);
758
- }
759
- }
760
-
761
- // Resolve KV cache state and build dispatch parameters (shared with record.js)
762
- let kvState;
763
- if (diffusionGemmaDecoder) {
764
- const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
765
- let decoderKTensor = kTensor;
766
- let decoderVTensor = vTensor;
767
- if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
768
- const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
769
- if (!hasExplicitF16KvContract) {
770
- assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
771
- }
772
- decoderKTensor = await castF32ToF16(decoderKTensor);
773
- decoderKTemp = decoderKTensor;
774
- } else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
775
- assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
776
- decoderKTensor = await castF16ToF32(decoderKTensor);
777
- decoderKTemp = decoderKTensor;
778
- }
779
- if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
780
- const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
781
- if (!hasExplicitF16KvContract) {
782
- assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
783
- }
784
- decoderVTensor = await castF32ToF16(decoderVTensor);
785
- decoderVTemp = decoderVTensor;
786
- } else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
787
- assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
788
- decoderVTensor = await castF16ToF32(decoderVTensor);
789
- decoderVTemp = decoderVTensor;
790
- }
791
- kvState = await createDiffusionGemmaDecoderKVState({
792
- state,
793
- layerIdx,
794
- kTensor: decoderKTensor,
795
- vTensor: decoderVTensor,
796
- currentSeqLen,
797
- numTokens,
798
- numKVHeads,
799
- headDim,
800
- layerType,
801
- slidingWindow,
802
- kvDtype: decoderKVDtype,
803
- });
804
- if (decoderKTemp) {
805
- releaseBuffer(decoderKTemp.buffer);
806
- decoderKTemp = null;
807
- }
808
- if (decoderVTemp && decoderVTemp.buffer !== decoderKTensor.buffer) {
809
- releaseBuffer(decoderVTemp.buffer);
810
- }
811
- decoderVTemp = null;
812
- } else {
813
- kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
814
- }
815
- const dispatchConfig = {
816
- layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
817
- slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
818
- layerType,
819
- layerTypes: config.layerTypes,
820
- queryPreAttnScalar,
821
- causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
822
- activationDtype: attentionActivationDtype,
823
- kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
824
- };
825
- const dispatchParams = buildAttentionDispatchParams(dispatchConfig, state, kTensor, vTensor, kvState);
826
- const {
827
- effectiveSlidingWindow, attentionKernelVariant, attnScale,
828
- cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
829
- prefillFallbackNeedsCast, causalForAttention,
830
- } = dispatchParams;
831
- const mergedSession = getRuntimeConfig()?.inference?.session;
832
- let attentionOutputGateFused = false;
833
-
834
- // 5. Attention (uses raw GPUBuffers)
835
-
836
- // Debug: log scale on layer 0
837
- if (layerIdx === 0 && isPrefill) {
838
- trace.attn(layerIdx, `Attention scale=${attnScale.toFixed(6)}, queryPreAttnScalar=${queryPreAttnScalar ?? 'undefined'}, headDim=${headDim}`);
839
- }
840
-
841
- // Kernel step debug: KV cache state after resolve
842
- if (isKernelDebugEnabled(layerIdx)) {
843
- trace.kv(layerIdx, `KV resolved: kvLen=${kvState.kvLenForAttention}, layout=${kvState.kvLayout}`);
844
- }
845
-
846
- recordAttentionInputs(state, buildAttentionInputsData(
847
- dispatchConfig, input, normed, kvState, dispatchParams,
848
- { useF16Activations, matmulOutputDtype },
849
- usedFusedQKV, qTensor, kTensor, vTensor,
850
- ));
851
-
852
- const attentionKernelRunners = {
853
- bdpa: async () => {
854
- const BDPA_MAX_HEAD_DIM = 256;
855
- const BDPA_MAX_KV_LEN = 2048;
856
- if (numTokens !== 1) {
857
- throw new Error(`BDPA attention supports decode-only seqLen=1; got seqLen=${numTokens}.`);
858
- }
859
- if (headDim > BDPA_MAX_HEAD_DIM) {
860
- throw new Error(
861
- `BDPA attention kernel supports headDim <= ${BDPA_MAX_HEAD_DIM}; got headDim=${headDim}.`
862
- );
863
- }
864
- if (kvState.kvLenForAttention > BDPA_MAX_KV_LEN) {
865
- throw new Error(
866
- `BDPA attention kernel supports kvLen <= ${BDPA_MAX_KV_LEN}; got kvLen=${kvState.kvLenForAttention}.`
867
- );
868
- }
869
- const basisKDtype = 'f16';
870
- const basisVDtype = 'f16';
871
- const basisCount = Math.max(1, kvState.bdpaBasisCount);
872
- const basisKTensor = createTensor(kvState.bdpaBasisK, basisKDtype, [basisCount, numKVHeads * headDim], 'bdpa_basis_k');
873
- const basisVTensor = createTensor(kvState.bdpaBasisV, basisVDtype, [basisCount, numKVHeads * headDim], 'bdpa_basis_v');
874
- let qForBDPA = qTensor;
875
- if (qForBDPA.dtype !== 'f16') {
876
- assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f16', 'Q would be narrowed implicitly for BDPA.');
877
- qForBDPA = await castF32ToF16(qTensor);
878
- }
879
- const output = await runAttentionBDPA(qForBDPA, basisKTensor, basisVTensor, kvState.bdpaPagedK, kvState.bdpaPagedV, kvState.bdpaIndex, numHeads, headDim, {
880
- seqLen: numTokens,
881
- kvLen: kvState.kvLenForAttention,
882
- numKVHeads,
883
- causal: causalForAttention,
884
- startPos: kvState.startPosForMask,
885
- layerIdx,
886
- slidingWindow: effectiveSlidingWindow,
887
- attnSoftcap,
888
- scale: attnScale,
889
- ropeCos: state.ropeFreqsCos,
890
- ropeSin: state.ropeFreqsSin,
891
- });
892
- if (qForBDPA !== qTensor) {
893
- releaseBuffer(qForBDPA.buffer);
894
- }
895
- return output;
896
- },
897
- tieredQuant: async () => {
898
- let qForAttention = qTensor;
899
- let qTemp = null;
900
- if (kvState.coldQuantMode !== 'none' && qTensor.dtype !== 'f32') {
901
- assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Q would be widened implicitly for tiered quant attention.');
902
- qForAttention = await castF16ToF32(qTensor);
903
- qTemp = qForAttention;
904
- }
905
- const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
906
- const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
907
-
908
- if (kvState.coldQuantMode === 'none') {
909
- throw new Error('Tiered quant attention requires cold quant mode.');
910
- }
911
- if (!kvState.coldScalesK || !kvState.coldScalesV) {
912
- throw new Error('Tiered quant attention requires cold scale buffers.');
913
- }
914
-
915
- const output = await runAttentionTieredQuant(
916
- qForAttention,
917
- cachedHotKTensor,
918
- cachedHotVTensor,
919
- kvState.cachedKCold,
920
- kvState.cachedVCold,
921
- kvState.coldScalesK,
922
- kvState.coldScalesV,
923
- numHeads,
924
- headDim,
925
- buildTieredQuantAttentionOptions(kvState, {
926
- seqLen: numTokens,
927
- numKVHeads,
928
- causal: causalForAttention,
929
- startPos: kvState.startPosForMask,
930
- slidingWindow: effectiveSlidingWindow ?? 0,
931
- attnSoftcap,
932
- scale: attnScale,
933
- })
934
- );
935
-
936
- if (qTemp) {
937
- releaseBuffer(qTemp.buffer);
938
- }
939
- return output;
940
- },
941
- contiguousQuant: async () => {
942
- let qForAttention = qTensor;
943
- let qTemp = null;
944
- if (qTensor.dtype !== 'f32') {
945
- assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Q would be widened implicitly for contiguous quant attention.');
946
- qForAttention = await castF16ToF32(qTensor);
947
- qTemp = qForAttention;
948
- }
949
-
950
- if (!kvState.coldScalesK || !kvState.coldScalesV) {
951
- throw new Error('Contiguous quant attention requires scale buffers.');
952
- }
953
- if (!kvState.rotationMatrixBuffer || !kvState.codebookCentroidsBuffer) {
954
- throw new Error('Contiguous quant attention requires TurboQuant shared buffers.');
955
- }
956
-
957
- const output = await runAttentionContiguousQuant(
958
- qForAttention,
959
- kvState.cachedKCold,
960
- kvState.cachedVCold,
961
- kvState.coldScalesK,
962
- kvState.coldScalesV,
963
- numHeads,
964
- headDim,
965
- buildContiguousQuantAttentionOptions(kvState, {
966
- seqLen: numTokens,
967
- kvLen: kvState.kvLenForAttention,
968
- numKVHeads,
969
- causal: causalForAttention,
970
- startPos: kvState.startPosForMask,
971
- slidingWindow: effectiveSlidingWindow ?? 0,
972
- attnSoftcap,
973
- scale: attnScale,
974
- })
975
- );
976
-
977
- if (qTemp) {
978
- releaseBuffer(qTemp.buffer);
979
- }
980
- return output;
981
- },
982
- tiered: async () => {
983
- const qForAttention = qTensor;
984
- const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
985
- const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
986
- const cachedColdKTensor = createTensor(kvState.cachedKCold, cachedKDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_K_cold');
987
- const cachedColdVTensor = createTensor(kvState.cachedVCold, cachedVDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_V_cold');
988
- return runAttentionTiered(qForAttention, cachedHotKTensor, cachedHotVTensor, cachedColdKTensor, cachedColdVTensor, numHeads, headDim, {
989
- seqLen: numTokens,
990
- coldLen: kvState.coldLen,
991
- hotLen: kvState.hotLen,
992
- numKVHeads,
993
- causal: causalForAttention,
994
- startPos: kvState.startPosForMask,
995
- slidingWindow: effectiveSlidingWindow ?? 0,
996
- attnSoftcap,
997
- scale: attnScale,
998
- hotWindow: kvState.hotWindow,
999
- hotStart: kvState.hotStart,
1000
- coldPageTable: kvState.coldPageTable,
1001
- coldPageSize: kvState.coldPageSize,
1002
- coldLayout: kvState.coldPageTable ? 2 : 0,
1003
- hotLayout: kvState.hotWindow > 0 ? 1 : 0,
1004
- });
1005
- },
1006
- contiguous: async () => {
1007
- // Prefill fallback: quantized/tiered layouts use raw K/V for prefill, cast to f16 to match kernel path
1008
- let kForAttn = cachedKTensor;
1009
- let vForAttn = cachedVTensor;
1010
- if (prefillFallbackNeedsCast) {
1011
- const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
1012
- if (cachedKDtype === 'f16' && kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
1013
- assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for prefill fallback.');
1014
- }
1015
- if (cachedVDtype === 'f16' && vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
1016
- assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for prefill fallback.');
1017
- }
1018
- const kCasted = cachedKDtype === 'f16' && kTensor.dtype !== 'f16'
1019
- ? await castF32ToF16(kTensor) : kTensor;
1020
- const vCasted = cachedVDtype === 'f16' && vTensor.dtype !== 'f16'
1021
- ? await castF32ToF16(vTensor) : vTensor;
1022
- kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
1023
- vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
1024
- }
1025
- // Session precedence is runtime-over-manifest per config-style-guide
1026
- // §Category Rules: `resolvedSession = merge(manifest.session, runtime.session)`
1027
- // with runtime fields winning. getRuntimeConfig() returns that merged
1028
- // session (state.runtimeConfig.inference.session is a stale snapshot —
1029
- // see comment above at the rmsNorm fusion check). The kernel itself
1030
- // enforces remaining preconditions (head_dim=256, etc.).
1031
- const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
1032
- const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
1033
- const useOutputGateFusion = canUseAttentionOutputGateFusion({
1034
- session: mergedSession,
1035
- qGateTensor,
1036
- numTokens,
1037
- numHeads,
1038
- headDim,
1039
- cachedKDtype,
1040
- cachedVDtype,
1041
- kernelPath,
1042
- diffusionGemmaDecoder,
1043
- });
1044
- const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
1045
- seqLen: numTokens,
1046
- kvLen: kvState.kvLenForAttention,
1047
- numKVHeads,
1048
- causal: causalForAttention,
1049
- bidirectionalSpanStart: multimodalBidirectionalSpan?.start ?? 0,
1050
- bidirectionalSpanLength: multimodalBidirectionalSpan?.length ?? 0,
1051
- startPos: kvState.startPosForMask,
1052
- layerIdx,
1053
- slidingWindow: effectiveSlidingWindow,
1054
- attnSoftcap,
1055
- scale: attnScale,
1056
- kvStart: kvState.kvStart,
1057
- kvLayout: kvState.kvLayout,
1058
- kvPageTable: kvState.kvPageTable,
1059
- kvPageSize: kvState.kvPageSize,
1060
- kernelPath,
1061
- useFlashPrefill,
1062
- useOrtFlashPrefill,
1063
- outputGate: useOutputGateFusion ? qGateTensor : null,
1064
- });
1065
- attentionOutputGateFused = result?.outputGateFused === true;
1066
- if (prefillFallbackNeedsCast) {
1067
- if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
1068
- if (vTensor.dtype !== 'f16') releaseBuffer(vForAttn.buffer);
1069
- }
1070
- return result;
1071
- },
1072
- };
1073
-
1074
- const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
1075
- if (!runAttentionKernel) {
1076
- throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
1077
- }
1078
-
1079
- try {
1080
- attnOutput = await runAttentionKernel();
1081
- } finally {
1082
- if (decoderKVState?.ownedBuffers) {
1083
- for (const buffer of decoderKVState.ownedBuffers) {
1084
- releaseBuffer(buffer);
1085
- }
1086
- decoderKVState.ownedBuffers = null;
1087
- }
1088
- }
1089
- await runProbes('attn_core_out', attnOutput.buffer, {
1090
- layerIdx,
1091
- numTokens,
1092
- hiddenSize: numHeads * headDim,
1093
- probes: state.debugProbes,
1094
- operatorDiagnostics: state.operatorDiagnostics,
1095
- dtype: attnOutput.dtype,
1096
- });
1097
-
1098
- // Trace attention output
1099
- if (kernelTrace.enabled) {
1100
- await traceStep('attention', `L${layerIdx}.attention`, layerIdx, attnOutput.buffer, [numTokens, numHeads * headDim]);
1101
- }
1102
-
1103
- // Kernel step debug: attention output
1104
- if (isKernelDebugEnabled(layerIdx)) {
1105
- logKernelStep('attention', { layerIdx, label: `seqLen=${numTokens} kvLen=${kvState.kvLenForAttention}` });
1106
- await dumpTokenVector(attnOutput.buffer, 'attn_out', {
1107
- layerIdx,
1108
- tokenIdx: Math.max(0, numTokens - 1),
1109
- rowSize: numHeads * headDim,
1110
- dtype: attnOutput.dtype,
1111
- });
1112
- }
1113
-
1114
- // Debug: attention output for configured layers
1115
- if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'attn_out', debugFlags) && debugCheckBuffer) {
1116
- await debugCheckBuffer(attnOutput.buffer, `L${layerIdx} attention output (before o_proj, GPU)`, numTokens, numHeads * headDim);
1117
- }
1118
-
1119
- attnForProjection = attnOutput;
1120
- if (qGateTensor && !attentionOutputGateFused) {
1121
- // The Qwen3_5 reference (HF transformers) full-attention path
1122
- // applies sigmoid(gate) — NOT silu(gate) — to the attention
1123
- // output regardless of how the architecture's gate label is
1124
- // surfaced in the HF config. See Qwen3_5Attention.forward in
1125
- // transformers/models/qwen3_5/modeling_qwen3_5.py:
1126
- // attn_output = attn_output * torch.sigmoid(gate)
1127
- // The earlier `swish` -> `silu` mapping here was incorrect and
1128
- // produced sign-flipped output for any token whose gate value
1129
- // was negative (silu is sign-preserving via x * sigmoid(x);
1130
- // sigmoid is unconditionally in (0, 1)). At Qwen 3.6 27B scale
1131
- // this drove residual-stream divergence by L=3 (HF reference vs
1132
- // upstream rel_l2≈1.37, mean per-token correlation ≈ -0.96 — a
1133
- // sign-flip pattern). config.outputGateType is retained for
1134
- // future architectures whose gate truly is silu, but Qwen 3.6's
1135
- // `output_gate_type=swish` still routes here as sigmoid because
1136
- // that is what the reference forward computes.
1137
- const gateActivation = 'sigmoid';
1138
- attnForProjection = await runSiLU(attnOutput, {
1139
- size: numTokens * numHeads * headDim,
1140
- gate: qGateTensor,
1141
- useVec4: (numTokens * numHeads * headDim) % 4 === 0,
1142
- gateActivation,
1143
- inputActivation: 'identity',
1144
- swigluLimit: null,
1145
- });
1146
- releaseBuffer(attnOutput.buffer);
1147
- }
1148
-
1149
- // 6. Output projection (with optional fused residual for decode)
1150
-
1151
- output = null;
1152
- let residualFused = false;
1153
- let oProjInput = attnForProjection;
1154
- oProjInputTemp = null;
1155
- if (layerWeights.oProj && getWeightBuffer) {
1156
- ({ oProjInput, oProjInputTemp } = await prepareAttentionProjectionInput(
1157
- attnForProjection,
1158
- oProjInputDtype,
1159
- async (tensor) => {
1160
- assertAttentionDtypeTransitionAllowed(
1161
- state,
1162
- tensor.dtype,
1163
- oProjInputDtype,
1164
- 'o_proj input would change dtype implicitly.',
1165
- 'step_precision'
1166
- );
1167
- return oProjInputDtype === 'f16'
1168
- ? castF32ToF16(tensor)
1169
- : castF16ToF32(tensor);
1170
- }
1171
- ));
1172
- const oProjBuf = getWeightBuffer(layerWeights.oProj, 'o_proj');
1173
- const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
1174
-
1175
- // Use fused o_proj + residual for decode when possible
1176
- // Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
1177
- const oProjDtype = getWeightDtype(oProjBuf);
1178
- const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
1179
- allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
1180
- hasResidual: Boolean(residualTensor),
1181
- residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
1182
- attnIsF32: oProjInput.dtype === 'f32',
1183
- attnIsF16: oProjInput.dtype === 'f16',
1184
- hasLoRA: Boolean(loraO),
1185
- oProjIsF16: oProjDtype === 'f16',
1186
- }); // GEMV kernel expects f16 weights
1187
- const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
1188
- phase,
1189
- session: mergedSession,
1190
- hasResidual: Boolean(residualTensor),
1191
- residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
1192
- inputDtype: oProjInput.dtype,
1193
- outputDtype: oProjOutputDtype,
1194
- oProjDtype,
1195
- hasLoRA: Boolean(loraO),
1196
- });
1197
- if (canUseFused && residualTensor) {
1198
- // FUSED PATH: o_proj matmul + residual add in one dispatch
1199
- output = await runMatmulResidualFused(oProjInput, oProjBuf, residualTensor, {
1200
- N: hiddenSize,
1201
- K: numHeads * headDim,
1202
- });
1203
- residualFused = true;
1204
-
1205
- if (layerIdx === 0 && !isPrefill) {
1206
- trace.attn(layerIdx, `Using fused o_proj+residual path`);
1207
- }
1208
- } else {
1209
- // STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
1210
- output = await runMatmul(oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
1211
- transposeB: 'auto',
1212
- role: 'o_proj',
1213
- layerIdx,
1214
- kernelPath,
1215
- outputDtype: oProjOutputDtype,
1216
- executionPolicies: state.executionPolicies ?? null,
1217
- residualTensor: canUseWideTileResidual ? residualTensor : null,
1218
- });
1219
- residualFused = canUseWideTileResidual;
1220
- }
1221
- // Release temporary buffer if we created it (original was not already on GPU)
1222
- if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
1223
- releaseBuffer(isWeightBuffer(oProjBuf) ? oProjBuf.buffer : oProjBuf);
1224
- }
1225
-
1226
- // Trace output projection
1227
- if (kernelTrace.enabled) {
1228
- await traceStep('matmul', `L${layerIdx}.o_proj${residualFused ? '+residual' : ''}`, layerIdx, output.buffer, [numTokens, hiddenSize]);
1229
- }
1230
-
1231
- // Kernel step debug: output projection
1232
- if (isKernelDebugEnabled(layerIdx)) {
1233
- logKernelStep('matmul', { layerIdx, label: residualFused ? 'O_proj+residual' : 'O_proj', M: numTokens, N: hiddenSize, K: numHeads * headDim });
1234
- await dumpTokenVector(output.buffer, 'o_proj_out', {
1235
- layerIdx,
1236
- tokenIdx: Math.max(0, numTokens - 1),
1237
- rowSize: hiddenSize,
1238
- dtype: output.dtype,
1239
- });
1240
- }
1241
-
1242
- } else {
1243
- output = oProjInput;
1244
- }
1245
-
1246
- // Apply LoRA to output projection if present (only if not using fused path)
1247
- if (!residualFused) {
1248
- const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
1249
- if (loraO && getWeightBuffer) {
1250
- const combined = await applyLoRA(
1251
- oProjInput,
1252
- output,
1253
- loraO,
1254
- { M: numTokens, N: hiddenSize, K: numHeads * headDim },
1255
- getWeightBuffer,
1256
- undefined,
1257
- { kernelPath }
1258
- );
1259
- if (combined.buffer !== output.buffer) {
1260
- releaseBuffer(output.buffer);
1261
- output = combined;
1262
- }
1263
- }
1264
- }
1265
-
1266
- if (layerWeights.oProjBias) {
1267
- const { tensor: oProjBiasTensor, owned } = getVectorTensor(
1268
- layerWeights.oProjBias,
1269
- `L${layerIdx}.o_proj_bias`,
1270
- hiddenSize,
1271
- {},
1272
- debugFlags
1273
- );
1274
- try {
1275
- output = await runBiasAdd(output, oProjBiasTensor, numTokens, hiddenSize, {
1276
- label: `L${layerIdx}.o_proj_bias`,
1277
- layerIdx,
1278
- executionPolicies: state.executionPolicies ?? null,
1279
- });
1280
- } finally {
1281
- if (owned) releaseBuffer(oProjBiasTensor.buffer);
1282
- }
1283
- }
1284
-
1285
- if (oProjInputTemp) {
1286
- releaseBuffer(oProjInputTemp.buffer);
1287
- }
1288
-
1289
- // Debug: o_proj output for configured layers
1290
- if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'o_proj', debugFlags) && debugCheckBuffer) {
1291
- await debugCheckBuffer(output.buffer, `L${layerIdx} attention output (after o_proj, GPU)`, numTokens, hiddenSize);
1292
- }
1293
-
1294
- finalOutput = output;
1295
-
1296
- const buffersToRelease = [];
1297
- if (output.buffer !== attnForProjection.buffer) {
1298
- buffersToRelease.push(attnForProjection.buffer);
1299
- }
1300
-
1301
- if (output.dtype !== oProjOutputDtype) {
1302
- assertAttentionDtypeTransitionAllowed(
1303
- state,
1304
- output.dtype,
1305
- oProjOutputDtype,
1306
- 'Attention output would change dtype implicitly at the end of the op.',
1307
- 'step_precision'
1308
- );
1309
- const coercedOutput = oProjOutputDtype === 'f16'
1310
- ? await castF32ToF16(output)
1311
- : await castF16ToF32(output);
1312
- buffersToRelease.push(output.buffer);
1313
- finalOutput = coercedOutput;
1314
- }
1315
-
1316
- // Cleanup
1317
- releaseBuffer(qTensor.buffer);
1318
- if (qGateTensor) {
1319
- releaseBuffer(qGateTensor.buffer);
1320
- }
1321
- if (!reusesSharedKV && !storeSharedKV) {
1322
- releaseBuffer(kTensor.buffer);
1323
- if (vTensor.buffer !== kTensor.buffer) {
1324
- releaseBuffer(vTensor.buffer);
1325
- }
1326
- }
1327
- for (const buffer of buffersToRelease) {
1328
- releaseBuffer(buffer);
1329
- }
1330
-
1331
- return { output: finalOutput, residualFused };
1332
- } catch (error) {
1333
- const released = new Set();
1334
- const releaseOnce = (buffer) => {
1335
- if (!buffer || released.has(buffer)) return;
1336
- released.add(buffer);
1337
- releaseBuffer(buffer);
1338
- };
1339
- if (finalOutput?.buffer && finalOutput.buffer !== output?.buffer) {
1340
- releaseOnce(finalOutput.buffer);
1341
- }
1342
- if (output?.buffer && output.buffer !== attnForProjection?.buffer) {
1343
- releaseOnce(output.buffer);
1344
- }
1345
- if (oProjInputTemp?.buffer) {
1346
- releaseOnce(oProjInputTemp.buffer);
1347
- }
1348
- if (attnForProjection?.buffer && attnForProjection.buffer !== attnOutput?.buffer) {
1349
- releaseOnce(attnForProjection.buffer);
1350
- }
1351
- if (attnOutput?.buffer) {
1352
- releaseOnce(attnOutput.buffer);
1353
- }
1354
- if (qGateTensor?.buffer) {
1355
- releaseOnce(qGateTensor.buffer);
1356
- }
1357
- if (qTensor?.buffer) {
1358
- releaseOnce(qTensor.buffer);
1359
- }
1360
- if (kTensor?.buffer && !reusesSharedKV && !storeSharedKV) {
1361
- releaseOnce(kTensor.buffer);
1362
- }
1363
- if (vTensor?.buffer && !reusesSharedKV && !storeSharedKV) {
1364
- releaseOnce(vTensor.buffer);
1365
- }
1366
- if (normed?.buffer && normed.buffer !== attentionInput?.buffer) {
1367
- releaseOnce(normed.buffer);
1368
- }
1369
- if (attentionInputTemp && attentionInput?.buffer) {
1370
- releaseOnce(attentionInput.buffer);
1371
- }
1372
- if (decoderKTemp?.buffer) {
1373
- releaseOnce(decoderKTemp.buffer);
1374
- }
1375
- if (decoderVTemp?.buffer) {
1376
- releaseOnce(decoderVTemp.buffer);
1377
- }
1378
- if (decoderKVState?.ownedBuffers) {
1379
- for (const buffer of decoderKVState.ownedBuffers) {
1380
- releaseOnce(buffer);
1381
- }
1382
- decoderKVState.ownedBuffers = null;
1383
- }
1384
- throw error;
1385
- }
1386
- }
1
+ /**
2
+ * Immediate attention facade.
3
+ *
4
+ * Semantic decisions are resolved in plan.js. The immediate adapter records
5
+ * the canonical interpreter and submits it immediately.
6
+ */
7
+
8
+ export { runLayerAttentionGPU } from './executor-immediate.js';