doppler-gpu 0.4.15 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/README.md +9 -0
  2. package/package.json +25 -2
  3. package/src/cli/doppler-cli.js +253 -94
  4. package/src/cli/runtime-profiles.js +3 -7
  5. package/src/client/doppler-api.d.ts +28 -0
  6. package/src/client/doppler-api.js +2 -0
  7. package/src/client/inspection.d.ts +122 -0
  8. package/src/client/inspection.js +343 -0
  9. package/src/client/runtime/index.d.ts +54 -0
  10. package/src/client/runtime/index.js +190 -18
  11. package/src/client/runtime/model-manager.d.ts +0 -4
  12. package/src/client/runtime/model-manager.js +0 -28
  13. package/src/client/runtime/model-session.d.ts +48 -1
  14. package/src/client/runtime/model-session.js +149 -30
  15. package/src/client/runtime/model-source.d.ts +10 -1
  16. package/src/client/runtime/scoped-session.d.ts +117 -0
  17. package/src/client/runtime/scoped-session.js +257 -0
  18. package/src/client/runtime/types.d.ts +0 -1
  19. package/src/client/runtime/types.js +0 -1
  20. package/src/config/evidence/boundary-evidence-policy.json +54 -0
  21. package/src/config/evidence/source-intake-policy.json +122 -0
  22. package/src/config/inspection/observation-policies.json +80 -0
  23. package/src/config/param-validator.js +21 -36
  24. package/src/config/runtime/compute/f16-activations.json +1 -0
  25. package/src/config/runtime/compute/f16-batched.json +1 -0
  26. package/src/config/runtime/default.json +1 -0
  27. package/src/config/runtime/diagnostics/debug-logits.json +1 -0
  28. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
  29. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
  30. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
  31. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
  32. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
  33. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
  34. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
  35. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
  36. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
  37. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
  38. package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
  39. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
  40. package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
  41. package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
  42. package/src/config/runtime/model/gemma2-debug.json +1 -0
  43. package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
  44. package/src/config/runtime/model/gemma2-pipeline.json +1 -0
  45. package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
  46. package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
  47. package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
  48. package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
  49. package/src/config/runtime/optimization-candidates/registry.json +5 -0
  50. package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
  51. package/src/config/runtime/profiles/default.json +1 -0
  52. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
  53. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
  54. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
  55. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
  56. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
  57. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
  58. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
  59. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
  60. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
  61. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
  62. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
  63. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
  64. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
  65. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
  66. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
  67. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
  68. package/src/config/runtime/profiles/low-memory.json +1 -0
  69. package/src/config/runtime/profiles/production.json +1 -3
  70. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
  71. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
  72. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
  73. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
  74. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
  75. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
  76. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
  77. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
  78. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
  79. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
  80. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
  81. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
  82. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
  83. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
  84. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
  85. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
  86. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
  88. package/src/config/runtime/profiles/rerank-stability.json +1 -3
  89. package/src/config/runtime/profiles/rerank-throughput.json +1 -3
  90. package/src/config/runtime/profiles/simulation.json +1 -3
  91. package/src/config/runtime/profiles/throughput.json +1 -3
  92. package/src/config/runtime/profiles/trace-layers.json +1 -3
  93. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
  94. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
  95. package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
  96. package/src/config/runtime/profiles/turboquant.json +1 -0
  97. package/src/config/runtime/profiles/vector-stability.json +1 -3
  98. package/src/config/runtime/profiles/vector-throughput.json +1 -0
  99. package/src/config/runtime/profiles/verbose-trace.json +1 -3
  100. package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
  101. package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
  102. package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
  103. package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
  104. package/src/config/schema/benchmark.schema.d.ts +7 -0
  105. package/src/config/schema/benchmark.schema.js +5 -0
  106. package/src/config/schema/index.d.ts +8 -16
  107. package/src/config/schema/index.js +5 -8
  108. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
  109. package/src/config/schema/kernel-thresholds.schema.js +0 -12
  110. package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
  111. package/src/config/schema/kernel-warmup.schema.js +0 -1
  112. package/src/config/schema/runtime-profile.schema.d.ts +24 -0
  113. package/src/config/schema/runtime-profile.schema.js +92 -0
  114. package/src/config/schema/shared-runtime.schema.d.ts +1 -4
  115. package/src/config/schema/shared-runtime.schema.js +0 -2
  116. package/src/config/schema/tooling.schema.d.ts +8 -5
  117. package/src/config/schema/tooling.schema.js +3 -2
  118. package/src/debug/tensor.d.ts +2 -0
  119. package/src/debug/tensor.js +21 -2
  120. package/src/gpu/command-recorder.d.ts +5 -0
  121. package/src/gpu/command-recorder.js +17 -0
  122. package/src/gpu/kernel-runtime.js +2 -10
  123. package/src/gpu/kernels/index.d.ts +0 -2
  124. package/src/gpu/kernels/index.js +0 -2
  125. package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
  126. package/src/gpu/kernels/kernel-prewarm.js +60 -0
  127. package/src/gpu/kernels/utils.d.ts +2 -4
  128. package/src/gpu/kernels/utils.js +2 -4
  129. package/src/index.d.ts +13 -1
  130. package/src/index.js +7 -1
  131. package/src/inference/browser-harness-model-helpers.js +4 -0
  132. package/src/inference/browser-harness-runtime-helpers.js +25 -10
  133. package/src/inference/browser-harness-text-helpers.js +4 -1
  134. package/src/inference/browser-harness.d.ts +4 -0
  135. package/src/inference/browser-harness.js +91 -4
  136. package/src/inference/observation-context.d.ts +22 -0
  137. package/src/inference/observation-context.js +41 -0
  138. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  139. package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
  140. package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
  141. package/src/inference/pipelines/text/attention/index.js +7 -0
  142. package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
  143. package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
  144. package/src/inference/pipelines/text/attention/plan.js +328 -0
  145. package/src/inference/pipelines/text/attention/receipt.js +60 -0
  146. package/src/inference/pipelines/text/attention/record.js +8 -1278
  147. package/src/inference/pipelines/text/attention/run.js +8 -1386
  148. package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
  149. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  150. package/src/inference/pipelines/text/generator-steps.js +32 -15
  151. package/src/inference/pipelines/text/generator.js +10 -5
  152. package/src/inference/pipelines/text/init.d.ts +3 -0
  153. package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
  154. package/src/inference/pipelines/text/layer.js +2 -0
  155. package/src/inference/pipelines/text/model-load.d.ts +0 -1
  156. package/src/inference/pipelines/text/model-load.js +2 -11
  157. package/src/inference/pipelines/text/probes.js +10 -0
  158. package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
  159. package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
  160. package/src/inference/pipelines/text/state.d.ts +6 -0
  161. package/src/inference/pipelines/text/state.js +3 -0
  162. package/src/inference/pipelines/text/types.d.ts +6 -0
  163. package/src/inference/pipelines/text.js +16 -1
  164. package/src/inference/refactor-receipt.d.ts +49 -0
  165. package/src/inference/refactor-receipt.js +188 -0
  166. package/src/inference/resource-scope.d.ts +33 -0
  167. package/src/inference/resource-scope.js +168 -0
  168. package/src/inference/test-harness.d.ts +3 -0
  169. package/src/inference/test-harness.js +1 -0
  170. package/src/storage/model-cache.d.ts +53 -0
  171. package/src/storage/model-cache.js +550 -0
  172. package/src/tooling/boundary-evidence.d.ts +51 -0
  173. package/src/tooling/boundary-evidence.js +408 -0
  174. package/src/tooling/browser-command-runner.js +4 -4
  175. package/src/tooling/command-context.d.ts +25 -0
  176. package/src/tooling/command-context.js +53 -0
  177. package/src/tooling/command-runner-shared.js +22 -5
  178. package/src/tooling/demo-receipts.d.ts +56 -0
  179. package/src/tooling/demo-receipts.js +113 -0
  180. package/src/tooling/diagnose-runner.js +2 -2
  181. package/src/tooling/execution-cost-ledger.d.ts +21 -0
  182. package/src/tooling/execution-cost-ledger.js +199 -0
  183. package/src/tooling/node-command-runner.js +2 -2
  184. package/src/tooling/opfs-cache.d.ts +11 -49
  185. package/src/tooling/opfs-cache.js +4 -510
  186. package/src/tooling/precision-replay-math.d.ts +42 -0
  187. package/src/tooling/precision-replay-math.js +174 -0
  188. package/src/tooling/program-bundle-parity.d.ts +3 -0
  189. package/src/tooling/program-bundle-parity.js +8 -0
  190. package/src/tooling/program-bundle.d.ts +1 -52
  191. package/src/tooling/program-bundle.js +0 -139
  192. package/src/tooling/registered-variant-calibration.d.ts +24 -0
  193. package/src/tooling/registered-variant-calibration.js +286 -0
  194. package/src/tooling/runtime-input-composition.js +105 -53
  195. package/src/tooling/runtime-intent.js +41 -0
  196. package/src/tooling/runtime-optimization-index.d.ts +6 -0
  197. package/src/tooling/runtime-optimization-index.js +72 -0
  198. package/src/tooling/runtime-optimization.d.ts +47 -6
  199. package/src/tooling/runtime-optimization.js +559 -40
  200. package/src/tooling/source-intake.d.ts +49 -0
  201. package/src/tooling/source-intake.js +414 -0
  202. package/src/tooling-exports/evidence.d.ts +34 -0
  203. package/src/tooling-exports/evidence.js +34 -0
  204. package/src/tooling-exports/runtime.d.ts +7 -0
  205. package/src/tooling-exports/runtime.js +7 -0
  206. package/src/tooling-exports.d.ts +11 -0
  207. package/src/tooling-exports.js +6 -3
  208. package/src/tooling-exports.shared.d.ts +32 -0
  209. package/src/tooling-exports.shared.js +31 -0
  210. package/src/version.js +1 -1
  211. package/src/config/schema/tuner.schema.d.ts +0 -39
  212. package/src/config/schema/tuner.schema.js +0 -13
  213. package/src/gpu/kernel-tuner/benchmarks.js +0 -902
  214. package/src/gpu/kernel-tuner/cache.js +0 -145
  215. package/src/gpu/kernel-tuner/index.js +0 -38
  216. package/src/gpu/kernel-tuner/tuner.js +0 -247
  217. package/src/gpu/kernel-tuner.js +0 -12
  218. package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
  219. package/src/gpu/kernels/kernel-tuning.js +0 -158
  220. package/src/gpu/profiler.js +0 -382
  221. package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
@@ -0,0 +1,1334 @@
1
+
2
+
3
+ import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
+ import { getKernelCapabilities } from '../../../../gpu/device.js';
5
+ import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
6
+ import {
7
+ recordMatmul,
8
+ recordRMSNorm,
9
+ recordLayerNorm,
10
+ recordBiasAdd,
11
+ recordRoPE,
12
+ canUseRoPEQK,
13
+ recordRoPEQK,
14
+ recordAttention,
15
+ recordAttentionTiered,
16
+ recordAttentionTieredQuant,
17
+ recordAttentionContiguousQuant,
18
+ recordAttentionBDPA,
19
+ recordSiLU,
20
+ recordCastF16ToF32,
21
+ recordCastF32ToF16,
22
+ recordMatmulResidualFused,
23
+ } from '../../../../gpu/kernel-selector.js';
24
+ import { createTensor } from '../../../../gpu/tensor.js';
25
+ import { applyLoRA } from '../lora-apply.js';
26
+ import { getLoRAModule } from '../lora.js';
27
+ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
28
+ import { selectRuleValue } from '../../../../rules/rule-registry.js';
29
+ import {
30
+ recordAttentionInputs,
31
+ shouldForceF32AttentionProjectionForRoPE,
32
+ resolveAttentionProjectionOutputDtype,
33
+ projectAttentionQKV,
34
+ applyAttentionQKNorm,
35
+ applyAttentionValueNorm,
36
+ hasAttentionProjectionDiagnostics,
37
+ hasAttentionStageDiagnostics,
38
+ resolveAttentionQKNormState,
39
+ } from './projections.js';
40
+ import { prepareAttentionProjectionInput } from './output-projection.js';
41
+ import { runProbes } from '../probes.js';
42
+ import { decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
43
+
44
+ import { shouldDebugLayer } from './types.js';
45
+ import {
46
+ getKernelPathMatmulPrecision,
47
+ getKernelPathMatmulVariant,
48
+ } from '../../../../config/kernel-path-loader.js';
49
+ import {
50
+ resolveKVCacheState,
51
+ createDiffusionGemmaDecoderKVState,
52
+ buildAttentionDispatchParams,
53
+ buildAttentionInputsData,
54
+ } from './dispatch-params.js';
55
+ import {
56
+ buildTieredQuantAttentionOptions,
57
+ buildContiguousQuantAttentionOptions,
58
+ } from './quant-options.js';
59
+ import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
60
+ import {
61
+ resolveAttentionPrecisionContract,
62
+ isAttentionKvDtypeExplicit,
63
+ } from './precision-contract.js';
64
+ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
65
+ import { getVectorTensor } from '../weights.js';
66
+ import { resolveAttentionRuntimeSession } from '../resolved-runtime-session.js';
67
+ import {
68
+ bindAttentionPlan,
69
+ createAttentionExecutor,
70
+ executeBoundAttentionPlan,
71
+ resolveAttentionPlanForDispatch,
72
+ } from './plan.js';
73
+ import { createRecordedResourceScope } from '../../../resource-scope.js';
74
+ import { captureAttentionRefactorReceipt } from './receipt.js';
75
+
76
+ const ATTENTION_DTYPE_LOGGED = new Set();
77
+
78
+ function shouldTraceRecordedHealth(layerIdx, debugFlags) {
79
+ const debugLayers = debugFlags?.debugLayers;
80
+ return isTraceEnabled('logits')
81
+ && Array.isArray(debugLayers)
82
+ && shouldDebugLayer(layerIdx, debugLayers);
83
+ }
84
+
85
+ function enqueueRecordedTensorHealth(recorder, label, tensor, dtype, elementCount) {
86
+ if (!recorder || !tensor?.buffer || !Number.isFinite(elementCount) || elementCount <= 0) {
87
+ return;
88
+ }
89
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
90
+ recorder.enqueueCompletionTask(async () => {
91
+ const data = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
92
+ trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
93
+ });
94
+ }
95
+
96
+ function resolveDirectF16KVCacheWrite(options) {
97
+ const {
98
+ state,
99
+ layerIdx,
100
+ currentSeqLen,
101
+ numTokens,
102
+ numKVHeads,
103
+ headDim,
104
+ reusesSharedKV,
105
+ storeSharedKV,
106
+ diffusionGemmaDecoder,
107
+ valueNorm,
108
+ diagnosticsEnabled,
109
+ disableRoPE,
110
+ } = options;
111
+ if (
112
+ numTokens !== 1
113
+ || reusesSharedKV === true
114
+ || storeSharedKV === true
115
+ || diffusionGemmaDecoder === true
116
+ || valueNorm === true
117
+ || diagnosticsEnabled === true
118
+ || disableRoPE === true
119
+ ) {
120
+ return null;
121
+ }
122
+ const cache = state?.kvCache;
123
+ if (
124
+ !cache?.hasGPUCache?.()
125
+ || cache.kvDtype !== 'f16'
126
+ || cache.layout !== 'contiguous'
127
+ || cache.windowSize != null
128
+ || cache.layerSpecs != null
129
+ || typeof cache.recordF16UpdateAlreadyWrittenFromGPU !== 'function'
130
+ ) {
131
+ return null;
132
+ }
133
+ const gpuBuffers = cache.getGPUBuffers(layerIdx);
134
+ const layout = gpuBuffers?.layout ?? cache.layout;
135
+ if (layout !== 'contiguous' || !gpuBuffers?.keysGPU || !gpuBuffers?.valuesGPU) {
136
+ return null;
137
+ }
138
+ return {
139
+ keysBuffer: gpuBuffers.keysGPU,
140
+ valuesBuffer: gpuBuffers.valuesGPU,
141
+ dstOffset: currentSeqLen * numKVHeads * headDim,
142
+ };
143
+ }
144
+
145
+ function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
146
+ assertImplicitDtypeTransitionAllowed({
147
+ executionPolicies: state?.executionPolicies ?? null,
148
+ fromDtype,
149
+ toDtype,
150
+ op: 'attention',
151
+ detail,
152
+ transitionDeclaredBy,
153
+ });
154
+ }
155
+
156
+
157
+ export async function interpretAttentionWithRecorder(
158
+ recorder,
159
+ input,
160
+ layerWeights,
161
+ config,
162
+ state,
163
+ debug = false,
164
+ debugFlags = {},
165
+ getWeightBuffer,
166
+ getNormWeightBuffer,
167
+ debugCheckBuffer,
168
+ lora
169
+ ) {
170
+ const {
171
+ layerIdx,
172
+ numTokens,
173
+ isPrefill,
174
+ numHeads,
175
+ numKVHeads,
176
+ headDim,
177
+ hiddenSize,
178
+ rmsNormEps,
179
+ currentSeqLen,
180
+ slidingWindow,
181
+ layerType,
182
+ residualTensor,
183
+ attnSoftcap,
184
+ queryPreAttnScalar,
185
+ skipInputNorm = false,
186
+ tokenIds = null,
187
+ kernelPath = null,
188
+ disableRoPE = false,
189
+ multimodalBidirectionalSpan = null,
190
+ sharedKVSourceLayerIdx = null,
191
+ storeSharedKV = false,
192
+ diffusionGemmaDecoder = false,
193
+ precomputedInputNorm = null,
194
+ } = config;
195
+
196
+ const phase = isPrefill ? 'prefill' : 'decode';
197
+ const runtimeSession = resolveAttentionRuntimeSession(state);
198
+ const resourceScope = createRecordedResourceScope(recorder);
199
+ const attentionPrecisionContract = resolveAttentionPrecisionContract(config, state);
200
+ const attentionActivationDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
201
+ dtype: attentionPrecisionContract.resolvedActivationDtype ?? config.activationDtype,
202
+ });
203
+ const oProjPrecision = getKernelPathMatmulPrecision('o_proj', phase, layerIdx, kernelPath);
204
+ const oProjInputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
205
+ dtype: oProjPrecision?.inputDtype ?? attentionActivationDtype,
206
+ });
207
+ const oProjOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
208
+ dtype: oProjPrecision?.outputDtype
209
+ ?? attentionPrecisionContract.resolvedOutputDtype
210
+ ?? config.activationDtype,
211
+ });
212
+ const wantsF16Output = oProjOutputDtype === 'f16';
213
+ const useF16Activations = attentionActivationDtype === 'f16';
214
+ const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
215
+ const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
216
+ const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
217
+ const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
218
+ ? 'step_precision'
219
+ : null;
220
+ let attentionInput = input;
221
+ let attentionInputTemp = false;
222
+ let normed = attentionInput;
223
+ let qTensor = null;
224
+ let qGateTensor = null;
225
+ let kTensor = null;
226
+ let vTensor = null;
227
+ let attnOutput = null;
228
+ let attnForProjection = null;
229
+ let output = null;
230
+ let finalOutput = null;
231
+ let oProjInputTemp = null;
232
+ let retainSharedKvBuffers = false;
233
+ let valueAliasesKey = false;
234
+ let decoderKVState = null;
235
+ let decoderKTemp = null;
236
+ let decoderVTemp = null;
237
+ let attentionPlan = null;
238
+ if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
239
+ assertAttentionDtypeTransitionAllowed(
240
+ state,
241
+ input.dtype,
242
+ attentionActivationDtype,
243
+ 'The attention kernel selection would widen the input implicitly.',
244
+ attentionInputTransitionDeclaredBy
245
+ );
246
+ attentionInput = attentionActivationDtype === 'f16'
247
+ ? await recordCastF32ToF16(recorder, input)
248
+ : await recordCastF16ToF32(recorder, input);
249
+ resourceScope.register(attentionInput.buffer, 'attention-input-cast', 'submitOwned');
250
+ attentionInputTemp = true;
251
+ normed = attentionInput;
252
+ }
253
+
254
+ if (!layerWeights) {
255
+ const bytesPerElement = wantsF16Output ? 2 : 4;
256
+ const outputBuf = acquireBuffer(numTokens * hiddenSize * bytesPerElement, undefined, 'attn_output');
257
+ const output = createTensor(outputBuf, oProjOutputDtype, [numTokens, hiddenSize], 'attn_output');
258
+ resourceScope.register(output.buffer, 'attention-output', 'retained');
259
+ resourceScope.close('success');
260
+ return { output, residualFused: false };
261
+ }
262
+
263
+ const qSize = numTokens * numHeads * headDim;
264
+ const kvSize = numTokens * numKVHeads * headDim;
265
+
266
+ // 1. Input norm
267
+ // Opt-in fusion: when useFusedRmsnormWideTile is set, defer the standalone
268
+ // rmsnorm into each q/k/v_proj matmul (fused kernel runs norm internally).
269
+ let fusedNormWeightRec = null;
270
+ let fusedNormEpsRec = null;
271
+ let fusedNormOffsetRec = false;
272
+ let fusedNormOwnedRec = false;
273
+ const rmsNormFusionFlagRec = runtimeSession.useFusedRmsnormWideTile === true;
274
+ const canSelectFusedRmsNormProjectionRec = canUseRmsNormWideTileProjectionFusion(
275
+ layerWeights,
276
+ sharedKVSourceLayerIdx != null
277
+ );
278
+ const canFuseInputNormProjRec = rmsNormFusionFlagRec
279
+ && config.normalizationType === 'rmsnorm'
280
+ && canSelectFusedRmsNormProjectionRec
281
+ && !skipInputNorm
282
+ && layerWeights.inputNorm
283
+ && getNormWeightBuffer
284
+ && isPrefill
285
+ && numTokens > 1
286
+ && attentionInput.dtype === 'f32';
287
+
288
+ try {
289
+ if (precomputedInputNorm) {
290
+ if (
291
+ isPrefill
292
+ || numTokens !== 1
293
+ || skipInputNorm
294
+ || !layerWeights.inputNorm
295
+ || !getNormWeightBuffer
296
+ || precomputedInputNorm.dtype !== attentionInput.dtype
297
+ || precomputedInputNorm.dtype !== 'f32'
298
+ ) {
299
+ throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
300
+ }
301
+ normed = precomputedInputNorm;
302
+ await runProbes('post_input_norm', normed.buffer, {
303
+ layerIdx, numTokens, hiddenSize,
304
+ probes: state.debugProbes, recorder,
305
+ operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
306
+ });
307
+ } else if (canFuseInputNormProjRec) {
308
+ fusedNormWeightRec = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
309
+ fusedNormEpsRec = rmsNormEps;
310
+ fusedNormOffsetRec = config.rmsNormWeightOffset === true;
311
+ fusedNormOwnedRec = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
312
+ if (fusedNormOwnedRec) {
313
+ resourceScope.register(fusedNormWeightRec, 'fused-input-norm-weight', 'submitOwned');
314
+ }
315
+ // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
316
+ } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
317
+ const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
318
+ let normBiasTensor = null;
319
+ let normBiasOwned = false;
320
+ if (config.normalizationType === 'layernorm') {
321
+ if (!layerWeights.inputNormBias) {
322
+ throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
323
+ }
324
+ ({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
325
+ layerWeights.inputNormBias,
326
+ `L${layerIdx}.input_norm_bias`,
327
+ hiddenSize,
328
+ {},
329
+ debugFlags
330
+ ));
331
+ normed = await recordLayerNorm(
332
+ recorder,
333
+ attentionInput,
334
+ normWeightBuf,
335
+ normBiasTensor.buffer,
336
+ rmsNormEps,
337
+ { batchSize: numTokens, hiddenSize, label: 'input_norm' }
338
+ );
339
+ } else {
340
+ normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
341
+ batchSize: numTokens,
342
+ hiddenSize,
343
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
344
+ label: 'input_norm',
345
+ });
346
+ }
347
+ if (normBiasOwned && normBiasTensor) resourceScope.release(normBiasTensor.buffer);
348
+ if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) resourceScope.release(normWeightBuf);
349
+ await runProbes('post_input_norm', normed.buffer, {
350
+ layerIdx, numTokens, hiddenSize,
351
+ probes: state.debugProbes, recorder,
352
+ operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
353
+ });
354
+ }
355
+
356
+ const debugLayers = debugFlags.debugLayers;
357
+ const shouldLogLayer = debugLayers === null ? layerIdx === 0 : shouldDebugLayer(layerIdx, debugLayers);
358
+ if (shouldLogLayer) {
359
+ const phase = selectRuleValue('kernels', 'attention', 'phase', { isDecode: !isPrefill });
360
+ const logKey = `L${layerIdx}_${phase}_dtypes`;
361
+ if (!ATTENTION_DTYPE_LOGGED.has(logKey)) {
362
+ ATTENTION_DTYPE_LOGGED.add(logKey);
363
+ trace.attn(layerIdx, `dtypes: activation=${attentionActivationDtype}, input=${input.dtype}, normed=${normed.dtype}`);
364
+ }
365
+ }
366
+
367
+ // 2. Q/K/V projections
368
+ const qProjVariant = getKernelPathMatmulVariant('q_proj', phase, layerIdx, kernelPath);
369
+ const kernelPathIsF16 = qProjVariant != null && qProjVariant.includes('f16') && !qProjVariant.includes('f32');
370
+ const matmulOutputDtype = resolveAttentionProjectionOutputDtype(attentionActivationDtype, {
371
+ forceF32: shouldForceF32AttentionProjectionForRoPE({
372
+ attentionInputDtype: attentionActivationDtype,
373
+ headDim,
374
+ rotaryDim: config.ropeRotaryDim,
375
+ interleaved: config.ropeInterleaved,
376
+ kernelPathIsF16,
377
+ }),
378
+ });
379
+ let usedFusedQKV = false;
380
+ const sharedKVEntry = sharedKVSourceLayerIdx == null
381
+ ? null
382
+ : (state.sharedAttentionState?.get(sharedKVSourceLayerIdx) ?? null);
383
+ if (sharedKVSourceLayerIdx != null && !sharedKVEntry) {
384
+ throw new Error(
385
+ `Layer ${layerIdx} requires shared K/V from layer ${sharedKVSourceLayerIdx}, ` +
386
+ 'but no shared K/V state was stored for that source layer.'
387
+ );
388
+ }
389
+ if (sharedKVEntry && (
390
+ sharedKVEntry.headDim !== headDim
391
+ || sharedKVEntry.numKVHeads !== numKVHeads
392
+ )) {
393
+ throw new Error(
394
+ `Layer ${layerIdx} shared K/V geometry mismatch. ` +
395
+ `Expected numKVHeads=${numKVHeads}, headDim=${headDim}; ` +
396
+ `got numKVHeads=${sharedKVEntry.numKVHeads}, headDim=${sharedKVEntry.headDim}.`
397
+ );
398
+ }
399
+ const reusesSharedKV = sharedKVEntry != null;
400
+ retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
401
+ let qkNormApplied = false;
402
+ let ropeApplied = false;
403
+ const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
404
+ const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
405
+ || shouldTraceRecordedHealth(layerIdx, debugFlags);
406
+ const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
407
+ state,
408
+ ['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
409
+ )
410
+ || shouldTraceRecordedHealth(layerIdx, debugFlags);
411
+ const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
412
+ const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
413
+ const skipKVCacheWrites = state.skipKVCacheWrites === true;
414
+ const directF16KVCacheWrite = skipKVCacheWrites
415
+ ? null
416
+ : resolveDirectF16KVCacheWrite({
417
+ state,
418
+ layerIdx,
419
+ currentSeqLen,
420
+ numTokens,
421
+ numKVHeads,
422
+ headDim,
423
+ reusesSharedKV,
424
+ storeSharedKV,
425
+ diffusionGemmaDecoder,
426
+ valueNorm: config.valueNorm === true,
427
+ diagnosticsEnabled: qkNormRoPEDiagnostics,
428
+ disableRoPE,
429
+ });
430
+ let kvCacheWriteFused = false;
431
+ ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
432
+ recorder,
433
+ normed,
434
+ layerWeights,
435
+ numTokens,
436
+ numHeads,
437
+ numKVHeads,
438
+ headDim,
439
+ hiddenSize,
440
+ layerIdx,
441
+ kernelPath,
442
+ matmulOutputDtype,
443
+ getWeightBuffer,
444
+ lora,
445
+ matmulDebug: state.runtimeConfig?.shared?.debug?.matmul ?? null,
446
+ attentionOutputGate: config.attentionOutputGate === true,
447
+ sharedKTensor: sharedKVEntry?.kTensor ?? null,
448
+ sharedVTensor: sharedKVEntry?.vTensor ?? null,
449
+ executionPolicies: state.executionPolicies ?? null,
450
+ releaseTemporary: (buffer) => resourceScope.release(buffer),
451
+ onFusedQKV: layerIdx === 0 && isPrefill
452
+ ? ({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize }) => {
453
+ trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
454
+ }
455
+ : null,
456
+ fusedNormWeight: fusedNormWeightRec,
457
+ fusedNormEps: fusedNormEpsRec,
458
+ fusedNormOffset: fusedNormOffsetRec,
459
+ qkNormFusion: {
460
+ enabled: qkNormFusionFlag
461
+ && qkNormState.wantsQKNorm
462
+ && config.queryKeyNormType === 'rmsnorm'
463
+ && config.queryKeyNormAxis === 'head',
464
+ getNormWeightBuffer,
465
+ rmsNormEps,
466
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
467
+ skipKNorm: qkNormState.skipKNorm,
468
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
469
+ projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
470
+ },
471
+ qkNormRoPEFusion: {
472
+ enabled: qkNormRoPEFusionFlag
473
+ && qkNormState.wantsQKNorm
474
+ && config.queryKeyNormType === 'rmsnorm'
475
+ && config.queryKeyNormAxis === 'head'
476
+ && !disableRoPE
477
+ && !!state.ropeFreqsCos
478
+ && !!state.ropeFreqsSin,
479
+ getNormWeightBuffer,
480
+ rmsNormEps,
481
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
482
+ skipKNorm: qkNormState.skipKNorm,
483
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
484
+ projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
485
+ freqsCos: state.ropeFreqsCos,
486
+ freqsSin: state.ropeFreqsSin,
487
+ startPos: currentSeqLen,
488
+ rotaryDim: config.ropeRotaryDim,
489
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
490
+ interleaved: config.ropeInterleaved,
491
+ reusesSharedKV,
492
+ f16KVCacheWrite: directF16KVCacheWrite,
493
+ },
494
+ }));
495
+ if (!kvCacheWriteFused && (!kTensor || !vTensor)) {
496
+ throw new Error('Recorded attention projection returned missing K/V tensors without a fused KV cache write.');
497
+ }
498
+ // Deferred release of the norm weight buffer for fused path.
499
+ if (fusedNormWeightRec && fusedNormOwnedRec) {
500
+ resourceScope.release(fusedNormWeightRec);
501
+ }
502
+ if (qTensor) {
503
+ await runProbes('q_proj', qTensor.buffer, {
504
+ layerIdx, numTokens, hiddenSize: numHeads * headDim,
505
+ probes: state.debugProbes, recorder,
506
+ operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
507
+ });
508
+ }
509
+ if (kTensor) {
510
+ await runProbes('k_proj', kTensor.buffer, {
511
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
512
+ probes: state.debugProbes, recorder,
513
+ operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
514
+ });
515
+ }
516
+ if (vTensor) {
517
+ await runProbes('v_proj', vTensor.buffer, {
518
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
519
+ probes: state.debugProbes, recorder,
520
+ operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
521
+ });
522
+ }
523
+ if (qTensor && shouldTraceRecordedHealth(layerIdx, debugFlags)) {
524
+ enqueueRecordedTensorHealth(
525
+ recorder,
526
+ `L${layerIdx}.q_proj_HEALTH`,
527
+ qTensor,
528
+ qTensor.dtype,
529
+ numTokens * numHeads * headDim
530
+ );
531
+ enqueueRecordedTensorHealth(
532
+ recorder,
533
+ `L${layerIdx}.k_proj_HEALTH`,
534
+ kTensor,
535
+ kTensor?.dtype ?? null,
536
+ numTokens * numKVHeads * headDim
537
+ );
538
+ enqueueRecordedTensorHealth(
539
+ recorder,
540
+ `L${layerIdx}.v_proj_HEALTH`,
541
+ vTensor,
542
+ vTensor?.dtype ?? null,
543
+ numTokens * numKVHeads * headDim
544
+ );
545
+ }
546
+
547
+ // Optional Q/K normalization. The manifest owns both its numerical family
548
+ // and whether it applies per head or across the complete projection.
549
+ if (qkNormState.wantsQKNorm && !qkNormApplied) {
550
+ ({ qTensor, kTensor } = await applyAttentionQKNorm({
551
+ recorder,
552
+ qTensor,
553
+ kTensor,
554
+ layerWeights,
555
+ getNormWeightBuffer,
556
+ rmsNormEps,
557
+ numTokens,
558
+ numHeads,
559
+ numKVHeads,
560
+ headDim,
561
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
562
+ normalizationType: config.queryKeyNormType,
563
+ normalizationAxis: config.queryKeyNormAxis,
564
+ releaseTemporary: (buffer) => resourceScope.release(buffer),
565
+ skipKNorm: qkNormState.skipKNorm,
566
+ retainKInput: valueAliasesKey,
567
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
568
+ }));
569
+ }
570
+ if (qkNormState.wantsQKNorm) {
571
+ await runProbes('q_norm', qTensor.buffer, {
572
+ layerIdx, numTokens, hiddenSize: numHeads * headDim,
573
+ probes: state.debugProbes, recorder,
574
+ operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
575
+ });
576
+ if (kTensor) {
577
+ await runProbes('k_norm', kTensor.buffer, {
578
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
579
+ probes: state.debugProbes, recorder,
580
+ operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
581
+ });
582
+ }
583
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
584
+ enqueueRecordedTensorHealth(
585
+ recorder,
586
+ `L${layerIdx}.q_norm_HEALTH`,
587
+ qTensor,
588
+ qTensor.dtype,
589
+ numTokens * numHeads * headDim
590
+ );
591
+ enqueueRecordedTensorHealth(
592
+ recorder,
593
+ `L${layerIdx}.k_norm_HEALTH`,
594
+ kTensor,
595
+ kTensor?.dtype ?? null,
596
+ numTokens * numKVHeads * headDim
597
+ );
598
+ }
599
+ }
600
+
601
+ if (!kvCacheWriteFused && config.valueNorm === true && !reusesSharedKV) {
602
+ const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
603
+ vTensor = await applyAttentionValueNorm({
604
+ recorder,
605
+ vTensor,
606
+ rmsNormEps,
607
+ numTokens,
608
+ numKVHeads,
609
+ headDim,
610
+ releaseTemporary: (buffer) => {
611
+ if (!valueNormInputAliasesKey) {
612
+ resourceScope.release(buffer);
613
+ }
614
+ },
615
+ });
616
+ await runProbes('v_norm', vTensor.buffer, {
617
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
618
+ probes: state.debugProbes, recorder,
619
+ operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
620
+ });
621
+ }
622
+
623
+ if (normed !== attentionInput) {
624
+ resourceScope.register(normed.buffer, 'normalized-input', 'submitOwned');
625
+ resourceScope.release(normed.buffer);
626
+ }
627
+ if (attentionInputTemp) resourceScope.release(attentionInput.buffer);
628
+
629
+ // 3. RoPE (modifies tensor in-place)
630
+ if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
631
+ const ropeOptions = {
632
+ headDim,
633
+ rotaryDim: config.ropeRotaryDim,
634
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
635
+ interleaved: config.ropeInterleaved,
636
+ startPos: currentSeqLen,
637
+ executionPolicies: state.executionPolicies ?? null,
638
+ };
639
+ if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
640
+ await recordRoPEQK(recorder, qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
641
+ ...ropeOptions,
642
+ numQHeads: numHeads,
643
+ numKVHeads,
644
+ });
645
+ } else {
646
+ await recordRoPE(recorder, qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
647
+ ...ropeOptions,
648
+ numHeads,
649
+ });
650
+ if (!reusesSharedKV) {
651
+ await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
652
+ ...ropeOptions,
653
+ numHeads: numKVHeads,
654
+ });
655
+ }
656
+ }
657
+ }
658
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
659
+ enqueueRecordedTensorHealth(
660
+ recorder,
661
+ `L${layerIdx}.q_rope_HEALTH`,
662
+ qTensor,
663
+ qTensor.dtype,
664
+ numTokens * numHeads * headDim
665
+ );
666
+ enqueueRecordedTensorHealth(
667
+ recorder,
668
+ `L${layerIdx}.k_rope_HEALTH`,
669
+ kTensor,
670
+ kTensor?.dtype ?? null,
671
+ numTokens * numKVHeads * headDim
672
+ );
673
+ }
674
+
675
+ if (storeSharedKV && state.sharedAttentionState) {
676
+ state.sharedAttentionState.set(layerIdx, {
677
+ kTensor,
678
+ vTensor,
679
+ headDim,
680
+ numKVHeads,
681
+ });
682
+ }
683
+ resourceScope.register(qTensor.buffer, 'query', 'submitOwned');
684
+ if (qGateTensor?.buffer) {
685
+ resourceScope.register(qGateTensor.buffer, 'query-output-gate', 'submitOwned');
686
+ }
687
+ const kvOwnership = reusesSharedKV
688
+ ? 'borrowed'
689
+ : (storeSharedKV ? 'transferred' : 'submitOwned');
690
+ resourceScope.register(kTensor.buffer, 'key', kvOwnership);
691
+ resourceScope.register(vTensor.buffer, valueAliasesKey ? 'value-key-alias' : 'value', kvOwnership);
692
+
693
+ // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
694
+ if (!skipKVCacheWrites && !diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
695
+ if (kvCacheWriteFused) {
696
+ state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
697
+ } else if (state.kvCache.kvDtype === 'f16') {
698
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
699
+ if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
700
+ assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
701
+ }
702
+ if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
703
+ assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
704
+ }
705
+ const canWriteDirectF32ToF16 = kTensor.dtype === 'f32'
706
+ && vTensor.dtype === 'f32'
707
+ && typeof state.kvCache.recordUpdateF32ToF16FromGPU === 'function';
708
+ if (canWriteDirectF32ToF16) {
709
+ await state.kvCache.recordUpdateF32ToF16FromGPU(
710
+ recorder,
711
+ layerIdx,
712
+ kTensor.buffer,
713
+ vTensor.buffer,
714
+ currentSeqLen,
715
+ numTokens,
716
+ tokenIds
717
+ );
718
+ } else {
719
+ const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
720
+ const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
721
+
722
+ await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
723
+
724
+ if (kTensor.dtype !== 'f16') resourceScope.release(kCasted.buffer);
725
+ if (vTensor.dtype !== 'f16') resourceScope.release(vCasted.buffer);
726
+ }
727
+ } else {
728
+ await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kTensor.buffer, vTensor.buffer, currentSeqLen, numTokens, tokenIds);
729
+ }
730
+ }
731
+
732
+ // Resolve KV cache state and build dispatch parameters (shared with the immediate executor)
733
+ let kvState;
734
+ if (diffusionGemmaDecoder) {
735
+ const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
736
+ let decoderKTensor = kTensor;
737
+ let decoderVTensor = vTensor;
738
+ if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
739
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
740
+ if (!hasExplicitF16KvContract) {
741
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
742
+ }
743
+ decoderKTensor = await recordCastF32ToF16(recorder, decoderKTensor);
744
+ decoderKTemp = decoderKTensor;
745
+ } else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
746
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
747
+ decoderKTensor = await recordCastF16ToF32(recorder, decoderKTensor);
748
+ decoderKTemp = decoderKTensor;
749
+ }
750
+ if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
751
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
752
+ if (!hasExplicitF16KvContract) {
753
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
754
+ }
755
+ decoderVTensor = await recordCastF32ToF16(recorder, decoderVTensor);
756
+ decoderVTemp = decoderVTensor;
757
+ } else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
758
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
759
+ decoderVTensor = await recordCastF16ToF32(recorder, decoderVTensor);
760
+ decoderVTemp = decoderVTensor;
761
+ }
762
+ kvState = await createDiffusionGemmaDecoderKVState({
763
+ state,
764
+ layerIdx,
765
+ kTensor: decoderKTensor,
766
+ vTensor: decoderVTensor,
767
+ currentSeqLen,
768
+ numTokens,
769
+ numKVHeads,
770
+ headDim,
771
+ layerType,
772
+ slidingWindow,
773
+ kvDtype: decoderKVDtype,
774
+ recorder,
775
+ });
776
+ if (decoderKTemp) {
777
+ resourceScope.release(decoderKTemp.buffer);
778
+ decoderKTemp = null;
779
+ }
780
+ if (decoderVTemp) {
781
+ resourceScope.release(decoderVTemp.buffer);
782
+ decoderVTemp = null;
783
+ }
784
+ decoderKVState = kvState;
785
+ } else {
786
+ kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens, {
787
+ forceCurrentTensors: skipKVCacheWrites,
788
+ });
789
+ }
790
+ const dispatchConfig = {
791
+ layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
792
+ slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
793
+ layerType, layerTypes: config.layerTypes,
794
+ queryPreAttnScalar,
795
+ causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
796
+ activationDtype: attentionActivationDtype,
797
+ kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
798
+ };
799
+ const dispatchParams = buildAttentionDispatchParams(dispatchConfig, state, kTensor, vTensor, kvState);
800
+ const {
801
+ effectiveSlidingWindow, attentionKernelVariant, attnScale,
802
+ cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
803
+ prefillFallbackNeedsCast, causalForAttention,
804
+ } = dispatchParams;
805
+ const attentionKernelPath = skipKVCacheWrites ? null : kernelPath;
806
+
807
+ // 5. Attention
808
+
809
+ recordAttentionInputs(state, buildAttentionInputsData(
810
+ dispatchConfig, input, normed, kvState, dispatchParams,
811
+ { useF16Activations, matmulOutputDtype },
812
+ usedFusedQKV, qTensor, kTensor, vTensor,
813
+ ));
814
+ const mergedSessionRec = runtimeSession;
815
+ const plannedOProjLoRA = getLoRAModule(lora, layerIdx, 'o_proj');
816
+ attentionPlan = resolveAttentionPlanForDispatch({
817
+ config: {
818
+ ...config,
819
+ kernelPath: attentionKernelPath,
820
+ },
821
+ inputDtype: input.dtype,
822
+ activationDtype: attentionActivationDtype,
823
+ projectionDtype: matmulOutputDtype,
824
+ kvDtype: kvCacheDtype,
825
+ outputProjectionInputDtype: oProjInputDtype,
826
+ outputDtype: oProjOutputDtype,
827
+ outputProjectionWeightDtype: layerWeights.oProj
828
+ ? getWeightDtype(layerWeights.oProj)
829
+ : null,
830
+ outputProjectionHasLoRA: plannedOProjLoRA != null,
831
+ session: mergedSessionRec,
832
+ capabilities: getKernelCapabilities(),
833
+ fusion: {
834
+ inputNormProjection: canFuseInputNormProjRec,
835
+ qkvProjection: usedFusedQKV,
836
+ qkNorm: qkNormApplied,
837
+ qkNormRoPE: qkNormApplied && ropeApplied,
838
+ },
839
+ qGateTensor,
840
+ diagnosticsEligible: hasAttentionProjectionDiagnostics(state)
841
+ || hasAttentionStageDiagnostics(state, ['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']),
842
+ sharedKV: {
843
+ reuses: reusesSharedKV,
844
+ stores: storeSharedKV,
845
+ },
846
+ kvState,
847
+ dispatchParams,
848
+ });
849
+ if (attentionPlan.attention.implementation !== attentionKernelVariant) {
850
+ throw new Error(
851
+ `Attention plan/dispatch mismatch at layer ${layerIdx}: ` +
852
+ `plan=${attentionPlan.attention.implementation}, dispatch=${attentionKernelVariant}.`
853
+ );
854
+ }
855
+ if (state.observationContext?.receiptPolicy !== 'off') {
856
+ state.stats.attentionPlans ??= [];
857
+ state.stats.attentionPlans.push(attentionPlan);
858
+ }
859
+ let attentionOutputGateFused = false;
860
+
861
+ const attentionKernelRunners = {
862
+ bdpa: async () => {
863
+ const basisKDtype = 'f16';
864
+ const basisVDtype = 'f16';
865
+ const numBasisVectors = Math.max(1, kvState.bdpaBasisCount);
866
+ const basisKTensor = createTensor(kvState.bdpaBasisK, basisKDtype, [numBasisVectors, numKVHeads * headDim], 'bdpa_basis_k');
867
+ const basisVTensor = createTensor(kvState.bdpaBasisV, basisVDtype, [numBasisVectors, numKVHeads * headDim], 'bdpa_basis_v');
868
+
869
+ let qForBDPA = qTensor;
870
+ if (qForBDPA.dtype !== 'f16') {
871
+ assertAttentionDtypeTransitionAllowed(state, qForBDPA.dtype, 'f16', 'BDPA attention would narrow Q implicitly.');
872
+ qForBDPA = await recordCastF32ToF16(recorder, qTensor);
873
+ resourceScope.release(qForBDPA.buffer);
874
+ }
875
+
876
+ return recordAttentionBDPA(recorder, qForBDPA, basisKTensor, basisVTensor, kvState.bdpaPagedK, kvState.bdpaPagedV, kvState.bdpaIndex, numHeads, headDim, {
877
+ seqLen: numTokens,
878
+ kvLen: kvState.kvLenForAttention,
879
+ numKVHeads,
880
+ causal: causalForAttention,
881
+ startPos: kvState.startPosForMask,
882
+ layerIdx,
883
+ slidingWindow: effectiveSlidingWindow,
884
+ attnSoftcap,
885
+ scale: attnScale,
886
+ ropeCos: state.ropeFreqsCos,
887
+ ropeSin: state.ropeFreqsSin,
888
+ });
889
+ },
890
+ tieredQuant: async () => {
891
+ let qForAttention = qTensor;
892
+ if (kvState.coldQuantMode !== 'none' && qTensor.dtype !== 'f32') {
893
+ assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Tiered quant attention would widen Q implicitly.');
894
+ qForAttention = await recordCastF16ToF32(recorder, qTensor);
895
+ resourceScope.release(qForAttention.buffer);
896
+ }
897
+ if (kvState.coldQuantMode === 'none') {
898
+ throw new Error('Tiered quant attention requires cold quant mode.');
899
+ }
900
+ if (!kvState.coldScalesK || !kvState.coldScalesV) {
901
+ throw new Error('Tiered quant attention requires cold scale buffers.');
902
+ }
903
+
904
+ const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
905
+ const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
906
+ return recordAttentionTieredQuant(
907
+ recorder,
908
+ qForAttention,
909
+ cachedHotKTensor,
910
+ cachedHotVTensor,
911
+ kvState.cachedKCold,
912
+ kvState.cachedVCold,
913
+ kvState.coldScalesK,
914
+ kvState.coldScalesV,
915
+ numHeads,
916
+ headDim,
917
+ buildTieredQuantAttentionOptions(kvState, {
918
+ seqLen: numTokens,
919
+ numKVHeads,
920
+ causal: causalForAttention,
921
+ startPos: kvState.startPosForMask,
922
+ slidingWindow: effectiveSlidingWindow ?? 0,
923
+ attnSoftcap,
924
+ scale: attnScale,
925
+ })
926
+ );
927
+ },
928
+ contiguousQuant: async () => {
929
+ let qForAttention = qTensor;
930
+ if (qTensor.dtype !== 'f32') {
931
+ assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Contiguous quant attention would widen Q implicitly.');
932
+ qForAttention = await recordCastF16ToF32(recorder, qTensor);
933
+ resourceScope.release(qForAttention.buffer);
934
+ }
935
+
936
+ if (!kvState.coldScalesK || !kvState.coldScalesV) {
937
+ throw new Error('Contiguous quant attention requires scale buffers.');
938
+ }
939
+ if (!kvState.rotationMatrixBuffer || !kvState.codebookCentroidsBuffer) {
940
+ throw new Error('Contiguous quant attention requires TurboQuant shared buffers.');
941
+ }
942
+
943
+ return recordAttentionContiguousQuant(
944
+ recorder,
945
+ qForAttention,
946
+ kvState.cachedKCold,
947
+ kvState.cachedVCold,
948
+ kvState.coldScalesK,
949
+ kvState.coldScalesV,
950
+ numHeads,
951
+ headDim,
952
+ buildContiguousQuantAttentionOptions(kvState, {
953
+ seqLen: numTokens,
954
+ kvLen: kvState.kvLenForAttention,
955
+ numKVHeads,
956
+ causal: causalForAttention,
957
+ startPos: kvState.startPosForMask,
958
+ slidingWindow: effectiveSlidingWindow ?? 0,
959
+ attnSoftcap,
960
+ scale: attnScale,
961
+ })
962
+ );
963
+ },
964
+ tiered: async () => {
965
+ const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
966
+ const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
967
+ const cachedColdKTensor = createTensor(kvState.cachedKCold, cachedKDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_K_cold');
968
+ const cachedColdVTensor = createTensor(kvState.cachedVCold, cachedVDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_V_cold');
969
+ return recordAttentionTiered(recorder, qTensor, cachedHotKTensor, cachedHotVTensor, cachedColdKTensor, cachedColdVTensor, numHeads, headDim, {
970
+ seqLen: numTokens,
971
+ coldLen: kvState.coldLen,
972
+ hotLen: kvState.hotLen,
973
+ numKVHeads,
974
+ causal: causalForAttention,
975
+ startPos: kvState.startPosForMask,
976
+ slidingWindow: effectiveSlidingWindow ?? 0,
977
+ attnSoftcap,
978
+ scale: attnScale,
979
+ hotWindow: kvState.hotWindow,
980
+ hotStart: kvState.hotStart,
981
+ coldPageTable: kvState.coldPageTable,
982
+ coldPageSize: kvState.coldPageSize,
983
+ coldLayout: kvState.coldPageTable ? 2 : 0,
984
+ hotLayout: kvState.hotWindow > 0 ? 1 : 0,
985
+ });
986
+ },
987
+ contiguous: async () => {
988
+ // Prefill fallback: quantized/tiered layouts use raw K/V for prefill, cast to f16 to match kernel path
989
+ let kForAttn = cachedKTensor;
990
+ let vForAttn = cachedVTensor;
991
+ if (prefillFallbackNeedsCast) {
992
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
993
+ if (cachedKDtype === 'f16' && kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
994
+ assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'Prefill fallback attention would narrow K implicitly.');
995
+ }
996
+ if (cachedVDtype === 'f16' && vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
997
+ assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'Prefill fallback attention would narrow V implicitly.');
998
+ }
999
+ const kCasted = cachedKDtype === 'f16' && kTensor.dtype !== 'f16'
1000
+ ? await recordCastF32ToF16(recorder, kTensor) : kTensor;
1001
+ const vCasted = cachedVDtype === 'f16' && vTensor.dtype !== 'f16'
1002
+ ? await recordCastF32ToF16(recorder, vTensor) : vTensor;
1003
+ kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
1004
+ vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
1005
+ if (kTensor.dtype !== 'f16') resourceScope.release(kCasted.buffer);
1006
+ if (vTensor.dtype !== 'f16') resourceScope.release(vCasted.buffer);
1007
+ }
1008
+ // Session precedence was resolved once at pipeline construction. The
1009
+ // immutable runtime session is the sole attention policy input.
1010
+ // Kernel enforces head_dim=256, f16 KV, contiguous layout; only applies
1011
+ // when numTokens > 1 (prefill). The semantic plan owns the flag.
1012
+ const useFlashPrefillRec = attentionPlan.attention.flashPrefill === true;
1013
+ const useOrtFlashPrefillRec = attentionPlan.attention.ortFlashPrefill === true;
1014
+ const useOutputGateFusion = attentionPlan.outputGate.mode === 'attention-epilogue';
1015
+ const result = await recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
1016
+ seqLen: numTokens,
1017
+ kvLen: kvState.kvLenForAttention,
1018
+ numKVHeads,
1019
+ causal: causalForAttention,
1020
+ bidirectionalSpanStart: multimodalBidirectionalSpan?.start ?? 0,
1021
+ bidirectionalSpanLength: multimodalBidirectionalSpan?.length ?? 0,
1022
+ startPos: kvState.startPosForMask,
1023
+ layerIdx,
1024
+ slidingWindow: effectiveSlidingWindow,
1025
+ attnSoftcap,
1026
+ scale: attnScale,
1027
+ kvStart: kvState.kvStart,
1028
+ kvLayout: kvState.kvLayout,
1029
+ kvPageTable: kvState.kvPageTable,
1030
+ kvPageSize: kvState.kvPageSize,
1031
+ kernelPath: attentionKernelPath,
1032
+ useFlashPrefill: skipKVCacheWrites ? false : useFlashPrefillRec,
1033
+ useOrtFlashPrefill: skipKVCacheWrites ? false : useOrtFlashPrefillRec,
1034
+ outputGate: useOutputGateFusion ? qGateTensor : null,
1035
+ });
1036
+ attentionOutputGateFused = result?.outputGateFused === true;
1037
+ return result;
1038
+ },
1039
+ };
1040
+ const runAttentionKernel = attentionKernelRunners[attentionPlan.attention.implementation];
1041
+ if (!runAttentionKernel) {
1042
+ throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
1043
+ }
1044
+
1045
+ try {
1046
+ const boundPlan = bindAttentionPlan(attentionPlan, { runAttentionKernel });
1047
+ attnOutput = await executeBoundAttentionPlan(
1048
+ boundPlan,
1049
+ createAttentionExecutor('recorded', {
1050
+ attention: (bound) => bound.resources.runAttentionKernel(),
1051
+ })
1052
+ );
1053
+ resourceScope.register(attnOutput.buffer, 'attention-core-output', 'submitOwned');
1054
+ } finally {
1055
+ if (decoderKVState?.ownedBuffers) {
1056
+ for (const buffer of decoderKVState.ownedBuffers) {
1057
+ resourceScope.release(buffer);
1058
+ }
1059
+ decoderKVState.ownedBuffers = null;
1060
+ }
1061
+ }
1062
+ await runProbes('attn_core_out', attnOutput.buffer, {
1063
+ layerIdx,
1064
+ numTokens,
1065
+ hiddenSize: numHeads * headDim,
1066
+ probes: state.debugProbes,
1067
+ recorder,
1068
+ operatorDiagnostics: state.operatorDiagnostics,
1069
+ dtype: attnOutput.dtype,
1070
+ });
1071
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
1072
+ enqueueRecordedTensorHealth(
1073
+ recorder,
1074
+ `L${layerIdx}.attn_core_out_HEALTH`,
1075
+ attnOutput,
1076
+ attnOutput.dtype,
1077
+ numTokens * numHeads * headDim
1078
+ );
1079
+ }
1080
+
1081
+ attnForProjection = attnOutput;
1082
+ if (qGateTensor && !attentionOutputGateFused) {
1083
+ // The shared plan selects the same gate semantics for both executors.
1084
+ // sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
1085
+ const gateActivation = attentionPlan.outputGate.semantics;
1086
+ attnForProjection = await recordSiLU(recorder, attnOutput, {
1087
+ size: numTokens * numHeads * headDim,
1088
+ gate: qGateTensor,
1089
+ useVec4: (numTokens * numHeads * headDim) % 4 === 0,
1090
+ gateActivation,
1091
+ inputActivation: 'identity',
1092
+ swigluLimit: null,
1093
+ });
1094
+ resourceScope.register(attnForProjection.buffer, 'gated-attention-output', 'submitOwned');
1095
+ resourceScope.release(attnOutput.buffer);
1096
+ }
1097
+
1098
+ // 6. Output projection (with optional fused residual for decode)
1099
+
1100
+ output = null;
1101
+ let residualFused = false;
1102
+ let oProjInput = attnForProjection;
1103
+ oProjInputTemp = null;
1104
+ if (layerWeights.oProj && getWeightBuffer) {
1105
+ ({ oProjInput, oProjInputTemp } = await prepareAttentionProjectionInput(
1106
+ attnForProjection,
1107
+ oProjInputDtype,
1108
+ (tensor) => {
1109
+ assertAttentionDtypeTransitionAllowed(
1110
+ state,
1111
+ tensor.dtype,
1112
+ oProjInputDtype,
1113
+ 'Attention output projection would change activations implicitly.',
1114
+ 'step_precision'
1115
+ );
1116
+ return oProjInputDtype === 'f16'
1117
+ ? recordCastF32ToF16(recorder, tensor)
1118
+ : recordCastF16ToF32(recorder, tensor);
1119
+ }
1120
+ ));
1121
+ const oProjBuf = getWeightBuffer(layerWeights.oProj, 'o_proj');
1122
+ const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
1123
+
1124
+ // Use fused o_proj + residual for decode when possible
1125
+ // Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
1126
+ const oProjDtype = getWeightDtype(oProjBuf);
1127
+ const canUseFused = attentionPlan.outputProjection.residualMode === 'matmul-residual';
1128
+ const canUseWideTileResidual =
1129
+ attentionPlan.outputProjection.residualMode === 'wide-tile-epilogue';
1130
+ if (
1131
+ attentionPlan.outputProjection.weightDtype != null
1132
+ && oProjDtype !== attentionPlan.outputProjection.weightDtype
1133
+ ) {
1134
+ throw new Error(
1135
+ `Attention output projection dtype changed after planning at layer ${layerIdx}: ` +
1136
+ `plan=${attentionPlan.outputProjection.weightDtype}, bound=${oProjDtype}.`
1137
+ );
1138
+ }
1139
+ if (canUseFused && residualTensor) {
1140
+ // FUSED PATH: o_proj matmul + residual add in one dispatch
1141
+ output = await recordMatmulResidualFused(recorder, oProjInput, oProjBuf, residualTensor, {
1142
+ N: hiddenSize,
1143
+ K: numHeads * headDim,
1144
+ });
1145
+ residualFused = true;
1146
+ } else {
1147
+ // STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
1148
+ output = await recordMatmul(recorder, oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
1149
+ transposeB: 'auto',
1150
+ role: 'o_proj',
1151
+ layerIdx,
1152
+ kernelPath,
1153
+ outputDtype: oProjOutputDtype,
1154
+ executionPolicies: state.executionPolicies ?? null,
1155
+ residualTensor: canUseWideTileResidual ? residualTensor : null,
1156
+ });
1157
+ residualFused = canUseWideTileResidual;
1158
+ }
1159
+ // Release temporary buffer if we created it (original was not already on GPU)
1160
+ if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
1161
+ resourceScope.release(isWeightBuffer(oProjBuf) ? oProjBuf.buffer : oProjBuf);
1162
+ }
1163
+ } else {
1164
+ output = oProjInput;
1165
+ }
1166
+
1167
+ // Apply LoRA to output projection if present (only if not using fused path)
1168
+ if (!residualFused) {
1169
+ const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
1170
+ if (loraO && getWeightBuffer) {
1171
+ const combined = await applyLoRA(
1172
+ oProjInput,
1173
+ output,
1174
+ loraO,
1175
+ { M: numTokens, N: hiddenSize, K: numHeads * headDim },
1176
+ getWeightBuffer,
1177
+ recorder,
1178
+ { kernelPath }
1179
+ );
1180
+ if (combined.buffer !== output.buffer) {
1181
+ resourceScope.release(output.buffer);
1182
+ output = combined;
1183
+ }
1184
+ }
1185
+ }
1186
+
1187
+ if (layerWeights.oProjBias) {
1188
+ const { tensor: oProjBiasTensor, owned } = getVectorTensor(
1189
+ layerWeights.oProjBias,
1190
+ `L${layerIdx}.o_proj_bias`,
1191
+ hiddenSize,
1192
+ {},
1193
+ debugFlags
1194
+ );
1195
+ output = await recordBiasAdd(
1196
+ recorder,
1197
+ output,
1198
+ oProjBiasTensor,
1199
+ numTokens,
1200
+ hiddenSize,
1201
+ {
1202
+ label: `L${layerIdx}.o_proj_bias`,
1203
+ layerIdx,
1204
+ executionPolicies: state.executionPolicies ?? null,
1205
+ }
1206
+ );
1207
+ if (owned) resourceScope.release(oProjBiasTensor.buffer);
1208
+ }
1209
+
1210
+ finalOutput = output;
1211
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
1212
+ enqueueRecordedTensorHealth(
1213
+ recorder,
1214
+ `L${layerIdx}.o_proj_HEALTH`,
1215
+ output,
1216
+ output.dtype,
1217
+ numTokens * hiddenSize
1218
+ );
1219
+ }
1220
+
1221
+ const buffersToTrack = [];
1222
+ if (output.buffer !== attnForProjection.buffer) {
1223
+ buffersToTrack.push(attnForProjection.buffer);
1224
+ }
1225
+ if (oProjInputTemp && oProjInputTemp.buffer !== attnForProjection.buffer) {
1226
+ buffersToTrack.push(oProjInputTemp.buffer);
1227
+ }
1228
+ if (output.dtype !== oProjOutputDtype) {
1229
+ assertAttentionDtypeTransitionAllowed(state, output.dtype, oProjOutputDtype, 'Attention output would change implicitly before leaving the layer.');
1230
+ const coercedOutput = oProjOutputDtype === 'f16'
1231
+ ? await recordCastF32ToF16(recorder, output)
1232
+ : await recordCastF16ToF32(recorder, output);
1233
+ buffersToTrack.push(output.buffer);
1234
+ finalOutput = coercedOutput;
1235
+ resourceScope.register(finalOutput.buffer, 'coerced-attention-output', 'submitOwned');
1236
+ }
1237
+
1238
+ // Track intermediate buffers for cleanup after submit (not release!)
1239
+ // These buffers are used by recorded operations that haven't executed yet.
1240
+ // Releasing them back to the pool would allow reuse before the encoder is submitted,
1241
+ // causing data corruption (especially for small decode buffers).
1242
+ if (qTensor) {
1243
+ resourceScope.release(qTensor.buffer);
1244
+ }
1245
+ if (qGateTensor) {
1246
+ resourceScope.release(qGateTensor.buffer);
1247
+ }
1248
+ if (!retainSharedKvBuffers) {
1249
+ if (kTensor) {
1250
+ resourceScope.release(kTensor.buffer);
1251
+ }
1252
+ if (vTensor?.buffer && vTensor.buffer !== kTensor?.buffer) {
1253
+ resourceScope.release(vTensor.buffer);
1254
+ }
1255
+ }
1256
+ for (const buffer of buffersToTrack) {
1257
+ resourceScope.release(buffer);
1258
+ }
1259
+ resourceScope.register(finalOutput.buffer, 'attention-output', 'submitOwned');
1260
+ resourceScope.retain(finalOutput.buffer, 'attention-output', 'returned-to-layer');
1261
+
1262
+ const resourceEvents = resourceScope.close('success');
1263
+ if (state.observationContext?.receiptPolicy !== 'off') {
1264
+ state.stats.attentionResourceEvents ??= [];
1265
+ state.stats.attentionResourceEvents.push(...resourceEvents);
1266
+ }
1267
+ captureAttentionRefactorReceipt({
1268
+ state,
1269
+ plan: attentionPlan,
1270
+ resourceEvents,
1271
+ });
1272
+ return { output: finalOutput, residualFused };
1273
+ } catch (error) {
1274
+ const trackOnce = (buffer) => resourceScope.release(buffer, 'attention-failure');
1275
+ if (finalOutput?.buffer && finalOutput.buffer !== output?.buffer) {
1276
+ trackOnce(finalOutput.buffer);
1277
+ }
1278
+ if (output?.buffer && output.buffer !== attnForProjection?.buffer) {
1279
+ trackOnce(output.buffer);
1280
+ }
1281
+ if (oProjInputTemp?.buffer) {
1282
+ trackOnce(oProjInputTemp.buffer);
1283
+ }
1284
+ if (attnForProjection?.buffer && attnForProjection.buffer !== attnOutput?.buffer) {
1285
+ trackOnce(attnForProjection.buffer);
1286
+ }
1287
+ if (attnOutput?.buffer) {
1288
+ trackOnce(attnOutput.buffer);
1289
+ }
1290
+ if (qGateTensor?.buffer) {
1291
+ trackOnce(qGateTensor.buffer);
1292
+ }
1293
+ if (qTensor?.buffer) {
1294
+ trackOnce(qTensor.buffer);
1295
+ }
1296
+ if (kTensor?.buffer && !retainSharedKvBuffers) {
1297
+ trackOnce(kTensor.buffer);
1298
+ }
1299
+ if (vTensor?.buffer && !retainSharedKvBuffers) {
1300
+ trackOnce(vTensor.buffer);
1301
+ }
1302
+ if (normed?.buffer && normed.buffer !== attentionInput?.buffer) {
1303
+ trackOnce(normed.buffer);
1304
+ }
1305
+ if (attentionInputTemp && attentionInput?.buffer) {
1306
+ trackOnce(attentionInput.buffer);
1307
+ }
1308
+ if (decoderKTemp?.buffer) {
1309
+ trackOnce(decoderKTemp.buffer);
1310
+ }
1311
+ if (decoderVTemp?.buffer) {
1312
+ trackOnce(decoderVTemp.buffer);
1313
+ }
1314
+ if (decoderKVState?.ownedBuffers) {
1315
+ for (const buffer of decoderKVState.ownedBuffers) {
1316
+ trackOnce(buffer);
1317
+ }
1318
+ decoderKVState.ownedBuffers = null;
1319
+ }
1320
+ const resourceEvents = resourceScope.close('failure');
1321
+ if (state.observationContext?.receiptPolicy !== 'off') {
1322
+ state.stats.attentionResourceEvents ??= [];
1323
+ state.stats.attentionResourceEvents.push(...resourceEvents);
1324
+ }
1325
+ captureAttentionRefactorReceipt({
1326
+ state,
1327
+ plan: attentionPlan,
1328
+ resourceEvents,
1329
+ error,
1330
+ failureBoundary: attentionPlan ? 'execute-bound-attention-plan' : 'bind-attention-plan',
1331
+ });
1332
+ throw error;
1333
+ }
1334
+ }