doppler-gpu 0.4.14 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (222) hide show
  1. package/CHANGELOG.md +9 -0
  2. package/README.md +83 -52
  3. package/package.json +25 -2
  4. package/src/cli/doppler-cli.js +253 -94
  5. package/src/cli/runtime-profiles.js +3 -7
  6. package/src/client/doppler-api.d.ts +28 -0
  7. package/src/client/doppler-api.js +2 -0
  8. package/src/client/inspection.d.ts +122 -0
  9. package/src/client/inspection.js +343 -0
  10. package/src/client/runtime/index.d.ts +54 -0
  11. package/src/client/runtime/index.js +190 -18
  12. package/src/client/runtime/model-manager.d.ts +0 -4
  13. package/src/client/runtime/model-manager.js +0 -28
  14. package/src/client/runtime/model-session.d.ts +116 -1
  15. package/src/client/runtime/model-session.js +351 -1
  16. package/src/client/runtime/model-source.d.ts +10 -1
  17. package/src/client/runtime/scoped-session.d.ts +117 -0
  18. package/src/client/runtime/scoped-session.js +257 -0
  19. package/src/client/runtime/types.d.ts +0 -1
  20. package/src/client/runtime/types.js +0 -1
  21. package/src/config/evidence/boundary-evidence-policy.json +54 -0
  22. package/src/config/evidence/source-intake-policy.json +122 -0
  23. package/src/config/inspection/observation-policies.json +80 -0
  24. package/src/config/param-validator.js +21 -36
  25. package/src/config/runtime/compute/f16-activations.json +1 -0
  26. package/src/config/runtime/compute/f16-batched.json +1 -0
  27. package/src/config/runtime/default.json +1 -0
  28. package/src/config/runtime/diagnostics/debug-logits.json +1 -0
  29. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
  30. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
  31. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
  32. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
  33. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
  34. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
  35. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
  36. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
  37. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
  38. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
  39. package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
  40. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
  41. package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
  42. package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
  43. package/src/config/runtime/model/gemma2-debug.json +1 -0
  44. package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
  45. package/src/config/runtime/model/gemma2-pipeline.json +1 -0
  46. package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
  47. package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
  48. package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
  49. package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
  50. package/src/config/runtime/optimization-candidates/registry.json +5 -0
  51. package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
  52. package/src/config/runtime/profiles/default.json +1 -0
  53. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
  54. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
  55. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
  56. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
  57. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
  58. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
  59. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
  60. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
  61. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
  62. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
  63. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
  64. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
  65. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
  66. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
  67. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
  68. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
  69. package/src/config/runtime/profiles/low-memory.json +1 -0
  70. package/src/config/runtime/profiles/production.json +1 -3
  71. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
  72. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
  73. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
  74. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
  75. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
  76. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
  77. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
  78. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
  79. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
  80. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
  81. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
  82. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
  83. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
  84. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
  85. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
  86. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
  87. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
  89. package/src/config/runtime/profiles/rerank-stability.json +1 -3
  90. package/src/config/runtime/profiles/rerank-throughput.json +1 -3
  91. package/src/config/runtime/profiles/simulation.json +1 -3
  92. package/src/config/runtime/profiles/throughput.json +1 -3
  93. package/src/config/runtime/profiles/trace-layers.json +1 -3
  94. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
  95. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
  96. package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
  97. package/src/config/runtime/profiles/turboquant.json +1 -0
  98. package/src/config/runtime/profiles/vector-stability.json +1 -3
  99. package/src/config/runtime/profiles/vector-throughput.json +1 -0
  100. package/src/config/runtime/profiles/verbose-trace.json +1 -3
  101. package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
  102. package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
  103. package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
  104. package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
  105. package/src/config/schema/benchmark.schema.d.ts +7 -0
  106. package/src/config/schema/benchmark.schema.js +5 -0
  107. package/src/config/schema/index.d.ts +8 -16
  108. package/src/config/schema/index.js +5 -8
  109. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
  110. package/src/config/schema/kernel-thresholds.schema.js +0 -12
  111. package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
  112. package/src/config/schema/kernel-warmup.schema.js +0 -1
  113. package/src/config/schema/runtime-profile.schema.d.ts +24 -0
  114. package/src/config/schema/runtime-profile.schema.js +92 -0
  115. package/src/config/schema/shared-runtime.schema.d.ts +1 -4
  116. package/src/config/schema/shared-runtime.schema.js +0 -2
  117. package/src/config/schema/tooling.schema.d.ts +8 -5
  118. package/src/config/schema/tooling.schema.js +3 -2
  119. package/src/debug/tensor.d.ts +2 -0
  120. package/src/debug/tensor.js +21 -2
  121. package/src/gpu/command-recorder.d.ts +5 -0
  122. package/src/gpu/command-recorder.js +17 -0
  123. package/src/gpu/kernel-runtime.js +2 -10
  124. package/src/gpu/kernels/index.d.ts +0 -2
  125. package/src/gpu/kernels/index.js +0 -2
  126. package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
  127. package/src/gpu/kernels/kernel-prewarm.js +60 -0
  128. package/src/gpu/kernels/utils.d.ts +2 -4
  129. package/src/gpu/kernels/utils.js +2 -4
  130. package/src/index.d.ts +13 -1
  131. package/src/index.js +7 -1
  132. package/src/inference/browser-harness-model-helpers.js +4 -0
  133. package/src/inference/browser-harness-runtime-helpers.js +25 -10
  134. package/src/inference/browser-harness-text-helpers.js +4 -1
  135. package/src/inference/browser-harness.d.ts +4 -0
  136. package/src/inference/browser-harness.js +91 -4
  137. package/src/inference/observation-context.d.ts +22 -0
  138. package/src/inference/observation-context.js +41 -0
  139. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  140. package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
  141. package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
  142. package/src/inference/pipelines/text/attention/index.js +7 -0
  143. package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
  144. package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
  145. package/src/inference/pipelines/text/attention/plan.js +328 -0
  146. package/src/inference/pipelines/text/attention/receipt.js +60 -0
  147. package/src/inference/pipelines/text/attention/record.js +8 -1278
  148. package/src/inference/pipelines/text/attention/run.js +8 -1386
  149. package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-steps.js +32 -15
  152. package/src/inference/pipelines/text/generator.js +10 -5
  153. package/src/inference/pipelines/text/init.d.ts +3 -0
  154. package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
  155. package/src/inference/pipelines/text/layer.js +2 -0
  156. package/src/inference/pipelines/text/model-load.d.ts +0 -1
  157. package/src/inference/pipelines/text/model-load.js +2 -11
  158. package/src/inference/pipelines/text/probes.js +10 -0
  159. package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
  160. package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
  161. package/src/inference/pipelines/text/state.d.ts +6 -0
  162. package/src/inference/pipelines/text/state.js +3 -0
  163. package/src/inference/pipelines/text/types.d.ts +6 -0
  164. package/src/inference/pipelines/text.js +16 -1
  165. package/src/inference/refactor-receipt.d.ts +49 -0
  166. package/src/inference/refactor-receipt.js +188 -0
  167. package/src/inference/resource-scope.d.ts +33 -0
  168. package/src/inference/resource-scope.js +168 -0
  169. package/src/inference/test-harness.d.ts +3 -0
  170. package/src/inference/test-harness.js +1 -0
  171. package/src/storage/model-cache.d.ts +53 -0
  172. package/src/storage/model-cache.js +550 -0
  173. package/src/tooling/boundary-evidence.d.ts +51 -0
  174. package/src/tooling/boundary-evidence.js +408 -0
  175. package/src/tooling/browser-command-runner.js +4 -4
  176. package/src/tooling/command-context.d.ts +25 -0
  177. package/src/tooling/command-context.js +53 -0
  178. package/src/tooling/command-runner-shared.js +22 -5
  179. package/src/tooling/demo-receipts.d.ts +56 -0
  180. package/src/tooling/demo-receipts.js +113 -0
  181. package/src/tooling/diagnose-runner.js +2 -2
  182. package/src/tooling/execution-cost-ledger.d.ts +21 -0
  183. package/src/tooling/execution-cost-ledger.js +199 -0
  184. package/src/tooling/node-command-runner.js +2 -2
  185. package/src/tooling/opfs-cache.d.ts +11 -49
  186. package/src/tooling/opfs-cache.js +4 -510
  187. package/src/tooling/precision-replay-math.d.ts +42 -0
  188. package/src/tooling/precision-replay-math.js +174 -0
  189. package/src/tooling/program-bundle-parity.d.ts +3 -0
  190. package/src/tooling/program-bundle-parity.js +8 -0
  191. package/src/tooling/program-bundle.d.ts +1 -52
  192. package/src/tooling/program-bundle.js +0 -139
  193. package/src/tooling/registered-variant-calibration.d.ts +24 -0
  194. package/src/tooling/registered-variant-calibration.js +286 -0
  195. package/src/tooling/runtime-input-composition.js +105 -53
  196. package/src/tooling/runtime-intent.js +41 -0
  197. package/src/tooling/runtime-optimization-index.d.ts +6 -0
  198. package/src/tooling/runtime-optimization-index.js +72 -0
  199. package/src/tooling/runtime-optimization.d.ts +47 -6
  200. package/src/tooling/runtime-optimization.js +559 -40
  201. package/src/tooling/source-intake.d.ts +49 -0
  202. package/src/tooling/source-intake.js +414 -0
  203. package/src/tooling-exports/evidence.d.ts +34 -0
  204. package/src/tooling-exports/evidence.js +34 -0
  205. package/src/tooling-exports/runtime.d.ts +7 -0
  206. package/src/tooling-exports/runtime.js +7 -0
  207. package/src/tooling-exports.d.ts +11 -0
  208. package/src/tooling-exports.js +6 -3
  209. package/src/tooling-exports.shared.d.ts +32 -0
  210. package/src/tooling-exports.shared.js +31 -0
  211. package/src/version.js +1 -1
  212. package/src/config/schema/tuner.schema.d.ts +0 -39
  213. package/src/config/schema/tuner.schema.js +0 -13
  214. package/src/gpu/kernel-tuner/benchmarks.js +0 -902
  215. package/src/gpu/kernel-tuner/cache.js +0 -145
  216. package/src/gpu/kernel-tuner/index.js +0 -38
  217. package/src/gpu/kernel-tuner/tuner.js +0 -247
  218. package/src/gpu/kernel-tuner.js +0 -12
  219. package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
  220. package/src/gpu/kernels/kernel-tuning.js +0 -158
  221. package/src/gpu/profiler.js +0 -382
  222. package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
@@ -1,1278 +1,8 @@
1
-
2
-
3
- import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
- import { getKernelCapabilities } from '../../../../gpu/device.js';
5
- import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
6
- import {
7
- recordMatmul,
8
- recordRMSNorm,
9
- recordLayerNorm,
10
- recordBiasAdd,
11
- recordRoPE,
12
- canUseRoPEQK,
13
- recordRoPEQK,
14
- recordAttention,
15
- recordAttentionTiered,
16
- recordAttentionTieredQuant,
17
- recordAttentionContiguousQuant,
18
- recordAttentionBDPA,
19
- recordSiLU,
20
- recordCastF16ToF32,
21
- recordCastF32ToF16,
22
- recordMatmulResidualFused,
23
- shouldUseFusedMatmulResidual,
24
- } from '../../../../gpu/kernel-selector.js';
25
- import { createTensor } from '../../../../gpu/tensor.js';
26
- import { applyLoRA } from '../lora-apply.js';
27
- import { getLoRAModule } from '../lora.js';
28
- import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
29
- import { selectRuleValue } from '../../../../rules/rule-registry.js';
30
- import {
31
- recordAttentionInputs,
32
- shouldForceF32AttentionProjectionForRoPE,
33
- resolveAttentionProjectionOutputDtype,
34
- projectAttentionQKV,
35
- applyAttentionQKNorm,
36
- applyAttentionValueNorm,
37
- hasAttentionProjectionDiagnostics,
38
- hasAttentionStageDiagnostics,
39
- resolveAttentionQKNormState,
40
- } from './projections.js';
41
- import { prepareAttentionProjectionInput } from './output-projection.js';
42
- import { runProbes } from '../probes.js';
43
- import { decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
44
-
45
- import { releaseOrTrack, shouldDebugLayer } from './types.js';
46
- import {
47
- getKernelPathMatmulPrecision,
48
- getKernelPathMatmulVariant,
49
- } from '../../../../config/kernel-path-loader.js';
50
- import {
51
- resolveKVCacheState,
52
- createDiffusionGemmaDecoderKVState,
53
- buildAttentionDispatchParams,
54
- buildAttentionInputsData,
55
- } from './dispatch-params.js';
56
- import {
57
- buildTieredQuantAttentionOptions,
58
- buildContiguousQuantAttentionOptions,
59
- } from './quant-options.js';
60
- import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
61
- import { getRuntimeConfig } from '../../../../config/runtime.js';
62
- import {
63
- resolveAttentionPrecisionContract,
64
- isAttentionKvDtypeExplicit,
65
- } from './precision-contract.js';
66
- import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
67
- import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
68
- import { getVectorTensor } from '../weights.js';
69
-
70
- const ATTENTION_DTYPE_LOGGED = new Set();
71
-
72
- function isWideTileQ4KPhaseEnabled(session, phase) {
73
- return phase === 'decode'
74
- ? session?.useWideTileQ4KDecode === true
75
- : session?.useWideTileQ4KPrefill === true;
76
- }
77
-
78
- function canUseQ4KWideTileResidualFusion(options = {}) {
79
- return options.hasResidual === true
80
- && options.oProjDtype === 'q4k'
81
- && options.inputDtype === 'f32'
82
- && options.outputDtype === 'f32'
83
- && options.residualMatches === true
84
- && options.hasLoRA !== true
85
- && getKernelCapabilities().hasF16 === true
86
- && getKernelCapabilities().hasSubgroups === true
87
- && options.session?.useWideTileResidualFusion === true
88
- && options.session?.retainQ4KMaterialization === true
89
- && isWideTileQ4KPhaseEnabled(options.session, options.phase);
90
- }
91
-
92
- function shouldTraceRecordedHealth(layerIdx, debugFlags) {
93
- const debugLayers = debugFlags?.debugLayers;
94
- return isTraceEnabled('logits')
95
- && Array.isArray(debugLayers)
96
- && shouldDebugLayer(layerIdx, debugLayers);
97
- }
98
-
99
- function enqueueRecordedTensorHealth(recorder, label, tensor, dtype, elementCount) {
100
- if (!recorder || !tensor?.buffer || !Number.isFinite(elementCount) || elementCount <= 0) {
101
- return;
102
- }
103
- const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
104
- recorder.enqueueCompletionTask(async () => {
105
- const data = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
106
- trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
107
- });
108
- }
109
-
110
- function resolveDirectF16KVCacheWrite(options) {
111
- const {
112
- state,
113
- layerIdx,
114
- currentSeqLen,
115
- numTokens,
116
- numKVHeads,
117
- headDim,
118
- reusesSharedKV,
119
- storeSharedKV,
120
- diffusionGemmaDecoder,
121
- valueNorm,
122
- diagnosticsEnabled,
123
- disableRoPE,
124
- } = options;
125
- if (
126
- numTokens !== 1
127
- || reusesSharedKV === true
128
- || storeSharedKV === true
129
- || diffusionGemmaDecoder === true
130
- || valueNorm === true
131
- || diagnosticsEnabled === true
132
- || disableRoPE === true
133
- ) {
134
- return null;
135
- }
136
- const cache = state?.kvCache;
137
- if (
138
- !cache?.hasGPUCache?.()
139
- || cache.kvDtype !== 'f16'
140
- || cache.layout !== 'contiguous'
141
- || cache.windowSize != null
142
- || cache.layerSpecs != null
143
- || typeof cache.recordF16UpdateAlreadyWrittenFromGPU !== 'function'
144
- ) {
145
- return null;
146
- }
147
- const gpuBuffers = cache.getGPUBuffers(layerIdx);
148
- const layout = gpuBuffers?.layout ?? cache.layout;
149
- if (layout !== 'contiguous' || !gpuBuffers?.keysGPU || !gpuBuffers?.valuesGPU) {
150
- return null;
151
- }
152
- return {
153
- keysBuffer: gpuBuffers.keysGPU,
154
- valuesBuffer: gpuBuffers.valuesGPU,
155
- dstOffset: currentSeqLen * numKVHeads * headDim,
156
- };
157
- }
158
-
159
- function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
160
- assertImplicitDtypeTransitionAllowed({
161
- executionPolicies: state?.executionPolicies ?? null,
162
- fromDtype,
163
- toDtype,
164
- op: 'attention',
165
- detail,
166
- transitionDeclaredBy,
167
- });
168
- }
169
-
170
-
171
- export async function recordLayerAttentionGPU(
172
- recorder,
173
- input,
174
- layerWeights,
175
- config,
176
- state,
177
- debug = false,
178
- debugFlags = {},
179
- getWeightBuffer,
180
- getNormWeightBuffer,
181
- debugCheckBuffer,
182
- lora
183
- ) {
184
- const {
185
- layerIdx,
186
- numTokens,
187
- isPrefill,
188
- numHeads,
189
- numKVHeads,
190
- headDim,
191
- hiddenSize,
192
- rmsNormEps,
193
- currentSeqLen,
194
- slidingWindow,
195
- layerType,
196
- residualTensor,
197
- attnSoftcap,
198
- queryPreAttnScalar,
199
- skipInputNorm = false,
200
- tokenIds = null,
201
- kernelPath = null,
202
- disableRoPE = false,
203
- multimodalBidirectionalSpan = null,
204
- sharedKVSourceLayerIdx = null,
205
- storeSharedKV = false,
206
- diffusionGemmaDecoder = false,
207
- precomputedInputNorm = null,
208
- } = config;
209
-
210
- const phase = isPrefill ? 'prefill' : 'decode';
211
- const attentionPrecisionContract = resolveAttentionPrecisionContract(config, state);
212
- const attentionActivationDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
213
- dtype: attentionPrecisionContract.resolvedActivationDtype ?? config.activationDtype,
214
- });
215
- const oProjPrecision = getKernelPathMatmulPrecision('o_proj', phase, layerIdx, kernelPath);
216
- const oProjInputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
217
- dtype: oProjPrecision?.inputDtype ?? attentionActivationDtype,
218
- });
219
- const oProjOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
220
- dtype: oProjPrecision?.outputDtype
221
- ?? attentionPrecisionContract.resolvedOutputDtype
222
- ?? config.activationDtype,
223
- });
224
- const wantsF16Output = oProjOutputDtype === 'f16';
225
- const useF16Activations = attentionActivationDtype === 'f16';
226
- const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
227
- const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
228
- const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
229
- const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
230
- ? 'step_precision'
231
- : null;
232
- let attentionInput = input;
233
- let attentionInputTemp = false;
234
- let normed = attentionInput;
235
- let qTensor = null;
236
- let qGateTensor = null;
237
- let kTensor = null;
238
- let vTensor = null;
239
- let attnOutput = null;
240
- let attnForProjection = null;
241
- let output = null;
242
- let finalOutput = null;
243
- let oProjInputTemp = null;
244
- let retainSharedKvBuffers = false;
245
- let valueAliasesKey = false;
246
- let decoderKVState = null;
247
- let decoderKTemp = null;
248
- let decoderVTemp = null;
249
- if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
250
- assertAttentionDtypeTransitionAllowed(
251
- state,
252
- input.dtype,
253
- attentionActivationDtype,
254
- 'The attention kernel selection would widen the input implicitly.',
255
- attentionInputTransitionDeclaredBy
256
- );
257
- attentionInput = attentionActivationDtype === 'f16'
258
- ? await recordCastF32ToF16(recorder, input)
259
- : await recordCastF16ToF32(recorder, input);
260
- attentionInputTemp = true;
261
- normed = attentionInput;
262
- }
263
-
264
- if (!layerWeights) {
265
- const bytesPerElement = wantsF16Output ? 2 : 4;
266
- const outputBuf = acquireBuffer(numTokens * hiddenSize * bytesPerElement, undefined, 'attn_output');
267
- const output = createTensor(outputBuf, oProjOutputDtype, [numTokens, hiddenSize], 'attn_output');
268
- return { output, residualFused: false };
269
- }
270
-
271
- const qSize = numTokens * numHeads * headDim;
272
- const kvSize = numTokens * numKVHeads * headDim;
273
-
274
- // 1. Input norm
275
- // Opt-in fusion: when useFusedRmsnormWideTile is set, defer the standalone
276
- // rmsnorm into each q/k/v_proj matmul (fused kernel runs norm internally).
277
- let fusedNormWeightRec = null;
278
- let fusedNormEpsRec = null;
279
- let fusedNormOffsetRec = false;
280
- let fusedNormOwnedRec = false;
281
- // state.runtimeConfig is a stale snapshot whose inference.session is an empty
282
- // object; the live module-level runtime config carries the merged
283
- // profile/override session. Read session-level flags from getRuntimeConfig().
284
- const rmsNormFusionFlagRec = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
285
- const canSelectFusedRmsNormProjectionRec = canUseRmsNormWideTileProjectionFusion(
286
- layerWeights,
287
- sharedKVSourceLayerIdx != null
288
- );
289
- const canFuseInputNormProjRec = rmsNormFusionFlagRec
290
- && config.normalizationType === 'rmsnorm'
291
- && canSelectFusedRmsNormProjectionRec
292
- && !skipInputNorm
293
- && layerWeights.inputNorm
294
- && getNormWeightBuffer
295
- && isPrefill
296
- && numTokens > 1
297
- && attentionInput.dtype === 'f32';
298
-
299
- try {
300
- if (precomputedInputNorm) {
301
- if (
302
- isPrefill
303
- || numTokens !== 1
304
- || skipInputNorm
305
- || !layerWeights.inputNorm
306
- || !getNormWeightBuffer
307
- || precomputedInputNorm.dtype !== attentionInput.dtype
308
- || precomputedInputNorm.dtype !== 'f32'
309
- ) {
310
- throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
311
- }
312
- normed = precomputedInputNorm;
313
- await runProbes('post_input_norm', normed.buffer, {
314
- layerIdx, numTokens, hiddenSize,
315
- probes: state.debugProbes, recorder,
316
- operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
317
- });
318
- } else if (canFuseInputNormProjRec) {
319
- fusedNormWeightRec = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
320
- fusedNormEpsRec = rmsNormEps;
321
- fusedNormOffsetRec = config.rmsNormWeightOffset === true;
322
- fusedNormOwnedRec = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
323
- // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
324
- } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
325
- const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
326
- let normBiasTensor = null;
327
- let normBiasOwned = false;
328
- if (config.normalizationType === 'layernorm') {
329
- if (!layerWeights.inputNormBias) {
330
- throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
331
- }
332
- ({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
333
- layerWeights.inputNormBias,
334
- `L${layerIdx}.input_norm_bias`,
335
- hiddenSize,
336
- {},
337
- debugFlags
338
- ));
339
- normed = await recordLayerNorm(
340
- recorder,
341
- attentionInput,
342
- normWeightBuf,
343
- normBiasTensor.buffer,
344
- rmsNormEps,
345
- { batchSize: numTokens, hiddenSize, label: 'input_norm' }
346
- );
347
- } else {
348
- normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
349
- batchSize: numTokens,
350
- hiddenSize,
351
- rmsNormWeightOffset: config.rmsNormWeightOffset,
352
- label: 'input_norm',
353
- });
354
- }
355
- if (normBiasOwned && normBiasTensor) releaseOrTrack(recorder, normBiasTensor.buffer);
356
- if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
357
- await runProbes('post_input_norm', normed.buffer, {
358
- layerIdx, numTokens, hiddenSize,
359
- probes: state.debugProbes, recorder,
360
- operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
361
- });
362
- }
363
-
364
- const debugLayers = debugFlags.debugLayers;
365
- const shouldLogLayer = debugLayers === null ? layerIdx === 0 : shouldDebugLayer(layerIdx, debugLayers);
366
- if (shouldLogLayer) {
367
- const phase = selectRuleValue('kernels', 'attention', 'phase', { isDecode: !isPrefill });
368
- const logKey = `L${layerIdx}_${phase}_dtypes`;
369
- if (!ATTENTION_DTYPE_LOGGED.has(logKey)) {
370
- ATTENTION_DTYPE_LOGGED.add(logKey);
371
- trace.attn(layerIdx, `dtypes: activation=${attentionActivationDtype}, input=${input.dtype}, normed=${normed.dtype}`);
372
- }
373
- }
374
-
375
- // 2. Q/K/V projections
376
- const qProjVariant = getKernelPathMatmulVariant('q_proj', phase, layerIdx, kernelPath);
377
- const kernelPathIsF16 = qProjVariant != null && qProjVariant.includes('f16') && !qProjVariant.includes('f32');
378
- const matmulOutputDtype = resolveAttentionProjectionOutputDtype(attentionActivationDtype, {
379
- forceF32: shouldForceF32AttentionProjectionForRoPE({
380
- attentionInputDtype: attentionActivationDtype,
381
- headDim,
382
- rotaryDim: config.ropeRotaryDim,
383
- interleaved: config.ropeInterleaved,
384
- kernelPathIsF16,
385
- }),
386
- });
387
- let usedFusedQKV = false;
388
- const sharedKVEntry = sharedKVSourceLayerIdx == null
389
- ? null
390
- : (state.sharedAttentionState?.get(sharedKVSourceLayerIdx) ?? null);
391
- if (sharedKVSourceLayerIdx != null && !sharedKVEntry) {
392
- throw new Error(
393
- `Layer ${layerIdx} requires shared K/V from layer ${sharedKVSourceLayerIdx}, ` +
394
- 'but no shared K/V state was stored for that source layer.'
395
- );
396
- }
397
- if (sharedKVEntry && (
398
- sharedKVEntry.headDim !== headDim
399
- || sharedKVEntry.numKVHeads !== numKVHeads
400
- )) {
401
- throw new Error(
402
- `Layer ${layerIdx} shared K/V geometry mismatch. ` +
403
- `Expected numKVHeads=${numKVHeads}, headDim=${headDim}; ` +
404
- `got numKVHeads=${sharedKVEntry.numKVHeads}, headDim=${sharedKVEntry.headDim}.`
405
- );
406
- }
407
- const reusesSharedKV = sharedKVEntry != null;
408
- retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
409
- let qkNormApplied = false;
410
- let ropeApplied = false;
411
- const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
412
- const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
413
- || shouldTraceRecordedHealth(layerIdx, debugFlags);
414
- const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
415
- state,
416
- ['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
417
- )
418
- || shouldTraceRecordedHealth(layerIdx, debugFlags);
419
- const runtimeSession = getRuntimeConfig()?.inference?.session;
420
- const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
421
- const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
422
- const skipKVCacheWrites = state.skipKVCacheWrites === true;
423
- const directF16KVCacheWrite = skipKVCacheWrites
424
- ? null
425
- : resolveDirectF16KVCacheWrite({
426
- state,
427
- layerIdx,
428
- currentSeqLen,
429
- numTokens,
430
- numKVHeads,
431
- headDim,
432
- reusesSharedKV,
433
- storeSharedKV,
434
- diffusionGemmaDecoder,
435
- valueNorm: config.valueNorm === true,
436
- diagnosticsEnabled: qkNormRoPEDiagnostics,
437
- disableRoPE,
438
- });
439
- let kvCacheWriteFused = false;
440
- ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
441
- recorder,
442
- normed,
443
- layerWeights,
444
- numTokens,
445
- numHeads,
446
- numKVHeads,
447
- headDim,
448
- hiddenSize,
449
- layerIdx,
450
- kernelPath,
451
- matmulOutputDtype,
452
- getWeightBuffer,
453
- lora,
454
- matmulDebug: state.runtimeConfig?.shared?.debug?.matmul ?? null,
455
- attentionOutputGate: config.attentionOutputGate === true,
456
- sharedKTensor: sharedKVEntry?.kTensor ?? null,
457
- sharedVTensor: sharedKVEntry?.vTensor ?? null,
458
- executionPolicies: state.executionPolicies ?? null,
459
- releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
460
- onFusedQKV: layerIdx === 0 && isPrefill
461
- ? ({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize }) => {
462
- trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
463
- }
464
- : null,
465
- fusedNormWeight: fusedNormWeightRec,
466
- fusedNormEps: fusedNormEpsRec,
467
- fusedNormOffset: fusedNormOffsetRec,
468
- qkNormFusion: {
469
- enabled: qkNormFusionFlag
470
- && qkNormState.wantsQKNorm
471
- && config.queryKeyNormType === 'rmsnorm'
472
- && config.queryKeyNormAxis === 'head',
473
- getNormWeightBuffer,
474
- rmsNormEps,
475
- rmsNormWeightOffset: config.rmsNormWeightOffset,
476
- skipKNorm: qkNormState.skipKNorm,
477
- allowUnitQKNorm: qkNormState.allowUnitQKNorm,
478
- projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
479
- },
480
- qkNormRoPEFusion: {
481
- enabled: qkNormRoPEFusionFlag
482
- && qkNormState.wantsQKNorm
483
- && config.queryKeyNormType === 'rmsnorm'
484
- && config.queryKeyNormAxis === 'head'
485
- && !disableRoPE
486
- && !!state.ropeFreqsCos
487
- && !!state.ropeFreqsSin,
488
- getNormWeightBuffer,
489
- rmsNormEps,
490
- rmsNormWeightOffset: config.rmsNormWeightOffset,
491
- skipKNorm: qkNormState.skipKNorm,
492
- allowUnitQKNorm: qkNormState.allowUnitQKNorm,
493
- projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
494
- freqsCos: state.ropeFreqsCos,
495
- freqsSin: state.ropeFreqsSin,
496
- startPos: currentSeqLen,
497
- rotaryDim: config.ropeRotaryDim,
498
- pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
499
- interleaved: config.ropeInterleaved,
500
- reusesSharedKV,
501
- f16KVCacheWrite: directF16KVCacheWrite,
502
- },
503
- }));
504
- if (!kvCacheWriteFused && (!kTensor || !vTensor)) {
505
- throw new Error('Recorded attention projection returned missing K/V tensors without a fused KV cache write.');
506
- }
507
- // Deferred release of the norm weight buffer for fused path.
508
- if (fusedNormWeightRec && fusedNormOwnedRec) {
509
- releaseOrTrack(recorder, fusedNormWeightRec);
510
- }
511
- if (qTensor) {
512
- await runProbes('q_proj', qTensor.buffer, {
513
- layerIdx, numTokens, hiddenSize: numHeads * headDim,
514
- probes: state.debugProbes, recorder,
515
- operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
516
- });
517
- }
518
- if (kTensor) {
519
- await runProbes('k_proj', kTensor.buffer, {
520
- layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
521
- probes: state.debugProbes, recorder,
522
- operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
523
- });
524
- }
525
- if (vTensor) {
526
- await runProbes('v_proj', vTensor.buffer, {
527
- layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
528
- probes: state.debugProbes, recorder,
529
- operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
530
- });
531
- }
532
- if (qTensor && shouldTraceRecordedHealth(layerIdx, debugFlags)) {
533
- enqueueRecordedTensorHealth(
534
- recorder,
535
- `L${layerIdx}.q_proj_HEALTH`,
536
- qTensor,
537
- qTensor.dtype,
538
- numTokens * numHeads * headDim
539
- );
540
- enqueueRecordedTensorHealth(
541
- recorder,
542
- `L${layerIdx}.k_proj_HEALTH`,
543
- kTensor,
544
- kTensor?.dtype ?? null,
545
- numTokens * numKVHeads * headDim
546
- );
547
- enqueueRecordedTensorHealth(
548
- recorder,
549
- `L${layerIdx}.v_proj_HEALTH`,
550
- vTensor,
551
- vTensor?.dtype ?? null,
552
- numTokens * numKVHeads * headDim
553
- );
554
- }
555
-
556
- // Optional Q/K normalization. The manifest owns both its numerical family
557
- // and whether it applies per head or across the complete projection.
558
- if (qkNormState.wantsQKNorm && !qkNormApplied) {
559
- ({ qTensor, kTensor } = await applyAttentionQKNorm({
560
- recorder,
561
- qTensor,
562
- kTensor,
563
- layerWeights,
564
- getNormWeightBuffer,
565
- rmsNormEps,
566
- numTokens,
567
- numHeads,
568
- numKVHeads,
569
- headDim,
570
- rmsNormWeightOffset: config.rmsNormWeightOffset,
571
- normalizationType: config.queryKeyNormType,
572
- normalizationAxis: config.queryKeyNormAxis,
573
- releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
574
- skipKNorm: qkNormState.skipKNorm,
575
- retainKInput: valueAliasesKey,
576
- allowUnitQKNorm: qkNormState.allowUnitQKNorm,
577
- }));
578
- }
579
- if (qkNormState.wantsQKNorm) {
580
- await runProbes('q_norm', qTensor.buffer, {
581
- layerIdx, numTokens, hiddenSize: numHeads * headDim,
582
- probes: state.debugProbes, recorder,
583
- operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
584
- });
585
- if (kTensor) {
586
- await runProbes('k_norm', kTensor.buffer, {
587
- layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
588
- probes: state.debugProbes, recorder,
589
- operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
590
- });
591
- }
592
- if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
593
- enqueueRecordedTensorHealth(
594
- recorder,
595
- `L${layerIdx}.q_norm_HEALTH`,
596
- qTensor,
597
- qTensor.dtype,
598
- numTokens * numHeads * headDim
599
- );
600
- enqueueRecordedTensorHealth(
601
- recorder,
602
- `L${layerIdx}.k_norm_HEALTH`,
603
- kTensor,
604
- kTensor?.dtype ?? null,
605
- numTokens * numKVHeads * headDim
606
- );
607
- }
608
- }
609
-
610
- if (!kvCacheWriteFused && config.valueNorm === true && !reusesSharedKV) {
611
- const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
612
- vTensor = await applyAttentionValueNorm({
613
- recorder,
614
- vTensor,
615
- rmsNormEps,
616
- numTokens,
617
- numKVHeads,
618
- headDim,
619
- releaseTemporary: (buffer) => {
620
- if (!valueNormInputAliasesKey) {
621
- releaseOrTrack(recorder, buffer);
622
- }
623
- },
624
- });
625
- await runProbes('v_norm', vTensor.buffer, {
626
- layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
627
- probes: state.debugProbes, recorder,
628
- operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
629
- });
630
- }
631
-
632
- if (normed !== attentionInput) releaseOrTrack(recorder, normed.buffer);
633
- if (attentionInputTemp) recorder.trackTemporaryBuffer(attentionInput.buffer);
634
-
635
- // 3. RoPE (modifies tensor in-place)
636
- if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
637
- const ropeOptions = {
638
- headDim,
639
- rotaryDim: config.ropeRotaryDim,
640
- pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
641
- interleaved: config.ropeInterleaved,
642
- startPos: currentSeqLen,
643
- executionPolicies: state.executionPolicies ?? null,
644
- };
645
- if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
646
- await recordRoPEQK(recorder, qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
647
- ...ropeOptions,
648
- numQHeads: numHeads,
649
- numKVHeads,
650
- });
651
- } else {
652
- await recordRoPE(recorder, qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
653
- ...ropeOptions,
654
- numHeads,
655
- });
656
- if (!reusesSharedKV) {
657
- await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
658
- ...ropeOptions,
659
- numHeads: numKVHeads,
660
- });
661
- }
662
- }
663
- }
664
- if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
665
- enqueueRecordedTensorHealth(
666
- recorder,
667
- `L${layerIdx}.q_rope_HEALTH`,
668
- qTensor,
669
- qTensor.dtype,
670
- numTokens * numHeads * headDim
671
- );
672
- enqueueRecordedTensorHealth(
673
- recorder,
674
- `L${layerIdx}.k_rope_HEALTH`,
675
- kTensor,
676
- kTensor?.dtype ?? null,
677
- numTokens * numKVHeads * headDim
678
- );
679
- }
680
-
681
- if (storeSharedKV && state.sharedAttentionState) {
682
- state.sharedAttentionState.set(layerIdx, {
683
- kTensor,
684
- vTensor,
685
- headDim,
686
- numKVHeads,
687
- });
688
- }
689
-
690
- // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
691
- if (!skipKVCacheWrites && !diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
692
- if (kvCacheWriteFused) {
693
- state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
694
- } else if (state.kvCache.kvDtype === 'f16') {
695
- const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
696
- if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
697
- assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
698
- }
699
- if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
700
- assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
701
- }
702
- const canWriteDirectF32ToF16 = kTensor.dtype === 'f32'
703
- && vTensor.dtype === 'f32'
704
- && typeof state.kvCache.recordUpdateF32ToF16FromGPU === 'function';
705
- if (canWriteDirectF32ToF16) {
706
- await state.kvCache.recordUpdateF32ToF16FromGPU(
707
- recorder,
708
- layerIdx,
709
- kTensor.buffer,
710
- vTensor.buffer,
711
- currentSeqLen,
712
- numTokens,
713
- tokenIds
714
- );
715
- } else {
716
- const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
717
- const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
718
-
719
- await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
720
-
721
- if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
722
- if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
723
- }
724
- } else {
725
- await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kTensor.buffer, vTensor.buffer, currentSeqLen, numTokens, tokenIds);
726
- }
727
- }
728
-
729
- // Resolve KV cache state and build dispatch parameters (shared with run.js)
730
- let kvState;
731
- if (diffusionGemmaDecoder) {
732
- const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
733
- let decoderKTensor = kTensor;
734
- let decoderVTensor = vTensor;
735
- if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
736
- const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
737
- if (!hasExplicitF16KvContract) {
738
- assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
739
- }
740
- decoderKTensor = await recordCastF32ToF16(recorder, decoderKTensor);
741
- decoderKTemp = decoderKTensor;
742
- } else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
743
- assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
744
- decoderKTensor = await recordCastF16ToF32(recorder, decoderKTensor);
745
- decoderKTemp = decoderKTensor;
746
- }
747
- if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
748
- const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
749
- if (!hasExplicitF16KvContract) {
750
- assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
751
- }
752
- decoderVTensor = await recordCastF32ToF16(recorder, decoderVTensor);
753
- decoderVTemp = decoderVTensor;
754
- } else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
755
- assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
756
- decoderVTensor = await recordCastF16ToF32(recorder, decoderVTensor);
757
- decoderVTemp = decoderVTensor;
758
- }
759
- kvState = await createDiffusionGemmaDecoderKVState({
760
- state,
761
- layerIdx,
762
- kTensor: decoderKTensor,
763
- vTensor: decoderVTensor,
764
- currentSeqLen,
765
- numTokens,
766
- numKVHeads,
767
- headDim,
768
- layerType,
769
- slidingWindow,
770
- kvDtype: decoderKVDtype,
771
- recorder,
772
- });
773
- if (decoderKTemp) {
774
- recorder.trackTemporaryBuffer(decoderKTemp.buffer);
775
- decoderKTemp = null;
776
- }
777
- if (decoderVTemp) {
778
- recorder.trackTemporaryBuffer(decoderVTemp.buffer);
779
- decoderVTemp = null;
780
- }
781
- decoderKVState = kvState;
782
- } else {
783
- kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens, {
784
- forceCurrentTensors: skipKVCacheWrites,
785
- });
786
- }
787
- const dispatchConfig = {
788
- layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
789
- slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
790
- layerType, layerTypes: config.layerTypes,
791
- queryPreAttnScalar,
792
- causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
793
- activationDtype: attentionActivationDtype,
794
- kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
795
- };
796
- const dispatchParams = buildAttentionDispatchParams(dispatchConfig, state, kTensor, vTensor, kvState);
797
- const {
798
- effectiveSlidingWindow, attentionKernelVariant, attnScale,
799
- cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
800
- prefillFallbackNeedsCast, causalForAttention,
801
- } = dispatchParams;
802
- const attentionKernelPath = skipKVCacheWrites ? null : kernelPath;
803
-
804
- // 5. Attention
805
-
806
- recordAttentionInputs(state, buildAttentionInputsData(
807
- dispatchConfig, input, normed, kvState, dispatchParams,
808
- { useF16Activations, matmulOutputDtype },
809
- usedFusedQKV, qTensor, kTensor, vTensor,
810
- ));
811
- const mergedSessionRec = getRuntimeConfig()?.inference?.session;
812
- let attentionOutputGateFused = false;
813
-
814
- const attentionKernelRunners = {
815
- bdpa: async () => {
816
- const basisKDtype = 'f16';
817
- const basisVDtype = 'f16';
818
- const numBasisVectors = Math.max(1, kvState.bdpaBasisCount);
819
- const basisKTensor = createTensor(kvState.bdpaBasisK, basisKDtype, [numBasisVectors, numKVHeads * headDim], 'bdpa_basis_k');
820
- const basisVTensor = createTensor(kvState.bdpaBasisV, basisVDtype, [numBasisVectors, numKVHeads * headDim], 'bdpa_basis_v');
821
-
822
- let qForBDPA = qTensor;
823
- if (qForBDPA.dtype !== 'f16') {
824
- assertAttentionDtypeTransitionAllowed(state, qForBDPA.dtype, 'f16', 'BDPA attention would narrow Q implicitly.');
825
- qForBDPA = await recordCastF32ToF16(recorder, qTensor);
826
- recorder.trackTemporaryBuffer(qForBDPA.buffer);
827
- }
828
-
829
- return recordAttentionBDPA(recorder, qForBDPA, basisKTensor, basisVTensor, kvState.bdpaPagedK, kvState.bdpaPagedV, kvState.bdpaIndex, numHeads, headDim, {
830
- seqLen: numTokens,
831
- kvLen: kvState.kvLenForAttention,
832
- numKVHeads,
833
- causal: causalForAttention,
834
- startPos: kvState.startPosForMask,
835
- layerIdx,
836
- slidingWindow: effectiveSlidingWindow,
837
- attnSoftcap,
838
- scale: attnScale,
839
- ropeCos: state.ropeFreqsCos,
840
- ropeSin: state.ropeFreqsSin,
841
- });
842
- },
843
- tieredQuant: async () => {
844
- let qForAttention = qTensor;
845
- if (kvState.coldQuantMode !== 'none' && qTensor.dtype !== 'f32') {
846
- assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Tiered quant attention would widen Q implicitly.');
847
- qForAttention = await recordCastF16ToF32(recorder, qTensor);
848
- recorder.trackTemporaryBuffer(qForAttention.buffer);
849
- }
850
- if (kvState.coldQuantMode === 'none') {
851
- throw new Error('Tiered quant attention requires cold quant mode.');
852
- }
853
- if (!kvState.coldScalesK || !kvState.coldScalesV) {
854
- throw new Error('Tiered quant attention requires cold scale buffers.');
855
- }
856
-
857
- const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
858
- const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
859
- return recordAttentionTieredQuant(
860
- recorder,
861
- qForAttention,
862
- cachedHotKTensor,
863
- cachedHotVTensor,
864
- kvState.cachedKCold,
865
- kvState.cachedVCold,
866
- kvState.coldScalesK,
867
- kvState.coldScalesV,
868
- numHeads,
869
- headDim,
870
- buildTieredQuantAttentionOptions(kvState, {
871
- seqLen: numTokens,
872
- numKVHeads,
873
- causal: causalForAttention,
874
- startPos: kvState.startPosForMask,
875
- slidingWindow: effectiveSlidingWindow ?? 0,
876
- attnSoftcap,
877
- scale: attnScale,
878
- })
879
- );
880
- },
881
- contiguousQuant: async () => {
882
- let qForAttention = qTensor;
883
- if (qTensor.dtype !== 'f32') {
884
- assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Contiguous quant attention would widen Q implicitly.');
885
- qForAttention = await recordCastF16ToF32(recorder, qTensor);
886
- recorder.trackTemporaryBuffer(qForAttention.buffer);
887
- }
888
-
889
- if (!kvState.coldScalesK || !kvState.coldScalesV) {
890
- throw new Error('Contiguous quant attention requires scale buffers.');
891
- }
892
- if (!kvState.rotationMatrixBuffer || !kvState.codebookCentroidsBuffer) {
893
- throw new Error('Contiguous quant attention requires TurboQuant shared buffers.');
894
- }
895
-
896
- return recordAttentionContiguousQuant(
897
- recorder,
898
- qForAttention,
899
- kvState.cachedKCold,
900
- kvState.cachedVCold,
901
- kvState.coldScalesK,
902
- kvState.coldScalesV,
903
- numHeads,
904
- headDim,
905
- buildContiguousQuantAttentionOptions(kvState, {
906
- seqLen: numTokens,
907
- kvLen: kvState.kvLenForAttention,
908
- numKVHeads,
909
- causal: causalForAttention,
910
- startPos: kvState.startPosForMask,
911
- slidingWindow: effectiveSlidingWindow ?? 0,
912
- attnSoftcap,
913
- scale: attnScale,
914
- })
915
- );
916
- },
917
- tiered: async () => {
918
- const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
919
- const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
920
- const cachedColdKTensor = createTensor(kvState.cachedKCold, cachedKDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_K_cold');
921
- const cachedColdVTensor = createTensor(kvState.cachedVCold, cachedVDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_V_cold');
922
- return recordAttentionTiered(recorder, qTensor, cachedHotKTensor, cachedHotVTensor, cachedColdKTensor, cachedColdVTensor, numHeads, headDim, {
923
- seqLen: numTokens,
924
- coldLen: kvState.coldLen,
925
- hotLen: kvState.hotLen,
926
- numKVHeads,
927
- causal: causalForAttention,
928
- startPos: kvState.startPosForMask,
929
- slidingWindow: effectiveSlidingWindow ?? 0,
930
- attnSoftcap,
931
- scale: attnScale,
932
- hotWindow: kvState.hotWindow,
933
- hotStart: kvState.hotStart,
934
- coldPageTable: kvState.coldPageTable,
935
- coldPageSize: kvState.coldPageSize,
936
- coldLayout: kvState.coldPageTable ? 2 : 0,
937
- hotLayout: kvState.hotWindow > 0 ? 1 : 0,
938
- });
939
- },
940
- contiguous: async () => {
941
- // Prefill fallback: quantized/tiered layouts use raw K/V for prefill, cast to f16 to match kernel path
942
- let kForAttn = cachedKTensor;
943
- let vForAttn = cachedVTensor;
944
- if (prefillFallbackNeedsCast) {
945
- const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
946
- if (cachedKDtype === 'f16' && kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
947
- assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'Prefill fallback attention would narrow K implicitly.');
948
- }
949
- if (cachedVDtype === 'f16' && vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
950
- assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'Prefill fallback attention would narrow V implicitly.');
951
- }
952
- const kCasted = cachedKDtype === 'f16' && kTensor.dtype !== 'f16'
953
- ? await recordCastF32ToF16(recorder, kTensor) : kTensor;
954
- const vCasted = cachedVDtype === 'f16' && vTensor.dtype !== 'f16'
955
- ? await recordCastF32ToF16(recorder, vTensor) : vTensor;
956
- kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
957
- vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
958
- if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
959
- if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
960
- }
961
- // Session precedence is runtime-over-manifest per config-style-guide
962
- // §Category Rules. getRuntimeConfig() returns the merged session (manifest
963
- // is the base layer, runtime fields win field-by-field via merge.js).
964
- // Kernel enforces head_dim=256, f16 KV, contiguous layout; only applies
965
- // when numTokens > 1 (prefill). Same flag semantics as the non-recorder
966
- // path in ./run.js.
967
- const useFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
968
- const useOrtFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
969
- const useOutputGateFusion = canUseAttentionOutputGateFusion({
970
- session: mergedSessionRec,
971
- qGateTensor,
972
- numTokens,
973
- numHeads,
974
- headDim,
975
- cachedKDtype,
976
- cachedVDtype,
977
- kernelPath: attentionKernelPath,
978
- diffusionGemmaDecoder,
979
- });
980
- const result = await recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
981
- seqLen: numTokens,
982
- kvLen: kvState.kvLenForAttention,
983
- numKVHeads,
984
- causal: causalForAttention,
985
- bidirectionalSpanStart: multimodalBidirectionalSpan?.start ?? 0,
986
- bidirectionalSpanLength: multimodalBidirectionalSpan?.length ?? 0,
987
- startPos: kvState.startPosForMask,
988
- layerIdx,
989
- slidingWindow: effectiveSlidingWindow,
990
- attnSoftcap,
991
- scale: attnScale,
992
- kvStart: kvState.kvStart,
993
- kvLayout: kvState.kvLayout,
994
- kvPageTable: kvState.kvPageTable,
995
- kvPageSize: kvState.kvPageSize,
996
- kernelPath: attentionKernelPath,
997
- useFlashPrefill: skipKVCacheWrites ? false : useFlashPrefillRec,
998
- useOrtFlashPrefill: skipKVCacheWrites ? false : useOrtFlashPrefillRec,
999
- outputGate: useOutputGateFusion ? qGateTensor : null,
1000
- });
1001
- attentionOutputGateFused = result?.outputGateFused === true;
1002
- return result;
1003
- },
1004
- };
1005
- const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
1006
- if (!runAttentionKernel) {
1007
- throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
1008
- }
1009
-
1010
- try {
1011
- attnOutput = await runAttentionKernel();
1012
- } finally {
1013
- if (decoderKVState?.ownedBuffers) {
1014
- for (const buffer of decoderKVState.ownedBuffers) {
1015
- recorder.trackTemporaryBuffer(buffer);
1016
- }
1017
- decoderKVState.ownedBuffers = null;
1018
- }
1019
- }
1020
- await runProbes('attn_core_out', attnOutput.buffer, {
1021
- layerIdx,
1022
- numTokens,
1023
- hiddenSize: numHeads * headDim,
1024
- probes: state.debugProbes,
1025
- recorder,
1026
- operatorDiagnostics: state.operatorDiagnostics,
1027
- dtype: attnOutput.dtype,
1028
- });
1029
- if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
1030
- enqueueRecordedTensorHealth(
1031
- recorder,
1032
- `L${layerIdx}.attn_core_out_HEALTH`,
1033
- attnOutput,
1034
- attnOutput.dtype,
1035
- numTokens * numHeads * headDim
1036
- );
1037
- }
1038
-
1039
- attnForProjection = attnOutput;
1040
- if (qGateTensor && !attentionOutputGateFused) {
1041
- // Mirror run.js gate dispatch. Qwen3_5/Qwen 3.6 full attention applies
1042
- // sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
1043
- const gateActivation = 'sigmoid';
1044
- attnForProjection = await recordSiLU(recorder, attnOutput, {
1045
- size: numTokens * numHeads * headDim,
1046
- gate: qGateTensor,
1047
- useVec4: (numTokens * numHeads * headDim) % 4 === 0,
1048
- gateActivation,
1049
- inputActivation: 'identity',
1050
- swigluLimit: null,
1051
- });
1052
- recorder.trackTemporaryBuffer(attnOutput.buffer);
1053
- }
1054
-
1055
- // 6. Output projection (with optional fused residual for decode)
1056
-
1057
- output = null;
1058
- let residualFused = false;
1059
- let oProjInput = attnForProjection;
1060
- oProjInputTemp = null;
1061
- if (layerWeights.oProj && getWeightBuffer) {
1062
- ({ oProjInput, oProjInputTemp } = await prepareAttentionProjectionInput(
1063
- attnForProjection,
1064
- oProjInputDtype,
1065
- (tensor) => {
1066
- assertAttentionDtypeTransitionAllowed(
1067
- state,
1068
- tensor.dtype,
1069
- oProjInputDtype,
1070
- 'Attention output projection would change activations implicitly.',
1071
- 'step_precision'
1072
- );
1073
- return oProjInputDtype === 'f16'
1074
- ? recordCastF32ToF16(recorder, tensor)
1075
- : recordCastF16ToF32(recorder, tensor);
1076
- }
1077
- ));
1078
- const oProjBuf = getWeightBuffer(layerWeights.oProj, 'o_proj');
1079
- const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
1080
-
1081
- // Use fused o_proj + residual for decode when possible
1082
- // Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
1083
- const oProjDtype = getWeightDtype(oProjBuf);
1084
- const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
1085
- allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
1086
- hasResidual: Boolean(residualTensor),
1087
- residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
1088
- attnIsF32: oProjInput.dtype === 'f32',
1089
- attnIsF16: oProjInput.dtype === 'f16',
1090
- hasLoRA: Boolean(loraO),
1091
- oProjIsF16: oProjDtype === 'f16',
1092
- });
1093
- const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
1094
- phase,
1095
- session: mergedSessionRec,
1096
- hasResidual: Boolean(residualTensor),
1097
- residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
1098
- inputDtype: oProjInput.dtype,
1099
- outputDtype: oProjOutputDtype,
1100
- oProjDtype,
1101
- hasLoRA: Boolean(loraO),
1102
- });
1103
- if (canUseFused && residualTensor) {
1104
- // FUSED PATH: o_proj matmul + residual add in one dispatch
1105
- output = await recordMatmulResidualFused(recorder, oProjInput, oProjBuf, residualTensor, {
1106
- N: hiddenSize,
1107
- K: numHeads * headDim,
1108
- });
1109
- residualFused = true;
1110
- } else {
1111
- // STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
1112
- output = await recordMatmul(recorder, oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
1113
- transposeB: 'auto',
1114
- role: 'o_proj',
1115
- layerIdx,
1116
- kernelPath,
1117
- outputDtype: oProjOutputDtype,
1118
- executionPolicies: state.executionPolicies ?? null,
1119
- residualTensor: canUseWideTileResidual ? residualTensor : null,
1120
- });
1121
- residualFused = canUseWideTileResidual;
1122
- }
1123
- // Release temporary buffer if we created it (original was not already on GPU)
1124
- if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
1125
- releaseOrTrack(recorder, isWeightBuffer(oProjBuf) ? oProjBuf.buffer : oProjBuf);
1126
- }
1127
- } else {
1128
- output = oProjInput;
1129
- }
1130
-
1131
- // Apply LoRA to output projection if present (only if not using fused path)
1132
- if (!residualFused) {
1133
- const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
1134
- if (loraO && getWeightBuffer) {
1135
- const combined = await applyLoRA(
1136
- oProjInput,
1137
- output,
1138
- loraO,
1139
- { M: numTokens, N: hiddenSize, K: numHeads * headDim },
1140
- getWeightBuffer,
1141
- recorder,
1142
- { kernelPath }
1143
- );
1144
- if (combined.buffer !== output.buffer) {
1145
- recorder.trackTemporaryBuffer(output.buffer);
1146
- output = combined;
1147
- }
1148
- }
1149
- }
1150
-
1151
- if (layerWeights.oProjBias) {
1152
- const { tensor: oProjBiasTensor, owned } = getVectorTensor(
1153
- layerWeights.oProjBias,
1154
- `L${layerIdx}.o_proj_bias`,
1155
- hiddenSize,
1156
- {},
1157
- debugFlags
1158
- );
1159
- output = await recordBiasAdd(
1160
- recorder,
1161
- output,
1162
- oProjBiasTensor,
1163
- numTokens,
1164
- hiddenSize,
1165
- {
1166
- label: `L${layerIdx}.o_proj_bias`,
1167
- layerIdx,
1168
- executionPolicies: state.executionPolicies ?? null,
1169
- }
1170
- );
1171
- if (owned) releaseOrTrack(recorder, oProjBiasTensor.buffer);
1172
- }
1173
-
1174
- finalOutput = output;
1175
- if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
1176
- enqueueRecordedTensorHealth(
1177
- recorder,
1178
- `L${layerIdx}.o_proj_HEALTH`,
1179
- output,
1180
- output.dtype,
1181
- numTokens * hiddenSize
1182
- );
1183
- }
1184
-
1185
- const buffersToTrack = [];
1186
- if (output.buffer !== attnForProjection.buffer) {
1187
- buffersToTrack.push(attnForProjection.buffer);
1188
- }
1189
- if (oProjInputTemp && oProjInputTemp.buffer !== attnForProjection.buffer) {
1190
- buffersToTrack.push(oProjInputTemp.buffer);
1191
- }
1192
- if (output.dtype !== oProjOutputDtype) {
1193
- assertAttentionDtypeTransitionAllowed(state, output.dtype, oProjOutputDtype, 'Attention output would change implicitly before leaving the layer.');
1194
- const coercedOutput = oProjOutputDtype === 'f16'
1195
- ? await recordCastF32ToF16(recorder, output)
1196
- : await recordCastF16ToF32(recorder, output);
1197
- buffersToTrack.push(output.buffer);
1198
- finalOutput = coercedOutput;
1199
- }
1200
-
1201
- // Track intermediate buffers for cleanup after submit (not release!)
1202
- // These buffers are used by recorded operations that haven't executed yet.
1203
- // Releasing them back to the pool would allow reuse before the encoder is submitted,
1204
- // causing data corruption (especially for small decode buffers).
1205
- if (qTensor) {
1206
- recorder.trackTemporaryBuffer(qTensor.buffer);
1207
- }
1208
- if (qGateTensor) {
1209
- recorder.trackTemporaryBuffer(qGateTensor.buffer);
1210
- }
1211
- if (!retainSharedKvBuffers) {
1212
- if (kTensor) {
1213
- recorder.trackTemporaryBuffer(kTensor.buffer);
1214
- }
1215
- if (vTensor?.buffer && vTensor.buffer !== kTensor?.buffer) {
1216
- recorder.trackTemporaryBuffer(vTensor.buffer);
1217
- }
1218
- }
1219
- for (const buffer of buffersToTrack) {
1220
- recorder.trackTemporaryBuffer(buffer);
1221
- }
1222
-
1223
- return { output: finalOutput, residualFused };
1224
- } catch (error) {
1225
- const tracked = new Set();
1226
- const trackOnce = (buffer) => {
1227
- if (!buffer || tracked.has(buffer)) return;
1228
- tracked.add(buffer);
1229
- recorder.trackTemporaryBuffer(buffer);
1230
- };
1231
- if (finalOutput?.buffer && finalOutput.buffer !== output?.buffer) {
1232
- trackOnce(finalOutput.buffer);
1233
- }
1234
- if (output?.buffer && output.buffer !== attnForProjection?.buffer) {
1235
- trackOnce(output.buffer);
1236
- }
1237
- if (oProjInputTemp?.buffer) {
1238
- trackOnce(oProjInputTemp.buffer);
1239
- }
1240
- if (attnForProjection?.buffer && attnForProjection.buffer !== attnOutput?.buffer) {
1241
- trackOnce(attnForProjection.buffer);
1242
- }
1243
- if (attnOutput?.buffer) {
1244
- trackOnce(attnOutput.buffer);
1245
- }
1246
- if (qGateTensor?.buffer) {
1247
- trackOnce(qGateTensor.buffer);
1248
- }
1249
- if (qTensor?.buffer) {
1250
- trackOnce(qTensor.buffer);
1251
- }
1252
- if (kTensor?.buffer && !retainSharedKvBuffers) {
1253
- trackOnce(kTensor.buffer);
1254
- }
1255
- if (vTensor?.buffer && !retainSharedKvBuffers) {
1256
- trackOnce(vTensor.buffer);
1257
- }
1258
- if (normed?.buffer && normed.buffer !== attentionInput?.buffer) {
1259
- trackOnce(normed.buffer);
1260
- }
1261
- if (attentionInputTemp && attentionInput?.buffer) {
1262
- trackOnce(attentionInput.buffer);
1263
- }
1264
- if (decoderKTemp?.buffer) {
1265
- trackOnce(decoderKTemp.buffer);
1266
- }
1267
- if (decoderVTemp?.buffer) {
1268
- trackOnce(decoderVTemp.buffer);
1269
- }
1270
- if (decoderKVState?.ownedBuffers) {
1271
- for (const buffer of decoderKVState.ownedBuffers) {
1272
- trackOnce(buffer);
1273
- }
1274
- decoderKVState.ownedBuffers = null;
1275
- }
1276
- throw error;
1277
- }
1278
- }
1
+ /**
2
+ * Recorded attention facade.
3
+ *
4
+ * Semantic decisions are resolved in plan.js. The recorded adapter binds the
5
+ * canonical interpreter to the caller's recorder.
6
+ */
7
+
8
+ export { recordLayerAttentionGPU } from './executor-recorded.js';