doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,6 +1,6 @@
1
1
 
2
2
 
3
- import { isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
3
+ import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
4
  import { acquireBuffer } from '../../../../memory/buffer-pool.js';
5
5
  import {
6
6
  recordMatmul,
@@ -9,6 +9,7 @@ import {
9
9
  recordAttention,
10
10
  recordAttentionTiered,
11
11
  recordAttentionTieredQuant,
12
+ recordAttentionContiguousQuant,
12
13
  recordAttentionBDPA,
13
14
  recordSiLU,
14
15
  recordCastF16ToF32,
@@ -27,19 +28,43 @@ import {
27
28
  resolveAttentionProjectionOutputDtype,
28
29
  projectAttentionQKV,
29
30
  applyAttentionQKNorm,
31
+ applyAttentionValueNorm,
30
32
  } from './projections.js';
31
33
  import { prepareAttentionProjectionInput } from './output-projection.js';
32
34
 
33
35
  import { releaseOrTrack, shouldDebugLayer } from './types.js';
34
- import { getKernelPathMatmulVariant } from '../../../../config/kernel-path-loader.js';
36
+ import {
37
+ getKernelPathMatmulPrecision,
38
+ getKernelPathMatmulVariant,
39
+ } from '../../../../config/kernel-path-loader.js';
35
40
  import {
36
41
  resolveKVCacheState,
37
42
  buildAttentionDispatchParams,
38
43
  buildAttentionInputsData,
39
44
  } from './dispatch-params.js';
45
+ import {
46
+ buildTieredQuantAttentionOptions,
47
+ buildContiguousQuantAttentionOptions,
48
+ } from './quant-options.js';
49
+ import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
50
+ import {
51
+ resolveAttentionPrecisionContract,
52
+ isAttentionKvDtypeExplicit,
53
+ } from './precision-contract.js';
40
54
 
41
55
  const ATTENTION_DTYPE_LOGGED = new Set();
42
56
 
57
+ function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
58
+ assertImplicitDtypeTransitionAllowed({
59
+ executionPolicies: state?.executionPolicies ?? null,
60
+ fromDtype,
61
+ toDtype,
62
+ op: 'attention',
63
+ detail,
64
+ transitionDeclaredBy,
65
+ });
66
+ }
67
+
43
68
 
44
69
  export async function recordLayerAttentionGPU(
45
70
  recorder,
@@ -73,16 +98,30 @@ export async function recordLayerAttentionGPU(
73
98
  tokenIds = null,
74
99
  kernelPath = null,
75
100
  disableRoPE = false,
101
+ multimodalBidirectionalSpan = null,
102
+ sharedKVSourceLayerIdx = null,
103
+ storeSharedKV = false,
76
104
  } = config;
77
105
 
78
- const desiredOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
79
- dtype: config.activationDtype,
106
+ const phase = isPrefill ? 'prefill' : 'decode';
107
+ const attentionPrecisionContract = resolveAttentionPrecisionContract(config, state);
108
+ const attentionActivationDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
109
+ dtype: attentionPrecisionContract.resolvedActivationDtype ?? config.activationDtype,
110
+ });
111
+ const oProjPrecision = getKernelPathMatmulPrecision('o_proj', phase, layerIdx, kernelPath);
112
+ const oProjInputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
113
+ dtype: oProjPrecision?.inputDtype ?? attentionActivationDtype,
80
114
  });
81
- const wantsF16Output = desiredOutputDtype === 'f16';
82
- const useF16Activations = wantsF16Output;
83
- const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: wantsF16Output });
84
- const kvCacheDtype = state.kvCache?.kvDtype ?? kvCacheFallback;
85
- const allowF16Attention = wantsF16Output && kvCacheDtype === 'f16';
115
+ const oProjOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
116
+ dtype: oProjPrecision?.outputDtype
117
+ ?? attentionPrecisionContract.resolvedOutputDtype
118
+ ?? config.activationDtype,
119
+ });
120
+ const wantsF16Output = oProjOutputDtype === 'f16';
121
+ const useF16Activations = attentionActivationDtype === 'f16';
122
+ const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
123
+ const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
124
+ const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
86
125
  let attentionInput = input;
87
126
  let attentionInputTemp = false;
88
127
  let normed = attentionInput;
@@ -95,8 +134,17 @@ export async function recordLayerAttentionGPU(
95
134
  let output = null;
96
135
  let finalOutput = null;
97
136
  let oProjInputTemp = null;
98
- if (wantsF16Output && !allowF16Attention) {
99
- attentionInput = await recordCastF16ToF32(recorder, input);
137
+ let retainSharedKvBuffers = false;
138
+ if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
139
+ assertAttentionDtypeTransitionAllowed(
140
+ state,
141
+ input.dtype,
142
+ attentionActivationDtype,
143
+ 'The attention kernel selection would widen the input implicitly.'
144
+ );
145
+ attentionInput = attentionActivationDtype === 'f16'
146
+ ? await recordCastF32ToF16(recorder, input)
147
+ : await recordCastF16ToF32(recorder, input);
100
148
  attentionInputTemp = true;
101
149
  normed = attentionInput;
102
150
  }
@@ -104,8 +152,7 @@ export async function recordLayerAttentionGPU(
104
152
  if (!layerWeights) {
105
153
  const bytesPerElement = wantsF16Output ? 2 : 4;
106
154
  const outputBuf = acquireBuffer(numTokens * hiddenSize * bytesPerElement, undefined, 'attn_output');
107
- const outputDtype = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: wantsF16Output });
108
- const output = createTensor(outputBuf, outputDtype, [numTokens, hiddenSize], 'attn_output');
155
+ const output = createTensor(outputBuf, oProjOutputDtype, [numTokens, hiddenSize], 'attn_output');
109
156
  return { output, residualFused: false };
110
157
  }
111
158
 
@@ -122,7 +169,7 @@ export async function recordLayerAttentionGPU(
122
169
  hiddenSize,
123
170
  rmsNormWeightOffset: config.rmsNormWeightOffset,
124
171
  });
125
- if (!(layerWeights.inputNorm instanceof GPUBuffer)) releaseOrTrack(recorder, normWeightBuf);
172
+ if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
126
173
  }
127
174
 
128
175
  const debugLayers = debugFlags.debugLayers;
@@ -132,16 +179,16 @@ export async function recordLayerAttentionGPU(
132
179
  const logKey = `L${layerIdx}_${phase}_dtypes`;
133
180
  if (!ATTENTION_DTYPE_LOGGED.has(logKey)) {
134
181
  ATTENTION_DTYPE_LOGGED.add(logKey);
135
- trace.attn(layerIdx, `dtypes: activation=${config.activationDtype ?? 'unknown'}, input=${input.dtype}, normed=${normed.dtype}`);
182
+ trace.attn(layerIdx, `dtypes: activation=${attentionActivationDtype}, input=${input.dtype}, normed=${normed.dtype}`);
136
183
  }
137
184
  }
138
185
 
139
186
  // 2. Q/K/V projections
140
- const qProjVariant = getKernelPathMatmulVariant('q_proj', isPrefill ? 'prefill' : 'decode', layerIdx, kernelPath);
187
+ const qProjVariant = getKernelPathMatmulVariant('q_proj', phase, layerIdx, kernelPath);
141
188
  const kernelPathIsF16 = qProjVariant != null && qProjVariant.includes('f16') && !qProjVariant.includes('f32');
142
- const matmulOutputDtype = resolveAttentionProjectionOutputDtype(desiredOutputDtype, {
189
+ const matmulOutputDtype = resolveAttentionProjectionOutputDtype(attentionActivationDtype, {
143
190
  forceF32: shouldForceF32AttentionProjectionForRoPE({
144
- attentionInputDtype: desiredOutputDtype,
191
+ attentionInputDtype: attentionActivationDtype,
145
192
  headDim,
146
193
  rotaryDim: config.ropeRotaryDim,
147
194
  interleaved: config.ropeInterleaved,
@@ -149,6 +196,27 @@ export async function recordLayerAttentionGPU(
149
196
  }),
150
197
  });
151
198
  let usedFusedQKV = false;
199
+ const sharedKVEntry = sharedKVSourceLayerIdx == null
200
+ ? null
201
+ : (state.sharedAttentionState?.get(sharedKVSourceLayerIdx) ?? null);
202
+ if (sharedKVSourceLayerIdx != null && !sharedKVEntry) {
203
+ throw new Error(
204
+ `Layer ${layerIdx} requires shared K/V from layer ${sharedKVSourceLayerIdx}, ` +
205
+ 'but no shared K/V state was stored for that source layer.'
206
+ );
207
+ }
208
+ if (sharedKVEntry && (
209
+ sharedKVEntry.headDim !== headDim
210
+ || sharedKVEntry.numKVHeads !== numKVHeads
211
+ )) {
212
+ throw new Error(
213
+ `Layer ${layerIdx} shared K/V geometry mismatch. ` +
214
+ `Expected numKVHeads=${numKVHeads}, headDim=${headDim}; ` +
215
+ `got numKVHeads=${sharedKVEntry.numKVHeads}, headDim=${sharedKVEntry.headDim}.`
216
+ );
217
+ }
218
+ const reusesSharedKV = sharedKVEntry != null;
219
+ retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
152
220
  ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
153
221
  recorder,
154
222
  normed,
@@ -165,6 +233,9 @@ export async function recordLayerAttentionGPU(
165
233
  lora,
166
234
  matmulDebug: state.runtimeConfig?.shared?.debug?.matmul ?? null,
167
235
  attentionOutputGate: config.attentionOutputGate === true,
236
+ sharedKTensor: sharedKVEntry?.kTensor ?? null,
237
+ sharedVTensor: sharedKVEntry?.vTensor ?? null,
238
+ executionPolicies: state.executionPolicies ?? null,
168
239
  releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
169
240
  onFusedQKV: layerIdx === 0 && isPrefill
170
241
  ? ({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize }) => {
@@ -192,8 +263,21 @@ export async function recordLayerAttentionGPU(
192
263
  headDim,
193
264
  rmsNormWeightOffset: config.rmsNormWeightOffset,
194
265
  releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
266
+ skipKNorm: reusesSharedKV,
195
267
  }));
196
268
 
269
+ if (config.valueNorm === true && !reusesSharedKV) {
270
+ vTensor = await applyAttentionValueNorm({
271
+ recorder,
272
+ vTensor,
273
+ rmsNormEps,
274
+ numTokens,
275
+ numKVHeads,
276
+ headDim,
277
+ releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
278
+ });
279
+ }
280
+
197
281
  if (normed !== attentionInput) releaseOrTrack(recorder, normed.buffer);
198
282
  if (attentionInputTemp) recorder.trackTemporaryBuffer(attentionInput.buffer);
199
283
 
@@ -205,22 +289,41 @@ export async function recordLayerAttentionGPU(
205
289
  rotaryDim: config.ropeRotaryDim,
206
290
  interleaved: config.ropeInterleaved,
207
291
  startPos: currentSeqLen,
292
+ executionPolicies: state.executionPolicies ?? null,
208
293
  });
209
- await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
210
- numHeads: numKVHeads,
294
+ if (!reusesSharedKV) {
295
+ await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
296
+ numHeads: numKVHeads,
297
+ headDim,
298
+ rotaryDim: config.ropeRotaryDim,
299
+ interleaved: config.ropeInterleaved,
300
+ startPos: currentSeqLen,
301
+ executionPolicies: state.executionPolicies ?? null,
302
+ });
303
+ }
304
+ }
305
+
306
+ if (storeSharedKV && state.sharedAttentionState) {
307
+ state.sharedAttentionState.set(layerIdx, {
308
+ kTensor,
309
+ vTensor,
211
310
  headDim,
212
- rotaryDim: config.ropeRotaryDim,
213
- interleaved: config.ropeInterleaved,
214
- startPos: currentSeqLen,
311
+ numKVHeads,
215
312
  });
216
313
  }
217
314
 
218
315
  // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
219
-
220
316
  if (state.kvCache?.hasGPUCache?.()) {
221
317
  // Use recordUpdateFromGPU to record copy operations to the recorder's encoder
222
318
  // This ensures K/V buffers are populated before copying (all ops submitted together)
223
319
  if (state.kvCache.kvDtype === 'f16') {
320
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
321
+ if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
322
+ assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
323
+ }
324
+ if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
325
+ assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
326
+ }
224
327
  const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
225
328
  const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
226
329
 
@@ -239,14 +342,14 @@ export async function recordLayerAttentionGPU(
239
342
  layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
240
343
  slidingWindow, layerType, layerTypes: config.layerTypes,
241
344
  queryPreAttnScalar, causalAttention: config.causalAttention,
242
- activationDtype: config.activationDtype,
243
- kvCacheDtype: state.kvCache?.kvDtype ?? null,
345
+ activationDtype: attentionActivationDtype,
346
+ kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
244
347
  };
245
348
  const dispatchParams = buildAttentionDispatchParams(dispatchConfig, state, kTensor, vTensor, kvState);
246
349
  const {
247
350
  effectiveSlidingWindow, attentionKernelVariant, attnScale,
248
351
  cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
249
- causalForAttention,
352
+ prefillFallbackNeedsCast, causalForAttention,
250
353
  } = dispatchParams;
251
354
 
252
355
  // 5. Attention
@@ -267,6 +370,7 @@ export async function recordLayerAttentionGPU(
267
370
 
268
371
  let qForBDPA = qTensor;
269
372
  if (qForBDPA.dtype !== 'f16') {
373
+ assertAttentionDtypeTransitionAllowed(state, qForBDPA.dtype, 'f16', 'BDPA attention would narrow Q implicitly.');
270
374
  qForBDPA = await recordCastF32ToF16(recorder, qTensor);
271
375
  recorder.trackTemporaryBuffer(qForBDPA.buffer);
272
376
  }
@@ -288,6 +392,7 @@ export async function recordLayerAttentionGPU(
288
392
  tieredQuant: async () => {
289
393
  let qForAttention = qTensor;
290
394
  if (kvState.coldQuantMode !== 'none' && qTensor.dtype !== 'f32') {
395
+ assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Tiered quant attention would widen Q implicitly.');
291
396
  qForAttention = await recordCastF16ToF32(recorder, qTensor);
292
397
  recorder.trackTemporaryBuffer(qForAttention.buffer);
293
398
  }
@@ -300,21 +405,63 @@ export async function recordLayerAttentionGPU(
300
405
 
301
406
  const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
302
407
  const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
303
- return recordAttentionTieredQuant(recorder, qForAttention, cachedHotKTensor, cachedHotVTensor, kvState.cachedKCold, kvState.cachedVCold, kvState.coldScalesK, kvState.coldScalesV, numHeads, headDim, {
304
- seqLen: numTokens,
305
- coldLen: kvState.coldLen,
306
- hotLen: kvState.hotLen,
307
- numKVHeads,
308
- causal: causalForAttention,
309
- startPos: kvState.startPosForMask,
310
- slidingWindow: effectiveSlidingWindow ?? 0,
311
- attnSoftcap,
312
- scale: attnScale,
313
- hotWindow: kvState.hotWindow,
314
- hotStart: kvState.hotStart,
315
- packedStride: kvState.coldPackedStride,
316
- mode: kvState.coldQuantMode,
317
- });
408
+ return recordAttentionTieredQuant(
409
+ recorder,
410
+ qForAttention,
411
+ cachedHotKTensor,
412
+ cachedHotVTensor,
413
+ kvState.cachedKCold,
414
+ kvState.cachedVCold,
415
+ kvState.coldScalesK,
416
+ kvState.coldScalesV,
417
+ numHeads,
418
+ headDim,
419
+ buildTieredQuantAttentionOptions(kvState, {
420
+ seqLen: numTokens,
421
+ numKVHeads,
422
+ causal: causalForAttention,
423
+ startPos: kvState.startPosForMask,
424
+ slidingWindow: effectiveSlidingWindow ?? 0,
425
+ attnSoftcap,
426
+ scale: attnScale,
427
+ })
428
+ );
429
+ },
430
+ contiguousQuant: async () => {
431
+ let qForAttention = qTensor;
432
+ if (qTensor.dtype !== 'f32') {
433
+ assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Contiguous quant attention would widen Q implicitly.');
434
+ qForAttention = await recordCastF16ToF32(recorder, qTensor);
435
+ recorder.trackTemporaryBuffer(qForAttention.buffer);
436
+ }
437
+
438
+ if (!kvState.coldScalesK || !kvState.coldScalesV) {
439
+ throw new Error('Contiguous quant attention requires scale buffers.');
440
+ }
441
+ if (!kvState.rotationMatrixBuffer || !kvState.codebookCentroidsBuffer) {
442
+ throw new Error('Contiguous quant attention requires TurboQuant shared buffers.');
443
+ }
444
+
445
+ return recordAttentionContiguousQuant(
446
+ recorder,
447
+ qForAttention,
448
+ kvState.cachedKCold,
449
+ kvState.cachedVCold,
450
+ kvState.coldScalesK,
451
+ kvState.coldScalesV,
452
+ numHeads,
453
+ headDim,
454
+ buildContiguousQuantAttentionOptions(kvState, {
455
+ seqLen: numTokens,
456
+ kvLen: kvState.kvLenForAttention,
457
+ numKVHeads,
458
+ causal: causalForAttention,
459
+ startPos: kvState.startPosForMask,
460
+ slidingWindow: effectiveSlidingWindow ?? 0,
461
+ attnSoftcap,
462
+ scale: attnScale,
463
+ })
464
+ );
318
465
  },
319
466
  tiered: async () => {
320
467
  const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
@@ -339,22 +486,46 @@ export async function recordLayerAttentionGPU(
339
486
  hotLayout: kvState.hotWindow > 0 ? 1 : 0,
340
487
  });
341
488
  },
342
- contiguous: async () => recordAttention(recorder, qTensor, cachedKTensor, cachedVTensor, null, numHeads, headDim, {
343
- seqLen: numTokens,
344
- kvLen: kvState.kvLenForAttention,
345
- numKVHeads,
346
- causal: causalForAttention,
347
- startPos: kvState.startPosForMask,
348
- layerIdx,
349
- slidingWindow: effectiveSlidingWindow,
350
- attnSoftcap,
351
- scale: attnScale,
352
- kvStart: kvState.kvStart,
353
- kvLayout: kvState.kvLayout,
354
- kvPageTable: kvState.kvPageTable,
355
- kvPageSize: kvState.kvPageSize,
356
- kernelPath,
357
- }),
489
+ contiguous: async () => {
490
+ // Prefill fallback: quantized/tiered layouts use raw K/V for prefill, cast to f16 to match kernel path
491
+ let kForAttn = cachedKTensor;
492
+ let vForAttn = cachedVTensor;
493
+ if (prefillFallbackNeedsCast) {
494
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
495
+ if (cachedKDtype === 'f16' && kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
496
+ assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'Prefill fallback attention would narrow K implicitly.');
497
+ }
498
+ if (cachedVDtype === 'f16' && vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
499
+ assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'Prefill fallback attention would narrow V implicitly.');
500
+ }
501
+ const kCasted = cachedKDtype === 'f16' && kTensor.dtype !== 'f16'
502
+ ? await recordCastF32ToF16(recorder, kTensor) : kTensor;
503
+ const vCasted = cachedVDtype === 'f16' && vTensor.dtype !== 'f16'
504
+ ? await recordCastF32ToF16(recorder, vTensor) : vTensor;
505
+ kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
506
+ vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
507
+ if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
508
+ if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
509
+ }
510
+ return recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
511
+ seqLen: numTokens,
512
+ kvLen: kvState.kvLenForAttention,
513
+ numKVHeads,
514
+ causal: causalForAttention,
515
+ bidirectionalSpanStart: multimodalBidirectionalSpan?.start ?? 0,
516
+ bidirectionalSpanLength: multimodalBidirectionalSpan?.length ?? 0,
517
+ startPos: kvState.startPosForMask,
518
+ layerIdx,
519
+ slidingWindow: effectiveSlidingWindow,
520
+ attnSoftcap,
521
+ scale: attnScale,
522
+ kvStart: kvState.kvStart,
523
+ kvLayout: kvState.kvLayout,
524
+ kvPageTable: kvState.kvPageTable,
525
+ kvPageSize: kvState.kvPageSize,
526
+ kernelPath,
527
+ });
528
+ },
358
529
  };
359
530
  const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
360
531
  if (!runAttentionKernel) {
@@ -384,8 +555,19 @@ export async function recordLayerAttentionGPU(
384
555
  if (layerWeights.oProj && getWeightBuffer) {
385
556
  ({ oProjInput, oProjInputTemp } = await prepareAttentionProjectionInput(
386
557
  attnForProjection,
387
- matmulOutputDtype,
388
- (tensor) => recordCastF32ToF16(recorder, tensor)
558
+ oProjInputDtype,
559
+ (tensor) => {
560
+ assertAttentionDtypeTransitionAllowed(
561
+ state,
562
+ tensor.dtype,
563
+ oProjInputDtype,
564
+ 'Attention output projection would change activations implicitly.',
565
+ 'step_precision'
566
+ );
567
+ return oProjInputDtype === 'f16'
568
+ ? recordCastF32ToF16(recorder, tensor)
569
+ : recordCastF16ToF32(recorder, tensor);
570
+ }
389
571
  ));
390
572
  const oProjBuf = getWeightBuffer(layerWeights.oProj, 'o_proj');
391
573
  const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
@@ -417,11 +599,12 @@ export async function recordLayerAttentionGPU(
417
599
  role: 'o_proj',
418
600
  layerIdx,
419
601
  kernelPath,
420
- outputDtype: matmulOutputDtype,
602
+ outputDtype: oProjOutputDtype,
603
+ executionPolicies: state.executionPolicies ?? null,
421
604
  });
422
605
  }
423
606
  // Release temporary buffer if we created it (original was not already on GPU)
424
- if (!(layerWeights.oProj instanceof GPUBuffer) && !isWeightBuffer(layerWeights.oProj)) {
607
+ if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
425
608
  releaseOrTrack(recorder, isWeightBuffer(oProjBuf) ? oProjBuf.buffer : oProjBuf);
426
609
  }
427
610
  } else {
@@ -457,10 +640,13 @@ export async function recordLayerAttentionGPU(
457
640
  if (oProjInputTemp && oProjInputTemp.buffer !== attnForProjection.buffer) {
458
641
  buffersToTrack.push(oProjInputTemp.buffer);
459
642
  }
460
- if (wantsF16Output && output.dtype !== 'f16') {
461
- const f16Output = await recordCastF32ToF16(recorder, output);
643
+ if (output.dtype !== oProjOutputDtype) {
644
+ assertAttentionDtypeTransitionAllowed(state, output.dtype, oProjOutputDtype, 'Attention output would change implicitly before leaving the layer.');
645
+ const coercedOutput = oProjOutputDtype === 'f16'
646
+ ? await recordCastF32ToF16(recorder, output)
647
+ : await recordCastF16ToF32(recorder, output);
462
648
  buffersToTrack.push(output.buffer);
463
- finalOutput = f16Output;
649
+ finalOutput = coercedOutput;
464
650
  }
465
651
 
466
652
  // Track intermediate buffers for cleanup after submit (not release!)
@@ -471,8 +657,10 @@ export async function recordLayerAttentionGPU(
471
657
  if (qGateTensor) {
472
658
  recorder.trackTemporaryBuffer(qGateTensor.buffer);
473
659
  }
474
- recorder.trackTemporaryBuffer(kTensor.buffer);
475
- recorder.trackTemporaryBuffer(vTensor.buffer);
660
+ if (!retainSharedKvBuffers) {
661
+ recorder.trackTemporaryBuffer(kTensor.buffer);
662
+ recorder.trackTemporaryBuffer(vTensor.buffer);
663
+ }
476
664
  for (const buffer of buffersToTrack) {
477
665
  recorder.trackTemporaryBuffer(buffer);
478
666
  }
@@ -506,10 +694,10 @@ export async function recordLayerAttentionGPU(
506
694
  if (qTensor?.buffer) {
507
695
  trackOnce(qTensor.buffer);
508
696
  }
509
- if (kTensor?.buffer) {
697
+ if (kTensor?.buffer && !retainSharedKvBuffers) {
510
698
  trackOnce(kTensor.buffer);
511
699
  }
512
- if (vTensor?.buffer) {
700
+ if (vTensor?.buffer && !retainSharedKvBuffers) {
513
701
  trackOnce(vTensor.buffer);
514
702
  }
515
703
  if (normed?.buffer && normed.buffer !== attentionInput?.buffer) {