doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,6 +1,6 @@
1
1
 
2
2
 
3
- import { isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
3
+ import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
4
  import { getDevice } from '../../../../gpu/device.js';
5
5
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
6
6
  import {
@@ -11,6 +11,7 @@ import {
11
11
  runAttentionBDPA,
12
12
  runAttentionTieredQuant,
13
13
  runAttentionTiered,
14
+ runAttentionContiguousQuant,
14
15
  runSiLU,
15
16
  castF16ToF32,
16
17
  castF32ToF16,
@@ -31,6 +32,7 @@ import {
31
32
  resolveAttentionProjectionOutputDtype,
32
33
  projectAttentionQKV,
33
34
  applyAttentionQKNorm,
35
+ applyAttentionValueNorm,
34
36
  } from './projections.js';
35
37
  import { prepareAttentionProjectionInput } from './output-projection.js';
36
38
 
@@ -38,15 +40,38 @@ import {
38
40
  shouldDebugLayer,
39
41
  markStageLogged,
40
42
  } from './types.js';
41
- import { getKernelPathMatmulVariant } from '../../../../config/kernel-path-loader.js';
43
+ import {
44
+ getKernelPathMatmulPrecision,
45
+ getKernelPathMatmulVariant,
46
+ } from '../../../../config/kernel-path-loader.js';
42
47
  import {
43
48
  resolveKVCacheState,
44
49
  buildAttentionDispatchParams,
45
50
  buildAttentionInputsData,
46
51
  } from './dispatch-params.js';
52
+ import {
53
+ buildTieredQuantAttentionOptions,
54
+ buildContiguousQuantAttentionOptions,
55
+ } from './quant-options.js';
56
+ import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
57
+ import {
58
+ resolveAttentionPrecisionContract,
59
+ isAttentionKvDtypeExplicit,
60
+ } from './precision-contract.js';
47
61
 
48
62
  const ATTENTION_DTYPE_LOGGED = new Set();
49
63
 
64
+ function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
65
+ assertImplicitDtypeTransitionAllowed({
66
+ executionPolicies: state?.executionPolicies ?? null,
67
+ fromDtype,
68
+ toDtype,
69
+ op: 'attention',
70
+ detail,
71
+ transitionDeclaredBy,
72
+ });
73
+ }
74
+
50
75
  export async function runLayerAttentionGPU(
51
76
  input,
52
77
  layerWeights,
@@ -78,18 +103,31 @@ export async function runLayerAttentionGPU(
78
103
  tokenIds = null,
79
104
  kernelPath = null,
80
105
  disableRoPE = false,
106
+ multimodalBidirectionalSpan = null,
107
+ sharedKVSourceLayerIdx = null,
108
+ storeSharedKV = false,
81
109
  } = config;
82
110
 
83
111
  const device = getDevice();
84
-
85
- const desiredOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
86
- dtype: config.activationDtype,
112
+ const phase = isPrefill ? 'prefill' : 'decode';
113
+ const attentionPrecisionContract = resolveAttentionPrecisionContract(config, state);
114
+ const attentionActivationDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
115
+ dtype: attentionPrecisionContract.resolvedActivationDtype ?? config.activationDtype,
116
+ });
117
+ const oProjPrecision = getKernelPathMatmulPrecision('o_proj', phase, layerIdx, kernelPath);
118
+ const oProjInputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
119
+ dtype: oProjPrecision?.inputDtype ?? attentionActivationDtype,
87
120
  });
88
- const wantsF16Output = desiredOutputDtype === 'f16';
89
- const useF16Activations = wantsF16Output;
90
- const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: wantsF16Output });
91
- const kvCacheDtype = state.kvCache?.kvDtype ?? kvCacheFallback;
92
- const allowF16Attention = wantsF16Output && kvCacheDtype === 'f16';
121
+ const oProjOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
122
+ dtype: oProjPrecision?.outputDtype
123
+ ?? attentionPrecisionContract.resolvedOutputDtype
124
+ ?? config.activationDtype,
125
+ });
126
+ const wantsF16Output = oProjOutputDtype === 'f16';
127
+ const useF16Activations = attentionActivationDtype === 'f16';
128
+ const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
129
+ const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
130
+ const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
93
131
  let attentionInput = input;
94
132
  let attentionInputTemp = false;
95
133
  let normed = attentionInput;
@@ -102,8 +140,16 @@ export async function runLayerAttentionGPU(
102
140
  let output = null;
103
141
  let finalOutput = null;
104
142
  let oProjInputTemp = null;
105
- if (wantsF16Output && !allowF16Attention) {
106
- attentionInput = await castF16ToF32(input);
143
+ if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
144
+ assertAttentionDtypeTransitionAllowed(
145
+ state,
146
+ input.dtype,
147
+ attentionActivationDtype,
148
+ 'The attention kernel selection would widen the input implicitly.'
149
+ );
150
+ attentionInput = attentionActivationDtype === 'f16'
151
+ ? await castF32ToF16(input)
152
+ : await castF16ToF32(input);
107
153
  attentionInputTemp = true;
108
154
  normed = attentionInput;
109
155
  }
@@ -116,16 +162,34 @@ export async function runLayerAttentionGPU(
116
162
  // Debug logging moved to debug-utils.ts (enable via setDebugConfig)
117
163
 
118
164
  if (!layerWeights) {
119
- // Return zeros if no weights
120
165
  const bytesPerElement = wantsF16Output ? 2 : 4;
121
166
  const outputBuf = acquireBuffer(numTokens * hiddenSize * bytesPerElement, undefined, 'attn_output');
122
- const outputDtype = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: wantsF16Output });
123
- const output = createTensor(outputBuf, outputDtype, [numTokens, hiddenSize], 'attn_output');
167
+ const output = createTensor(outputBuf, oProjOutputDtype, [numTokens, hiddenSize], 'attn_output');
124
168
  return { output, residualFused: false };
125
169
  }
126
170
 
127
171
  const qSize = numTokens * numHeads * headDim;
128
172
  const kvSize = numTokens * numKVHeads * headDim;
173
+ const sharedKVEntry = sharedKVSourceLayerIdx == null
174
+ ? null
175
+ : (state.sharedAttentionState?.get(sharedKVSourceLayerIdx) ?? null);
176
+ if (sharedKVSourceLayerIdx != null && !sharedKVEntry) {
177
+ throw new Error(
178
+ `Layer ${layerIdx} requires shared K/V from layer ${sharedKVSourceLayerIdx}, ` +
179
+ 'but no shared K/V state was stored for that source layer.'
180
+ );
181
+ }
182
+ if (sharedKVEntry && (
183
+ sharedKVEntry.headDim !== headDim
184
+ || sharedKVEntry.numKVHeads !== numKVHeads
185
+ )) {
186
+ throw new Error(
187
+ `Layer ${layerIdx} shared K/V geometry mismatch. ` +
188
+ `Expected numKVHeads=${numKVHeads}, headDim=${headDim}; ` +
189
+ `got numKVHeads=${sharedKVEntry.numKVHeads}, headDim=${sharedKVEntry.headDim}.`
190
+ );
191
+ }
192
+ const reusesSharedKV = sharedKVEntry != null;
129
193
 
130
194
  // 1. Input norm
131
195
 
@@ -144,7 +208,7 @@ export async function runLayerAttentionGPU(
144
208
  rmsNormWeightOffset: config.rmsNormWeightOffset,
145
209
  });
146
210
  } finally {
147
- if (!(layerWeights.inputNorm instanceof GPUBuffer)) releaseBuffer(normWeightBuf);
211
+ if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseBuffer(normWeightBuf);
148
212
  }
149
213
 
150
214
  // Trace input norm output
@@ -167,6 +231,7 @@ export async function runLayerAttentionGPU(
167
231
  numTokens,
168
232
  hiddenSize,
169
233
  probes: state.debugProbes,
234
+ operatorDiagnostics: state.operatorDiagnostics,
170
235
  dtype: normed.dtype,
171
236
  });
172
237
  }
@@ -184,7 +249,7 @@ export async function runLayerAttentionGPU(
184
249
  const logKey = `L${layerIdx}_${phase}_dtypes`;
185
250
  if (!ATTENTION_DTYPE_LOGGED.has(logKey)) {
186
251
  ATTENTION_DTYPE_LOGGED.add(logKey);
187
- trace.attn(layerIdx, `dtypes: activation=${config.activationDtype ?? 'unknown'}, input=${input.dtype}, normed=${normed.dtype}`);
252
+ trace.attn(layerIdx, `dtypes: activation=${attentionActivationDtype}, input=${input.dtype}, normed=${normed.dtype}`);
188
253
  }
189
254
  }
190
255
 
@@ -198,11 +263,11 @@ export async function runLayerAttentionGPU(
198
263
  }
199
264
 
200
265
  // 2. Q/K/V projections
201
- const qProjVariant = getKernelPathMatmulVariant('q_proj', isPrefill ? 'prefill' : 'decode', layerIdx, kernelPath);
266
+ const qProjVariant = getKernelPathMatmulVariant('q_proj', phase, layerIdx, kernelPath);
202
267
  const kernelPathIsF16 = qProjVariant != null && qProjVariant.includes('f16') && !qProjVariant.includes('f32');
203
- const matmulOutputDtype = resolveAttentionProjectionOutputDtype(desiredOutputDtype, {
268
+ const matmulOutputDtype = resolveAttentionProjectionOutputDtype(attentionActivationDtype, {
204
269
  forceF32: shouldForceF32AttentionProjectionForRoPE({
205
- attentionInputDtype: desiredOutputDtype,
270
+ attentionInputDtype: attentionActivationDtype,
206
271
  headDim,
207
272
  rotaryDim: config.ropeRotaryDim,
208
273
  interleaved: config.ropeInterleaved,
@@ -226,6 +291,9 @@ export async function runLayerAttentionGPU(
226
291
  lora,
227
292
  matmulDebug: state.runtimeConfig?.shared?.debug?.matmul ?? null,
228
293
  attentionOutputGate: config.attentionOutputGate === true,
294
+ sharedKTensor: sharedKVEntry?.kTensor ?? null,
295
+ sharedVTensor: sharedKVEntry?.vTensor ?? null,
296
+ executionPolicies: state.executionPolicies ?? null,
229
297
  releaseTemporary: (buffer) => releaseBuffer(buffer),
230
298
  onFusedQKV: layerIdx === 0 && isPrefill
231
299
  ? ({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize }) => {
@@ -245,22 +313,27 @@ export async function runLayerAttentionGPU(
245
313
  numTokens,
246
314
  hiddenSize: numHeads * headDim,
247
315
  probes: state.debugProbes,
316
+ operatorDiagnostics: state.operatorDiagnostics,
248
317
  dtype: qTensor.dtype,
249
318
  });
250
- await runProbes('k_proj', kTensor.buffer, {
251
- layerIdx,
252
- numTokens,
253
- hiddenSize: numKVHeads * headDim,
254
- probes: state.debugProbes,
255
- dtype: kTensor.dtype,
256
- });
257
- await runProbes('v_proj', vTensor.buffer, {
258
- layerIdx,
259
- numTokens,
260
- hiddenSize: numKVHeads * headDim,
261
- probes: state.debugProbes,
262
- dtype: vTensor.dtype,
263
- });
319
+ if (!reusesSharedKV) {
320
+ await runProbes('k_proj', kTensor.buffer, {
321
+ layerIdx,
322
+ numTokens,
323
+ hiddenSize: numKVHeads * headDim,
324
+ probes: state.debugProbes,
325
+ operatorDiagnostics: state.operatorDiagnostics,
326
+ dtype: kTensor.dtype,
327
+ });
328
+ await runProbes('v_proj', vTensor.buffer, {
329
+ layerIdx,
330
+ numTokens,
331
+ hiddenSize: numKVHeads * headDim,
332
+ probes: state.debugProbes,
333
+ operatorDiagnostics: state.operatorDiagnostics,
334
+ dtype: vTensor.dtype,
335
+ });
336
+ }
264
337
 
265
338
  // Kernel step debug: Q/K/V projections
266
339
  if (isKernelDebugEnabled(layerIdx)) {
@@ -319,6 +392,7 @@ export async function runLayerAttentionGPU(
319
392
  headDim,
320
393
  rmsNormWeightOffset: config.rmsNormWeightOffset,
321
394
  releaseTemporary: (buffer) => releaseBuffer(buffer),
395
+ skipKNorm: reusesSharedKV,
322
396
  onQNormApplied: isKernelDebugEnabled(layerIdx)
323
397
  ? async (tensor) => {
324
398
  await dumpTokenVector(tensor.buffer, 'Q_norm', {
@@ -341,20 +415,36 @@ export async function runLayerAttentionGPU(
341
415
  : null,
342
416
  }));
343
417
 
418
+ if (config.valueNorm === true && !reusesSharedKV) {
419
+ vTensor = await applyAttentionValueNorm({
420
+ recorder: null,
421
+ vTensor,
422
+ rmsNormEps,
423
+ numTokens,
424
+ numKVHeads,
425
+ headDim,
426
+ releaseTemporary: (buffer) => releaseBuffer(buffer),
427
+ });
428
+ }
429
+
344
430
  await runProbes('q_norm', qTensor.buffer, {
345
431
  layerIdx,
346
432
  numTokens,
347
433
  hiddenSize: numHeads * headDim,
348
434
  probes: state.debugProbes,
435
+ operatorDiagnostics: state.operatorDiagnostics,
349
436
  dtype: qTensor.dtype,
350
437
  });
351
- await runProbes('k_norm', kTensor.buffer, {
352
- layerIdx,
353
- numTokens,
354
- hiddenSize: numKVHeads * headDim,
355
- probes: state.debugProbes,
356
- dtype: kTensor.dtype,
357
- });
438
+ if (!reusesSharedKV) {
439
+ await runProbes('k_norm', kTensor.buffer, {
440
+ layerIdx,
441
+ numTokens,
442
+ hiddenSize: numKVHeads * headDim,
443
+ probes: state.debugProbes,
444
+ operatorDiagnostics: state.operatorDiagnostics,
445
+ dtype: kTensor.dtype,
446
+ });
447
+ }
358
448
 
359
449
  if (normed !== attentionInput) releaseBuffer(normed.buffer);
360
450
  if (attentionInputTemp) releaseBuffer(attentionInput.buffer);
@@ -368,19 +458,25 @@ export async function runLayerAttentionGPU(
368
458
  rotaryDim: config.ropeRotaryDim,
369
459
  interleaved: config.ropeInterleaved,
370
460
  startPos: currentSeqLen,
461
+ executionPolicies: state.executionPolicies ?? null,
371
462
  });
372
- await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
373
- numHeads: numKVHeads,
374
- headDim,
375
- rotaryDim: config.ropeRotaryDim,
376
- interleaved: config.ropeInterleaved,
377
- startPos: currentSeqLen,
378
- });
463
+ if (!reusesSharedKV) {
464
+ await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
465
+ numHeads: numKVHeads,
466
+ headDim,
467
+ rotaryDim: config.ropeRotaryDim,
468
+ interleaved: config.ropeInterleaved,
469
+ startPos: currentSeqLen,
470
+ executionPolicies: state.executionPolicies ?? null,
471
+ });
472
+ }
379
473
 
380
474
  // Trace RoPE outputs
381
475
  if (kernelTrace.enabled) {
382
476
  await traceStep('rope', `L${layerIdx}.q_rope`, layerIdx, qTensor.buffer, [numTokens, numHeads * headDim]);
383
- await traceStep('rope', `L${layerIdx}.k_rope`, layerIdx, kTensor.buffer, [numTokens, numKVHeads * headDim]);
477
+ if (!reusesSharedKV) {
478
+ await traceStep('rope', `L${layerIdx}.k_rope`, layerIdx, kTensor.buffer, [numTokens, numKVHeads * headDim]);
479
+ }
384
480
  }
385
481
  }
386
482
  await runProbes('q_rope', qTensor.buffer, {
@@ -388,6 +484,7 @@ export async function runLayerAttentionGPU(
388
484
  numTokens,
389
485
  hiddenSize: numHeads * headDim,
390
486
  probes: state.debugProbes,
487
+ operatorDiagnostics: state.operatorDiagnostics,
391
488
  dtype: qTensor.dtype,
392
489
  });
393
490
  await runProbes('k_rope', kTensor.buffer, {
@@ -395,6 +492,7 @@ export async function runLayerAttentionGPU(
395
492
  numTokens,
396
493
  hiddenSize: numKVHeads * headDim,
397
494
  probes: state.debugProbes,
495
+ operatorDiagnostics: state.operatorDiagnostics,
398
496
  dtype: kTensor.dtype,
399
497
  });
400
498
  if (isKernelDebugEnabled(layerIdx)) {
@@ -413,6 +511,15 @@ export async function runLayerAttentionGPU(
413
511
  });
414
512
  }
415
513
 
514
+ if (storeSharedKV && state.sharedAttentionState) {
515
+ state.sharedAttentionState.set(layerIdx, {
516
+ kTensor,
517
+ vTensor,
518
+ headDim,
519
+ numKVHeads,
520
+ });
521
+ }
522
+
416
523
  // Debug: Check Q/K after RoPE for L0 prefill
417
524
  // Debug: Q/K after RoPE for configured layers
418
525
  if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'qk_rope', debugFlags) && debugCheckBuffer) {
@@ -424,6 +531,13 @@ export async function runLayerAttentionGPU(
424
531
 
425
532
  if (state.kvCache?.hasGPUCache?.()) {
426
533
  if (state.kvCache.kvDtype === 'f16') {
534
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
535
+ if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
536
+ assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
537
+ }
538
+ if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
539
+ assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
540
+ }
427
541
  const kCasted = kTensor.dtype === 'f16' ? kTensor : await castF32ToF16(kTensor);
428
542
  const vCasted = vTensor.dtype === 'f16' ? vTensor : await castF32ToF16(vTensor);
429
543
 
@@ -448,14 +562,14 @@ export async function runLayerAttentionGPU(
448
562
  layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
449
563
  slidingWindow, layerType, layerTypes: config.layerTypes,
450
564
  queryPreAttnScalar, causalAttention: config.causalAttention,
451
- activationDtype: config.activationDtype,
452
- kvCacheDtype: state.kvCache?.kvDtype ?? null,
565
+ activationDtype: attentionActivationDtype,
566
+ kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
453
567
  };
454
568
  const dispatchParams = buildAttentionDispatchParams(dispatchConfig, state, kTensor, vTensor, kvState);
455
569
  const {
456
570
  effectiveSlidingWindow, attentionKernelVariant, attnScale,
457
571
  cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
458
- causalForAttention,
572
+ prefillFallbackNeedsCast, causalForAttention,
459
573
  } = dispatchParams;
460
574
 
461
575
  // 5. Attention (uses raw GPUBuffers)
@@ -500,6 +614,7 @@ export async function runLayerAttentionGPU(
500
614
  const basisVTensor = createTensor(kvState.bdpaBasisV, basisVDtype, [basisCount, numKVHeads * headDim], 'bdpa_basis_v');
501
615
  let qForBDPA = qTensor;
502
616
  if (qForBDPA.dtype !== 'f16') {
617
+ assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f16', 'Q would be narrowed implicitly for BDPA.');
503
618
  qForBDPA = await castF32ToF16(qTensor);
504
619
  }
505
620
  const output = await runAttentionBDPA(qForBDPA, basisKTensor, basisVTensor, kvState.bdpaPagedK, kvState.bdpaPagedV, kvState.bdpaIndex, numHeads, headDim, {
@@ -524,6 +639,7 @@ export async function runLayerAttentionGPU(
524
639
  let qForAttention = qTensor;
525
640
  let qTemp = null;
526
641
  if (kvState.coldQuantMode !== 'none' && qTensor.dtype !== 'f32') {
642
+ assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Q would be widened implicitly for tiered quant attention.');
527
643
  qForAttention = await castF16ToF32(qTensor);
528
644
  qTemp = qForAttention;
529
645
  }
@@ -537,21 +653,67 @@ export async function runLayerAttentionGPU(
537
653
  throw new Error('Tiered quant attention requires cold scale buffers.');
538
654
  }
539
655
 
540
- const output = await runAttentionTieredQuant(qForAttention, cachedHotKTensor, cachedHotVTensor, kvState.cachedKCold, kvState.cachedVCold, kvState.coldScalesK, kvState.coldScalesV, numHeads, headDim, {
541
- seqLen: numTokens,
542
- coldLen: kvState.coldLen,
543
- hotLen: kvState.hotLen,
544
- numKVHeads,
545
- causal: causalForAttention,
546
- startPos: kvState.startPosForMask,
547
- slidingWindow: effectiveSlidingWindow ?? 0,
548
- attnSoftcap,
549
- scale: attnScale,
550
- hotWindow: kvState.hotWindow,
551
- hotStart: kvState.hotStart,
552
- packedStride: kvState.coldPackedStride,
553
- mode: kvState.coldQuantMode,
554
- });
656
+ const output = await runAttentionTieredQuant(
657
+ qForAttention,
658
+ cachedHotKTensor,
659
+ cachedHotVTensor,
660
+ kvState.cachedKCold,
661
+ kvState.cachedVCold,
662
+ kvState.coldScalesK,
663
+ kvState.coldScalesV,
664
+ numHeads,
665
+ headDim,
666
+ buildTieredQuantAttentionOptions(kvState, {
667
+ seqLen: numTokens,
668
+ numKVHeads,
669
+ causal: causalForAttention,
670
+ startPos: kvState.startPosForMask,
671
+ slidingWindow: effectiveSlidingWindow ?? 0,
672
+ attnSoftcap,
673
+ scale: attnScale,
674
+ })
675
+ );
676
+
677
+ if (qTemp) {
678
+ releaseBuffer(qTemp.buffer);
679
+ }
680
+ return output;
681
+ },
682
+ contiguousQuant: async () => {
683
+ let qForAttention = qTensor;
684
+ let qTemp = null;
685
+ if (qTensor.dtype !== 'f32') {
686
+ assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Q would be widened implicitly for contiguous quant attention.');
687
+ qForAttention = await castF16ToF32(qTensor);
688
+ qTemp = qForAttention;
689
+ }
690
+
691
+ if (!kvState.coldScalesK || !kvState.coldScalesV) {
692
+ throw new Error('Contiguous quant attention requires scale buffers.');
693
+ }
694
+ if (!kvState.rotationMatrixBuffer || !kvState.codebookCentroidsBuffer) {
695
+ throw new Error('Contiguous quant attention requires TurboQuant shared buffers.');
696
+ }
697
+
698
+ const output = await runAttentionContiguousQuant(
699
+ qForAttention,
700
+ kvState.cachedKCold,
701
+ kvState.cachedVCold,
702
+ kvState.coldScalesK,
703
+ kvState.coldScalesV,
704
+ numHeads,
705
+ headDim,
706
+ buildContiguousQuantAttentionOptions(kvState, {
707
+ seqLen: numTokens,
708
+ kvLen: kvState.kvLenForAttention,
709
+ numKVHeads,
710
+ causal: causalForAttention,
711
+ startPos: kvState.startPosForMask,
712
+ slidingWindow: effectiveSlidingWindow ?? 0,
713
+ attnSoftcap,
714
+ scale: attnScale,
715
+ })
716
+ );
555
717
 
556
718
  if (qTemp) {
557
719
  releaseBuffer(qTemp.buffer);
@@ -582,22 +744,49 @@ export async function runLayerAttentionGPU(
582
744
  hotLayout: kvState.hotWindow > 0 ? 1 : 0,
583
745
  });
584
746
  },
585
- contiguous: async () => runAttention(qTensor, cachedKTensor, cachedVTensor, null, numHeads, headDim, {
586
- seqLen: numTokens,
587
- kvLen: kvState.kvLenForAttention,
588
- numKVHeads,
589
- causal: causalForAttention,
590
- startPos: kvState.startPosForMask,
591
- layerIdx,
592
- slidingWindow: effectiveSlidingWindow,
593
- attnSoftcap,
594
- scale: attnScale,
595
- kvStart: kvState.kvStart,
596
- kvLayout: kvState.kvLayout,
597
- kvPageTable: kvState.kvPageTable,
598
- kvPageSize: kvState.kvPageSize,
599
- kernelPath,
600
- }),
747
+ contiguous: async () => {
748
+ // Prefill fallback: quantized/tiered layouts use raw K/V for prefill, cast to f16 to match kernel path
749
+ let kForAttn = cachedKTensor;
750
+ let vForAttn = cachedVTensor;
751
+ if (prefillFallbackNeedsCast) {
752
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
753
+ if (cachedKDtype === 'f16' && kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
754
+ assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for prefill fallback.');
755
+ }
756
+ if (cachedVDtype === 'f16' && vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
757
+ assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for prefill fallback.');
758
+ }
759
+ const kCasted = cachedKDtype === 'f16' && kTensor.dtype !== 'f16'
760
+ ? await castF32ToF16(kTensor) : kTensor;
761
+ const vCasted = cachedVDtype === 'f16' && vTensor.dtype !== 'f16'
762
+ ? await castF32ToF16(vTensor) : vTensor;
763
+ kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
764
+ vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
765
+ }
766
+ const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
767
+ seqLen: numTokens,
768
+ kvLen: kvState.kvLenForAttention,
769
+ numKVHeads,
770
+ causal: causalForAttention,
771
+ bidirectionalSpanStart: multimodalBidirectionalSpan?.start ?? 0,
772
+ bidirectionalSpanLength: multimodalBidirectionalSpan?.length ?? 0,
773
+ startPos: kvState.startPosForMask,
774
+ layerIdx,
775
+ slidingWindow: effectiveSlidingWindow,
776
+ attnSoftcap,
777
+ scale: attnScale,
778
+ kvStart: kvState.kvStart,
779
+ kvLayout: kvState.kvLayout,
780
+ kvPageTable: kvState.kvPageTable,
781
+ kvPageSize: kvState.kvPageSize,
782
+ kernelPath,
783
+ });
784
+ if (prefillFallbackNeedsCast) {
785
+ if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
786
+ if (vTensor.dtype !== 'f16') releaseBuffer(vForAttn.buffer);
787
+ }
788
+ return result;
789
+ },
601
790
  };
602
791
 
603
792
  const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
@@ -649,8 +838,19 @@ export async function runLayerAttentionGPU(
649
838
  if (layerWeights.oProj && getWeightBuffer) {
650
839
  ({ oProjInput, oProjInputTemp } = await prepareAttentionProjectionInput(
651
840
  attnForProjection,
652
- matmulOutputDtype,
653
- castF32ToF16
841
+ oProjInputDtype,
842
+ async (tensor) => {
843
+ assertAttentionDtypeTransitionAllowed(
844
+ state,
845
+ tensor.dtype,
846
+ oProjInputDtype,
847
+ 'o_proj input would change dtype implicitly.',
848
+ 'step_precision'
849
+ );
850
+ return oProjInputDtype === 'f16'
851
+ ? castF32ToF16(tensor)
852
+ : castF16ToF32(tensor);
853
+ }
654
854
  ));
655
855
  const oProjBuf = getWeightBuffer(layerWeights.oProj, 'o_proj');
656
856
  const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
@@ -686,11 +886,12 @@ export async function runLayerAttentionGPU(
686
886
  role: 'o_proj',
687
887
  layerIdx,
688
888
  kernelPath,
689
- outputDtype: matmulOutputDtype,
889
+ outputDtype: oProjOutputDtype,
890
+ executionPolicies: state.executionPolicies ?? null,
690
891
  });
691
892
  }
692
893
  // Release temporary buffer if we created it (original was not already on GPU)
693
- if (!(layerWeights.oProj instanceof GPUBuffer) && !isWeightBuffer(layerWeights.oProj)) {
894
+ if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
694
895
  releaseBuffer(isWeightBuffer(oProjBuf) ? oProjBuf.buffer : oProjBuf);
695
896
  }
696
897
 
@@ -750,10 +951,19 @@ export async function runLayerAttentionGPU(
750
951
  buffersToRelease.push(attnForProjection.buffer);
751
952
  }
752
953
 
753
- if (wantsF16Output && output.dtype !== 'f16') {
754
- const f16Output = await castF32ToF16(output);
954
+ if (output.dtype !== oProjOutputDtype) {
955
+ assertAttentionDtypeTransitionAllowed(
956
+ state,
957
+ output.dtype,
958
+ oProjOutputDtype,
959
+ 'Attention output would change dtype implicitly at the end of the op.',
960
+ 'step_precision'
961
+ );
962
+ const coercedOutput = oProjOutputDtype === 'f16'
963
+ ? await castF32ToF16(output)
964
+ : await castF16ToF32(output);
755
965
  buffersToRelease.push(output.buffer);
756
- finalOutput = f16Output;
966
+ finalOutput = coercedOutput;
757
967
  }
758
968
 
759
969
  // Cleanup
@@ -761,8 +971,10 @@ export async function runLayerAttentionGPU(
761
971
  if (qGateTensor) {
762
972
  releaseBuffer(qGateTensor.buffer);
763
973
  }
764
- releaseBuffer(kTensor.buffer);
765
- releaseBuffer(vTensor.buffer);
974
+ if (!reusesSharedKV && !storeSharedKV) {
975
+ releaseBuffer(kTensor.buffer);
976
+ releaseBuffer(vTensor.buffer);
977
+ }
766
978
  for (const buffer of buffersToRelease) {
767
979
  releaseBuffer(buffer);
768
980
  }
@@ -796,10 +1008,10 @@ export async function runLayerAttentionGPU(
796
1008
  if (qTensor?.buffer) {
797
1009
  releaseOnce(qTensor.buffer);
798
1010
  }
799
- if (kTensor?.buffer) {
1011
+ if (kTensor?.buffer && !reusesSharedKV && !storeSharedKV) {
800
1012
  releaseOnce(kTensor.buffer);
801
1013
  }
802
- if (vTensor?.buffer) {
1014
+ if (vTensor?.buffer && !reusesSharedKV && !storeSharedKV) {
803
1015
  releaseOnce(vTensor.buffer);
804
1016
  }
805
1017
  if (normed?.buffer && normed.buffer !== attentionInput?.buffer) {