doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -5,8 +5,9 @@ import { getDevice } from '../../../gpu/device.js';
5
5
  import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
6
6
  import { allowReadback } from '../../../gpu/perf-guards.js';
7
7
  import { createTensor } from '../../../gpu/tensor.js';
8
+ import { recordScale, runScale } from '../../../gpu/kernel-selector.js';
8
9
  import {
9
- doAttention, doRMSNorm, doResidualAdd,
10
+ doAttention, doRMSNorm, doResidualAdd, doMatmul, doGeLU,
10
11
  doConv,
11
12
  doCast,
12
13
  releaseOrTrack
@@ -24,6 +25,8 @@ import { recordCheckFiniteness } from '../../../gpu/kernels/check-finiteness.js'
24
25
  import { shouldRunFinitenessGuard } from './finiteness-policy.js';
25
26
  import { runLinearAttentionLayer } from './linear-attention.js';
26
27
  import { validateAttnConfig } from './attention/attn-config.js';
28
+ import { createPerLayerInputTensor, resolveDensePleProjectionWeight } from './per-layer-inputs.js';
29
+ import { isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
27
30
 
28
31
  // ============================================================================
29
32
  // Architecture Detection
@@ -122,6 +125,306 @@ function isLinearLayerType(layerType) {
122
125
  || normalized === 'gated_delta_net';
123
126
  }
124
127
 
128
+ function resolveAttentionRotaryDim(config, layerType) {
129
+ if (isSlidingLayerType(layerType)) {
130
+ return config.ropeLocalRotaryDim ?? config.ropeRotaryDim;
131
+ }
132
+ return config.ropeRotaryDim;
133
+ }
134
+
135
+ function resolveAttentionHeadDim(config, layerType) {
136
+ if (isSlidingLayerType(layerType)) {
137
+ return config.headDim;
138
+ }
139
+ return config.globalHeadDim ?? config.headDim;
140
+ }
141
+
142
+ function resolveAttentionKVSharing(config, layerIdx, layerType) {
143
+ const layerTypes = Array.isArray(config?.layerTypes) ? config.layerTypes : null;
144
+ const numKvSharedLayers = Number(config?.numKvSharedLayers ?? 0);
145
+ if (!layerTypes || layerTypes.length === 0 || !Number.isFinite(numKvSharedLayers) || numKvSharedLayers <= 0) {
146
+ return { sharedKVSourceLayerIdx: null, storeSharedKV: false };
147
+ }
148
+
149
+ const firstKvSharedLayerIdx = layerTypes.length - Math.trunc(numKvSharedLayers);
150
+ if (firstKvSharedLayerIdx <= 0 || layerIdx < 0 || layerIdx >= layerTypes.length) {
151
+ return { sharedKVSourceLayerIdx: null, storeSharedKV: false };
152
+ }
153
+
154
+ const normalizedLayerType = normalizeLayerType(layerType);
155
+ if (!normalizedLayerType) {
156
+ return { sharedKVSourceLayerIdx: null, storeSharedKV: false };
157
+ }
158
+
159
+ let sourceLayerIdx = null;
160
+ for (let index = firstKvSharedLayerIdx - 1; index >= 0; index -= 1) {
161
+ if (normalizeLayerType(layerTypes[index]) === normalizedLayerType) {
162
+ sourceLayerIdx = index;
163
+ break;
164
+ }
165
+ }
166
+ if (sourceLayerIdx == null) {
167
+ return { sharedKVSourceLayerIdx: null, storeSharedKV: false };
168
+ }
169
+
170
+ if (layerIdx >= firstKvSharedLayerIdx) {
171
+ return { sharedKVSourceLayerIdx: sourceLayerIdx, storeSharedKV: false };
172
+ }
173
+
174
+ return {
175
+ sharedKVSourceLayerIdx: null,
176
+ storeSharedKV: layerIdx === sourceLayerIdx,
177
+ };
178
+ }
179
+
180
+ function hasPerLayerInputBlock(config) {
181
+ const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
182
+ return Number.isFinite(hiddenSizePerLayerInput) && hiddenSizePerLayerInput > 0;
183
+ }
184
+
185
+ function resolveLayerScalarValue(layerScalar) {
186
+ if (layerScalar == null) {
187
+ return 1;
188
+ }
189
+ if (!(layerScalar instanceof Float32Array) || layerScalar.length === 0) {
190
+ throw new Error(
191
+ 'Gemma 4 per-layer input layer_scalar must be CPU-resident Float32Array data. ' +
192
+ 'Re-convert or reload the model with the updated loader.'
193
+ );
194
+ }
195
+ const value = Number(layerScalar[0]);
196
+ if (!Number.isFinite(value)) {
197
+ throw new Error(`Gemma 4 layer_scalar must be finite; got "${String(layerScalar[0])}".`);
198
+ }
199
+ return value;
200
+ }
201
+
202
+ async function debugLayerTensor(context, layerIdx, label, tensor, numTokens, hiddenSize) {
203
+ if (!context.debugCheckBuffer) return;
204
+ if (!shouldDebugLayerOutput(layerIdx, context.debugLayers)) return;
205
+ if (!isGpuBufferInstance(tensor?.buffer)) return;
206
+ await context.debugCheckBuffer(tensor.buffer, `L${layerIdx} ${label} (GPU)`, numTokens, hiddenSize);
207
+ }
208
+
209
+ async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size, context, layerWeights) {
210
+ const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
211
+ if (!hasPerLayerInputBlock(config)) {
212
+ return hiddenTensor;
213
+ }
214
+
215
+ const hiddenSizePerLayerInput = Number(config.hiddenSizePerLayerInput);
216
+ const perLayerInputBuffer = context.perLayerInputBuffer ?? null;
217
+ if (!perLayerInputBuffer) {
218
+ throw new Error(
219
+ `Gemma 4 layer ${layerIdx} requires a per-layer input buffer, but context.perLayerInputBuffer was not set.`
220
+ );
221
+ }
222
+ if (!layerWeights?.perLayerInputGate || !layerWeights?.perLayerProjection || !layerWeights?.postPerLayerInputNorm) {
223
+ throw new Error(
224
+ `Gemma 4 layer ${layerIdx} is missing per-layer input weights. ` +
225
+ 'Expected per_layer_input_gate.weight, per_layer_projection.weight, and post_per_layer_input_norm.weight.'
226
+ );
227
+ }
228
+
229
+ const residualTensor = hiddenTensor;
230
+ let gateTensor = null;
231
+ let activatedTensor = null;
232
+ let projectedTensor = null;
233
+ let normalizedTensor = null;
234
+ let outputTensor = null;
235
+
236
+ try {
237
+ gateTensor = await processLayerPerLayerInputGate(
238
+ layerIdx,
239
+ hiddenTensor,
240
+ numTokens,
241
+ hiddenSizePerLayerInput,
242
+ context,
243
+ layerWeights
244
+ );
245
+ await runProbes('per_layer_input_gate', gateTensor.buffer, {
246
+ layerIdx,
247
+ numTokens,
248
+ hiddenSize: hiddenSizePerLayerInput,
249
+ probes: context.debugProbes,
250
+ recorder,
251
+ operatorDiagnostics: context.operatorDiagnostics,
252
+ dtype: gateTensor.dtype,
253
+ });
254
+ await debugLayerTensor(context, layerIdx, 'per-layer input gate', gateTensor, numTokens, hiddenSizePerLayerInput);
255
+
256
+ const perLayerInputTensor = createPerLayerInputTensor(
257
+ perLayerInputBuffer,
258
+ numTokens,
259
+ hiddenSizePerLayerInput,
260
+ hiddenTensor.dtype
261
+ );
262
+ activatedTensor = await doGeLU(perLayerInputTensor, {
263
+ size: numTokens * hiddenSizePerLayerInput,
264
+ gate: gateTensor,
265
+ label: `L${layerIdx}.per_layer_input_activation`,
266
+ layerIdx,
267
+ }, recorder);
268
+ await runProbes('per_layer_input_activation', activatedTensor.buffer, {
269
+ layerIdx,
270
+ numTokens,
271
+ hiddenSize: hiddenSizePerLayerInput,
272
+ probes: context.debugProbes,
273
+ recorder,
274
+ operatorDiagnostics: context.operatorDiagnostics,
275
+ dtype: activatedTensor.dtype,
276
+ });
277
+ await debugLayerTensor(context, layerIdx, 'per-layer input activation', activatedTensor, numTokens, hiddenSizePerLayerInput);
278
+ releaseOrTrack(recorder, gateTensor.buffer, decodeBuffers);
279
+ gateTensor = null;
280
+
281
+ projectedTensor = await processLayerPerLayerInputProjection(
282
+ layerIdx,
283
+ activatedTensor,
284
+ numTokens,
285
+ context,
286
+ layerWeights
287
+ );
288
+ await runProbes('per_layer_input_projection', projectedTensor.buffer, {
289
+ layerIdx,
290
+ numTokens,
291
+ hiddenSize: config.hiddenSize,
292
+ probes: context.debugProbes,
293
+ recorder,
294
+ operatorDiagnostics: context.operatorDiagnostics,
295
+ dtype: projectedTensor.dtype,
296
+ });
297
+ await debugLayerTensor(context, layerIdx, 'per-layer input projection', projectedTensor, numTokens, config.hiddenSize);
298
+ releaseOrTrack(recorder, activatedTensor.buffer, decodeBuffers);
299
+ activatedTensor = null;
300
+
301
+ const postNormWeight = getNormWeightBuffer(
302
+ layerWeights.postPerLayerInputNorm,
303
+ `L${layerIdx}.post_per_layer_input_norm`,
304
+ weightConfig,
305
+ debugFlags
306
+ );
307
+ normalizedTensor = await doRMSNorm(projectedTensor, postNormWeight, config.rmsNormEps, {
308
+ batchSize: numTokens,
309
+ hiddenSize: config.hiddenSize,
310
+ label: `L${layerIdx}.post_per_layer_input_norm`,
311
+ layerIdx,
312
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
313
+ }, recorder);
314
+ await runProbes('post_per_layer_input_norm', normalizedTensor.buffer, {
315
+ layerIdx,
316
+ numTokens,
317
+ hiddenSize: config.hiddenSize,
318
+ probes: context.debugProbes,
319
+ recorder,
320
+ operatorDiagnostics: context.operatorDiagnostics,
321
+ dtype: normalizedTensor.dtype,
322
+ });
323
+ await debugLayerTensor(context, layerIdx, 'post per-layer input norm', normalizedTensor, numTokens, config.hiddenSize);
324
+ if (!isGpuBufferInstance(layerWeights.postPerLayerInputNorm)) {
325
+ releaseOrTrack(recorder, postNormWeight, decodeBuffers);
326
+ }
327
+ releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
328
+ projectedTensor = null;
329
+
330
+ outputTensor = await doResidualAdd(normalizedTensor, residualTensor, size, recorder, {
331
+ label: `L${layerIdx}.per_layer_input_residual`,
332
+ layerIdx,
333
+ executionPolicies: context.executionPolicies ?? null,
334
+ });
335
+ releaseOrTrack(recorder, normalizedTensor.buffer, decodeBuffers);
336
+ normalizedTensor = null;
337
+
338
+ const layerScalar = resolveLayerScalarValue(layerWeights.layerScalar);
339
+ if (layerScalar !== 1) {
340
+ const scaledOutputTensor = recorder
341
+ ? await recordScale(recorder, outputTensor, layerScalar, {
342
+ count: size,
343
+ })
344
+ : await runScale(outputTensor, layerScalar, {
345
+ count: size,
346
+ });
347
+ releaseOrTrack(recorder, outputTensor.buffer, decodeBuffers);
348
+ outputTensor = scaledOutputTensor;
349
+ }
350
+
351
+ await runProbes('post_per_layer_input', outputTensor.buffer, {
352
+ layerIdx,
353
+ numTokens,
354
+ hiddenSize: config.hiddenSize,
355
+ probes: context.debugProbes,
356
+ recorder,
357
+ operatorDiagnostics: context.operatorDiagnostics,
358
+ dtype: outputTensor.dtype,
359
+ });
360
+ await debugLayerTensor(context, layerIdx, 'post per-layer input', outputTensor, numTokens, config.hiddenSize);
361
+
362
+ return outputTensor;
363
+ } catch (error) {
364
+ if (outputTensor?.buffer) releaseOrTrack(recorder, outputTensor.buffer, decodeBuffers);
365
+ if (normalizedTensor?.buffer) releaseOrTrack(recorder, normalizedTensor.buffer, decodeBuffers);
366
+ if (projectedTensor?.buffer) releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
367
+ if (activatedTensor?.buffer) releaseOrTrack(recorder, activatedTensor.buffer, decodeBuffers);
368
+ if (gateTensor?.buffer) releaseOrTrack(recorder, gateTensor.buffer, decodeBuffers);
369
+ throw error;
370
+ }
371
+ }
372
+
373
+ async function processLayerPerLayerInputGate(
374
+ layerIdx,
375
+ hiddenTensor,
376
+ numTokens,
377
+ hiddenSizePerLayerInput,
378
+ context,
379
+ layerWeights
380
+ ) {
381
+ return doMatmul(
382
+ hiddenTensor,
383
+ getWeightBuffer(layerWeights.perLayerInputGate, `L${layerIdx}.per_layer_input_gate`),
384
+ numTokens,
385
+ hiddenSizePerLayerInput,
386
+ context.config.hiddenSize,
387
+ {
388
+ transposeB: 'auto',
389
+ label: `L${layerIdx}.per_layer_input_gate`,
390
+ layerIdx,
391
+ kernelPath: context.kernelPath ?? null,
392
+ role: 'per_layer_input_gate',
393
+ outputDtype: hiddenTensor.dtype,
394
+ },
395
+ context.recorder
396
+ );
397
+ }
398
+
399
+ async function processLayerPerLayerInputProjection(
400
+ layerIdx,
401
+ inputTensor,
402
+ numTokens,
403
+ context,
404
+ layerWeights
405
+ ) {
406
+ const projectionWeight = resolveDensePleProjectionWeight(
407
+ getWeightBuffer(layerWeights.perLayerProjection, `L${layerIdx}.per_layer_projection`),
408
+ `L${layerIdx}.per_layer_projection`
409
+ );
410
+ return doMatmul(
411
+ inputTensor,
412
+ projectionWeight,
413
+ numTokens,
414
+ context.config.hiddenSize,
415
+ context.config.hiddenSizePerLayerInput,
416
+ {
417
+ transposeB: 'auto',
418
+ label: `L${layerIdx}.per_layer_projection`,
419
+ layerIdx,
420
+ kernelPath: context.kernelPath ?? null,
421
+ role: 'per_layer_projection',
422
+ outputDtype: inputTensor.dtype,
423
+ },
424
+ context.recorder
425
+ );
426
+ }
427
+
125
428
  // ============================================================================
126
429
  // Main Layer Processing
127
430
  // ============================================================================
@@ -137,11 +440,11 @@ export async function processLayer(layerIdx, hiddenStates, numTokens, isPrefill,
137
440
 
138
441
  // Debug: check path being taken for layer 0
139
442
  if (context.debug && layerIdx === 0) {
140
- trace.ffn(0, `routing: useGPU=${useGPU}, isGPUBuffer=${hiddenStates instanceof GPUBuffer}, constructor=${hiddenStates?.constructor?.name}`);
443
+ trace.ffn(0, `routing: useGPU=${useGPU}, isGPUBuffer=${isGpuBufferInstance(hiddenStates)}, constructor=${hiddenStates?.constructor?.name}`);
141
444
  }
142
445
 
143
446
  // GPU-native path
144
- if (useGPU && hiddenStates instanceof GPUBuffer) {
447
+ if (useGPU && isGpuBufferInstance(hiddenStates)) {
145
448
  return processLayerGPU(layerIdx, hiddenStates, numTokens, isPrefill, numTokens * hiddenSize, context);
146
449
  }
147
450
 
@@ -227,7 +530,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
227
530
  label: `L${layerIdx}.conv_input_norm`,
228
531
  layerIdx,
229
532
  }, recorder);
230
- if (!(inputNormWeight instanceof GPUBuffer)) releaseOrTrack(recorder, normWeightBuf);
533
+ if (!isGpuBufferInstance(inputNormWeight) && !isWeightBuffer(inputNormWeight)) releaseOrTrack(recorder, normWeightBuf);
231
534
  }
232
535
  attnOutput = await doConv(
233
536
  normedTensor,
@@ -261,14 +564,16 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
261
564
  getWeightBuffer: (weight, label) => getWeightBuffer(weight, label),
262
565
  getNormWeightBuffer: (weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags),
263
566
  debugProbes: context.debugProbes,
567
+ operatorDiagnostics: context.operatorDiagnostics,
264
568
  recorder: recorder ?? null,
265
569
  });
266
570
  } else {
267
571
  let attentionNumHeads = numHeads;
268
572
  let attentionNumKVHeads = numKVHeads;
269
- let attentionHeadDim = headDim;
573
+ let attentionHeadDim = resolveAttentionHeadDim(config, layerType);
270
574
  let disableRoPE = false;
271
575
  let queryKeyNorm = config.queryKeyNorm;
576
+ const { sharedKVSourceLayerIdx, storeSharedKV } = resolveAttentionKVSharing(config, layerIdx, layerType);
272
577
 
273
578
  const attnConfig = {
274
579
  layerIdx,
@@ -289,14 +594,20 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
289
594
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
290
595
  queryPreAttnScalar: config.queryPreAttnScalar,
291
596
  queryKeyNorm,
597
+ valueNorm: config.valueNorm,
292
598
  attentionOutputGate: config.attentionOutputGate,
293
599
  causalAttention: config.causalAttention,
600
+ multimodalBidirectionalSpan: isSlidingLayerType(layerType)
601
+ ? (context.multimodalBidirectionalSpan ?? null)
602
+ : null,
294
603
  rmsNormWeightOffset: config.rmsNormWeightOffset,
295
- ropeRotaryDim: config.ropeRotaryDim,
604
+ ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
296
605
  ropeInterleaved: config.ropeInterleaved,
297
606
  tokenIds: context.currentTokenIds ?? null,
298
607
  kernelPath: context.kernelPath ?? null,
299
608
  disableRoPE,
609
+ sharedKVSourceLayerIdx,
610
+ storeSharedKV,
300
611
  };
301
612
 
302
613
  validateAttnConfig(attnConfig, `L${layerIdx}`);
@@ -308,10 +619,13 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
308
619
  ropeFreqsSin: (isLocalLayer && context.ropeLocalSin)
309
620
  ? (context.ropeLocalSin)
310
621
  : (ropeFreqsSin),
622
+ sharedAttentionState: context.sharedAttentionState ?? null,
311
623
  kvCache: ((kvCache)),
312
624
  stats: context.stats,
313
625
  debugProbes: context.debugProbes,
626
+ operatorDiagnostics: context.operatorDiagnostics,
314
627
  linearRuntime: context.linearAttentionRuntime ?? null,
628
+ executionPolicies: context.executionPolicies ?? null,
315
629
  };
316
630
 
317
631
  const attnResult = await doAttention(
@@ -345,8 +659,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
345
659
  const stats = await getBufferStats(attnOutput.buffer);
346
660
  if (stats) logAttn(layerIdx, isPrefill, { numTokens, kvLen: context.currentSeqLen + (isPrefill ? numTokens : 1), maxAbsOut: stats.maxAbs });
347
661
 
348
- trace.attn(layerIdx, `attnOutput type check: isGPU=${attnOutput.buffer instanceof GPUBuffer}, type=${typeof attnOutput.buffer}, constructor=${attnOutput.buffer?.constructor?.name}, isPrefill=${isPrefill}`);
349
- if (shouldDebugLayerOutput(layerIdx, context.debugLayers) && attnOutput.buffer instanceof GPUBuffer && !recorder) {
662
+ trace.attn(layerIdx, `attnOutput type check: isGPU=${isGpuBufferInstance(attnOutput.buffer)}, type=${typeof attnOutput.buffer}, constructor=${attnOutput.buffer?.constructor?.name}, isPrefill=${isPrefill}`);
663
+ if (shouldDebugLayerOutput(layerIdx, context.debugLayers) && isGpuBufferInstance(attnOutput.buffer) && !recorder) {
350
664
  if (allowReadback(`layer.attn-out.${layerIdx}`)) {
351
665
  try {
352
666
  const sampleSize = Math.min(128, attnOutput.buffer.size);
@@ -362,7 +676,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
362
676
  trace.attn(layerIdx, `ATTN_OUT error: ${e}`);
363
677
  }
364
678
  }
365
- } else if (shouldDebugLayerOutput(layerIdx, context.debugLayers) && attnOutput.buffer instanceof GPUBuffer && recorder) {
679
+ } else if (shouldDebugLayerOutput(layerIdx, context.debugLayers) && isGpuBufferInstance(attnOutput.buffer) && recorder) {
366
680
  trace.attn(layerIdx, `ATTN_OUT: (skipped - using batched recorder, values not available until submit)`);
367
681
  }
368
682
  }
@@ -372,6 +686,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
372
686
  hiddenSize,
373
687
  probes: context.debugProbes,
374
688
  recorder,
689
+ operatorDiagnostics: context.operatorDiagnostics,
375
690
  dtype: attnOutput.dtype,
376
691
  });
377
692
 
@@ -388,7 +703,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
388
703
  layerIdx,
389
704
  rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
390
705
  }, recorder);
391
- if (!(layerWeights.postAttentionNorm instanceof GPUBuffer)) releaseOrTrack(recorder, normWeightBuf);
706
+ if (!isGpuBufferInstance(layerWeights.postAttentionNorm) && !isWeightBuffer(layerWeights.postAttentionNorm)) releaseOrTrack(recorder, normWeightBuf);
392
707
  if (recorder) {
393
708
  recorder.trackTemporaryBuffer(attnOutput.buffer);
394
709
  } else {
@@ -397,16 +712,21 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
397
712
  }
398
713
  } else if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
399
714
  const normWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
400
- postAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
715
+ const normalizedAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
401
716
  batchSize: numTokens,
402
717
  hiddenSize,
403
- residual: inputTensor,
404
718
  label: `L${layerIdx}.post_attn_norm`,
405
719
  layerIdx,
406
720
  rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
407
721
  }, recorder);
722
+ postAttn = await doResidualAdd(normalizedAttn, inputTensor, size, recorder, {
723
+ label: `L${layerIdx}.post_attn_residual`,
724
+ layerIdx,
725
+ executionPolicies: context.executionPolicies ?? null,
726
+ });
408
727
 
409
- if (!(layerWeights.postAttentionNorm instanceof GPUBuffer)) releaseOrTrack(recorder, normWeightBuf);
728
+ if (!isGpuBufferInstance(layerWeights.postAttentionNorm) && !isWeightBuffer(layerWeights.postAttentionNorm)) releaseOrTrack(recorder, normWeightBuf);
729
+ releaseOrTrack(recorder, normalizedAttn.buffer, context.decodeBuffers);
410
730
  if (recorder) {
411
731
  recorder.trackTemporaryBuffer(attnOutput.buffer);
412
732
  } else {
@@ -419,7 +739,11 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
419
739
  postAttn = attnOutput;
420
740
  fusedResidualForFFN = inputTensor;
421
741
  } else {
422
- postAttn = await doResidualAdd(attnOutput, inputTensor, size, recorder, { label: `L${layerIdx}.post_attn_residual`, layerIdx });
742
+ postAttn = await doResidualAdd(attnOutput, inputTensor, size, recorder, {
743
+ label: `L${layerIdx}.post_attn_residual`,
744
+ layerIdx,
745
+ executionPolicies: context.executionPolicies ?? null,
746
+ });
423
747
  if (recorder) {
424
748
  recorder.trackTemporaryBuffer(attnOutput.buffer);
425
749
  } else {
@@ -442,6 +766,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
442
766
  hiddenSize,
443
767
  probes: context.debugProbes,
444
768
  recorder,
769
+ operatorDiagnostics: context.operatorDiagnostics,
445
770
  dtype: postAttn.dtype,
446
771
  });
447
772
 
@@ -454,6 +779,21 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
454
779
  outputTensor = await processFFNStandard(layerIdx, postAttn, numTokens, size, context, layerWeights, fusedResidualForFFN);
455
780
  }
456
781
 
782
+ if (hasPerLayerInputBlock(config)) {
783
+ const outputWithPerLayerInput = await applyPerLayerInputBlock(
784
+ layerIdx,
785
+ outputTensor,
786
+ numTokens,
787
+ size,
788
+ context,
789
+ layerWeights
790
+ );
791
+ if (outputWithPerLayerInput.buffer !== outputTensor.buffer) {
792
+ releaseOrTrack(recorder, outputTensor.buffer, context.decodeBuffers);
793
+ }
794
+ outputTensor = outputWithPerLayerInput;
795
+ }
796
+
457
797
  // Keep activation dtype consistent across layers. Some FFN paths can emit f32
458
798
  // tensors even when the execution plan is f16; leaving that unnormalized causes
459
799
  // downstream kernels to decode the buffer with the wrong dtype contract.
@@ -466,6 +806,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
466
806
  );
467
807
  releaseOrTrack(recorder, outputTensor.buffer, context.decodeBuffers);
468
808
  }
809
+ await debugLayerTensor(context, layerIdx, 'final layer output', finalOutput, numTokens, hiddenSize);
469
810
 
470
811
  // Early-stop check for F16 NaN/Infinity bounds
471
812
  const computeConfig = context.runtimeComputeConfig ?? null;
@@ -547,8 +888,11 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
547
888
  ropeFreqsSin: (isLocalLayer && context.ropeLocalSin)
548
889
  ? (context.ropeLocalSin)
549
890
  : (ropeFreqsSin),
891
+ sharedAttentionState: context.sharedAttentionState ?? null,
550
892
  kvCache: ((kvCache)),
551
893
  linearRuntime: context.linearAttentionRuntime ?? null,
894
+ operatorDiagnostics: context.operatorDiagnostics,
895
+ executionPolicies: context.executionPolicies ?? null,
552
896
  };
553
897
 
554
898
  const allowResidualFuse = numTokens === 1 && !(sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm);
@@ -696,7 +1040,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
696
1040
  isPrefill,
697
1041
  numHeads,
698
1042
  numKVHeads,
699
- headDim,
1043
+ headDim: resolveAttentionHeadDim(config, layerType),
700
1044
  hiddenSize,
701
1045
  rmsNormEps,
702
1046
  currentSeqLen: context.currentSeqLen,
@@ -706,15 +1050,20 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
706
1050
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
707
1051
  queryPreAttnScalar: config.queryPreAttnScalar,
708
1052
  queryKeyNorm: config.queryKeyNorm,
1053
+ valueNorm: config.valueNorm,
709
1054
  attentionOutputGate: config.attentionOutputGate,
710
1055
  causalAttention: config.causalAttention,
711
1056
  rmsNormWeightOffset: config.rmsNormWeightOffset,
712
- ropeRotaryDim: config.ropeRotaryDim,
1057
+ ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
713
1058
  ropeInterleaved: config.ropeInterleaved,
714
1059
  tokenIds: context.currentTokenIds ?? null,
715
1060
  skipInputNorm: step.skipInputNorm === true,
716
1061
  activationDtype,
1062
+ inputDtype: step.inputDtype ?? undefined,
1063
+ outputDtype: step.outputDtype ?? undefined,
1064
+ kvDtype: step.kvDtype ?? undefined,
717
1065
  kernelPath: context.kernelPath ?? null,
1066
+ ...resolveAttentionKVSharing(config, layerIdx, layerType),
718
1067
  };
719
1068
 
720
1069
  const result = await doAttention(
@@ -740,6 +1089,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
740
1089
  hiddenSize,
741
1090
  probes: context.debugProbes,
742
1091
  recorder,
1092
+ operatorDiagnostics: context.operatorDiagnostics,
743
1093
  dtype: outputDtype,
744
1094
  });
745
1095
  }
@@ -772,6 +1122,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
772
1122
  label: `L${layerIdx}.plan_conv`,
773
1123
  swigluLimit: config.swigluLimit,
774
1124
  kernelPath: context.kernelPath ?? null,
1125
+ executionPolicies: context.executionPolicies ?? null,
775
1126
  convState: getConvLayerState(context.convLayerStates, layerIdx),
776
1127
  },
777
1128
  recorder
@@ -785,6 +1136,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
785
1136
  hiddenSize,
786
1137
  probes: context.debugProbes,
787
1138
  recorder,
1139
+ operatorDiagnostics: context.operatorDiagnostics,
788
1140
  dtype: outputDtype,
789
1141
  });
790
1142
  }
@@ -810,7 +1162,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
810
1162
  layerIdx,
811
1163
  rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
812
1164
  }, recorder);
813
- if (!(weight instanceof GPUBuffer)) releaseOrTrack(recorder, normWeightBuf);
1165
+ if (!isGpuBufferInstance(weight) && !isWeightBuffer(weight)) releaseOrTrack(recorder, normWeightBuf);
814
1166
  const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
815
1167
  setSlot(step.dst, outputTensor.buffer, outputDtype);
816
1168
  if (step.probeStage) {
@@ -820,6 +1172,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
820
1172
  hiddenSize,
821
1173
  probes: context.debugProbes,
822
1174
  recorder,
1175
+ operatorDiagnostics: context.operatorDiagnostics,
823
1176
  dtype: outputDtype,
824
1177
  });
825
1178
  }
@@ -830,6 +1183,15 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
830
1183
 
831
1184
  const activationDtype = resolveStepInputDtype(step, step.src);
832
1185
  const srcTensor = createTensor(srcBuf, activationDtype, [numTokens, hiddenSize], 'plan_ffn_src');
1186
+ const ffnContext = step.inputDtype || step.outputDtype
1187
+ ? {
1188
+ ...context,
1189
+ ffnStepPrecision: {
1190
+ inputDtype: step.inputDtype ?? null,
1191
+ outputDtype: step.outputDtype ?? null,
1192
+ },
1193
+ }
1194
+ : context;
833
1195
 
834
1196
  let outputTensor;
835
1197
  const { runMoEFFNGPU, runDenseFFNGPU } = await import('./ffn/index.js');
@@ -842,9 +1204,9 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
842
1204
  { variant: step.variant, canAutoMoe }
843
1205
  );
844
1206
  if (useMoe) {
845
- outputTensor = await runMoEFFNGPU(layerIdx, srcTensor, numTokens, context);
1207
+ outputTensor = await runMoEFFNGPU(layerIdx, srcTensor, numTokens, ffnContext);
846
1208
  } else {
847
- outputTensor = await runDenseFFNGPU(layerIdx, srcTensor, numTokens, context, layerWeights);
1209
+ outputTensor = await runDenseFFNGPU(layerIdx, srcTensor, numTokens, ffnContext, layerWeights);
848
1210
  }
849
1211
  const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
850
1212
  setSlot(step.dst, outputTensor.buffer, outputDtype);
@@ -855,6 +1217,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
855
1217
  hiddenSize,
856
1218
  probes: context.debugProbes,
857
1219
  recorder,
1220
+ operatorDiagnostics: context.operatorDiagnostics,
858
1221
  dtype: outputDtype,
859
1222
  });
860
1223
  }
@@ -870,6 +1233,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
870
1233
  const outputTensor = await doResidualAdd(aTensor, bTensor, size, recorder, {
871
1234
  label: `L${layerIdx}.residual_add`,
872
1235
  layerIdx,
1236
+ executionPolicies: context.executionPolicies ?? null,
873
1237
  });
874
1238
  const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
875
1239
  setSlot(step.dst, outputTensor.buffer, outputDtype);
@@ -880,6 +1244,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
880
1244
  hiddenSize,
881
1245
  probes: context.debugProbes,
882
1246
  recorder,
1247
+ operatorDiagnostics: context.operatorDiagnostics,
883
1248
  dtype: outputDtype,
884
1249
  });
885
1250
  }
@@ -907,6 +1272,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
907
1272
  hiddenSize,
908
1273
  probes: context.debugProbes,
909
1274
  recorder,
1275
+ operatorDiagnostics: context.operatorDiagnostics,
910
1276
  dtype: toDtype,
911
1277
  });
912
1278
  }
@@ -925,6 +1291,22 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
925
1291
  clearSlot(name);
926
1292
  }
927
1293
  }
1294
+
1295
+ if (hasPerLayerInputBlock(config)) {
1296
+ const stateBuffer = getSlot('state');
1297
+ const stateDtype = getSlotDtype('state') ?? activationDtype;
1298
+ const stateTensor = createTensor(stateBuffer, stateDtype, [numTokens, hiddenSize], 'plan_state');
1299
+ const outputTensor = await applyPerLayerInputBlock(
1300
+ layerIdx,
1301
+ stateTensor,
1302
+ numTokens,
1303
+ size,
1304
+ context,
1305
+ layerWeights
1306
+ );
1307
+ const outputDtype = resolveActivationDtype(outputTensor.dtype);
1308
+ setSlot('state', outputTensor.buffer, outputDtype);
1309
+ }
928
1310
  } catch (err) {
929
1311
  cleanupSlots();
930
1312
  throw err;
@@ -937,6 +1319,7 @@ async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill,
937
1319
  hiddenSize,
938
1320
  probes: context.debugProbes,
939
1321
  recorder,
1322
+ operatorDiagnostics: context.operatorDiagnostics,
940
1323
  dtype: getSlotDtype('state') ?? activationDtype,
941
1324
  });
942
1325