doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -6,23 +6,140 @@ import { runMatmul, runRMSNorm } from '../../../../gpu/kernel-selector.js';
6
6
  import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
7
7
  import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
8
8
  import { createTensor } from '../../../../gpu/tensor.js';
9
- import { castF16ToF32, castF32ToF16, recordCastF16ToF32 } from '../../../../gpu/kernels/cast.js';
9
+ import {
10
+ castF16ToF32,
11
+ castF32ToF16,
12
+ recordCastF16ToF32,
13
+ recordCastF32ToF16,
14
+ } from '../../../../gpu/kernels/cast.js';
10
15
  import { createWeightBuffer, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../../gpu/weight-buffer.js';
11
16
  import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
12
17
  import { getRuntimeConfig } from '../../../../config/runtime.js';
18
+ import { getKernelPathMatmulPrecision, getKernelPathStepPrecision } from '../../../../config/kernel-path-loader.js';
13
19
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
14
20
  import { runProbes } from '../probes.js';
21
+ import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
15
22
  import { f16BufferToF32 } from './cpu.js';
16
23
  import { readBufferWithCleanup } from './utils.js';
17
24
 
18
25
  function shouldForceStableF32Logits(config, inputDtype) {
19
- // Small Gemma-family checkpoints can overflow in pure F16 logits path after RMSNorm offset.
20
- return inputDtype === 'f16'
21
- && config.rmsNormWeightOffset === true
26
+ if (inputDtype !== 'f16') {
27
+ return false;
28
+ }
29
+ // Softcapped output heads are numerically sensitive in pure F16 on the
30
+ // final RMSNorm + LM-head path. Widen only the logits tail so the main
31
+ // layer stack and KV cache can stay on the faster F16 lane.
32
+ if (Number.isFinite(config.finalLogitSoftcapping) && config.finalLogitSoftcapping > 0) {
33
+ return true;
34
+ }
35
+ // Small Gemma-family checkpoints can also overflow in pure F16 logits path
36
+ // after RMSNorm offset even without output softcapping.
37
+ return config.rmsNormWeightOffset === true
22
38
  && Number.isFinite(config.hiddenSize)
23
39
  && config.hiddenSize <= 768;
24
40
  }
25
41
 
42
+ function resolvePrecisionFieldDtype(precision, fallback, field) {
43
+ const requested = precision?.[field] ?? fallback;
44
+ if (requested == null) {
45
+ return fallback;
46
+ }
47
+ return selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: requested });
48
+ }
49
+
50
+ function resolveMatmulStepDtype(role, phase, kernelPath, fallback, field) {
51
+ const precision = getKernelPathMatmulPrecision(role, phase, 0, kernelPath);
52
+ return resolvePrecisionFieldDtype(precision, fallback, field);
53
+ }
54
+
55
+ function resolvePostLayerStepDtype(op, phase, kernelPath, fallback, field) {
56
+ const precision = getKernelPathStepPrecision(op, 'postLayer', phase, 0, kernelPath);
57
+ return resolvePrecisionFieldDtype(precision, fallback, field);
58
+ }
59
+
60
+ async function coerceTensorDtype(tensor, targetDtype, recorder = null, options = {}) {
61
+ if (!targetDtype || tensor.dtype === targetDtype) {
62
+ return tensor;
63
+ }
64
+ assertImplicitDtypeTransitionAllowed({
65
+ executionPolicies: options.executionPolicies ?? null,
66
+ fromDtype: tensor.dtype,
67
+ toDtype: targetDtype,
68
+ op: options.op ?? 'logits',
69
+ detail: 'The execution graph must declare this cast explicitly.',
70
+ transitionDeclaredBy: options.transitionDeclaredBy ?? null,
71
+ });
72
+ if (tensor.dtype === 'f32' && targetDtype === 'f16') {
73
+ return recorder ? await recordCastF32ToF16(recorder, tensor) : await castF32ToF16(tensor);
74
+ }
75
+ if (tensor.dtype === 'f16' && targetDtype === 'f32') {
76
+ return recorder ? await recordCastF16ToF32(recorder, tensor) : await castF16ToF32(tensor);
77
+ }
78
+ throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
79
+ }
80
+
81
+ const STABLE_F32_LOGITS_KERNEL_MAP = new Map([
82
+ ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
83
+ ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
84
+ ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
85
+ ]);
86
+
87
+ function createStableF32LogitsKernelPath(kernelPath) {
88
+ if (!kernelPath?.postLayer) {
89
+ return kernelPath;
90
+ }
91
+ let changed = false;
92
+ const postLayer = kernelPath.postLayer.map((step) => {
93
+ if (step?.op === 'final_norm') {
94
+ const precision = {
95
+ ...(step.precision ?? {}),
96
+ inputDtype: 'f32',
97
+ outputDtype: 'f32',
98
+ };
99
+ if (
100
+ step.precision?.inputDtype === precision.inputDtype
101
+ && step.precision?.outputDtype === precision.outputDtype
102
+ ) {
103
+ return step;
104
+ }
105
+ changed = true;
106
+ return {
107
+ ...step,
108
+ precision,
109
+ };
110
+ }
111
+ if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
112
+ return step;
113
+ }
114
+ const replacement = STABLE_F32_LOGITS_KERNEL_MAP.get(step.kernel) ?? step.kernel;
115
+ const precision = {
116
+ ...(step.precision ?? {}),
117
+ inputDtype: 'f32',
118
+ outputDtype: 'f32',
119
+ };
120
+ if (
121
+ replacement === step.kernel
122
+ && step.precision?.inputDtype === precision.inputDtype
123
+ && step.precision?.outputDtype === precision.outputDtype
124
+ ) {
125
+ return step;
126
+ }
127
+ changed = true;
128
+ return {
129
+ ...step,
130
+ kernel: replacement,
131
+ precision,
132
+ };
133
+ });
134
+ if (!changed) {
135
+ return kernelPath;
136
+ }
137
+ return {
138
+ ...kernelPath,
139
+ postLayer,
140
+ };
141
+ }
142
+
26
143
 
27
144
  export function resolveCpuWeightDims(lmHead) {
28
145
  if (lmHead.shape.length !== 2) {
@@ -114,8 +231,10 @@ export async function computeChunkedLogitsGPU(
114
231
  vocabSize,
115
232
  weightVocabSize,
116
233
  debugProbes,
234
+ operatorDiagnostics,
117
235
  largeWeightConfig,
118
- kernelPath = null
236
+ kernelPath = null,
237
+ executionPolicies = null
119
238
  ) {
120
239
  const device = getDevice();
121
240
  if (!device) {
@@ -126,6 +245,9 @@ export async function computeChunkedLogitsGPU(
126
245
  }
127
246
 
128
247
  const chunkRows = resolveLmHeadChunkRows(device, numTokens, hiddenSize, largeWeightConfig);
248
+ const phase = numTokens === 1 ? 'decode' : 'prefill';
249
+ const lmHeadInputDtype = resolveMatmulStepDtype('lm_head', phase, kernelPath, normedTensor.dtype, 'inputDtype');
250
+ const lmHeadOutputDtype = resolveMatmulStepDtype('lm_head', phase, kernelPath, normedTensor.dtype, 'outputDtype');
129
251
  const caps = getKernelCapabilities();
130
252
  const weightDtype = selectRuleValue('inference', 'dtype', 'lmHeadChunkWeightDtype', {
131
253
  preferF16: largeWeightConfig.preferF16,
@@ -139,6 +261,14 @@ export async function computeChunkedLogitsGPU(
139
261
  trace.logits(`LM_HEAD_CHUNKED: vocab=${vocabSize}, chunkRows=${chunkRows}, layout=${lmHead.layout}, f16=${preferF16}`);
140
262
  }
141
263
 
264
+ const matmulInput = lmHeadInputDtype !== normedTensor.dtype
265
+ ? await coerceTensorDtype(normedTensor, lmHeadInputDtype, null, {
266
+ executionPolicies,
267
+ op: 'lm_head',
268
+ transitionDeclaredBy: 'step_precision',
269
+ })
270
+ : normedTensor;
271
+
142
272
  for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
143
273
  const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
144
274
  const chunkData = extractLmHeadChunk(
@@ -172,17 +302,20 @@ export async function computeChunkedLogitsGPU(
172
302
  weightBuffer = createWeightBuffer(f16Tensor.buffer, 'f16', lmHead.layout, chunkShape, 'lm_head_chunk_f16');
173
303
  }
174
304
 
175
- const logitsTensor = await runMatmul(normedTensor, weightBuffer, numTokens, rowCount, hiddenSize, {
305
+ const logitsTensor = await runMatmul(matmulInput, weightBuffer, numTokens, rowCount, hiddenSize, {
176
306
  transposeB: 'auto',
177
307
  role: 'lm_head',
178
308
  kernelPath,
309
+ outputDtype: lmHeadOutputDtype,
310
+ executionPolicies,
179
311
  });
180
312
 
181
- if (debugProbes?.length) {
313
+ if (debugProbes?.length || operatorDiagnostics?.enabled) {
182
314
  await runProbes('logits', logitsTensor.buffer, {
183
315
  numTokens,
184
316
  hiddenSize: rowCount,
185
317
  probes: debugProbes,
318
+ operatorDiagnostics,
186
319
  dtype: logitsTensor.dtype,
187
320
  });
188
321
  }
@@ -202,6 +335,10 @@ export async function computeChunkedLogitsGPU(
202
335
  writeChunkLogits(logits, chunkLogits, numTokens, vocabSize, rowOffset, rowCount);
203
336
  }
204
337
 
338
+ if (matmulInput !== normedTensor) {
339
+ releaseBuffer(matmulInput.buffer);
340
+ }
341
+
205
342
  return logits;
206
343
  }
207
344
 
@@ -212,6 +349,7 @@ export async function computeLogitsGPU(
212
349
  weights,
213
350
  config,
214
351
  debugFlags,
352
+ operatorDiagnostics = null,
215
353
  ) {
216
354
  const {
217
355
  hiddenSize,
@@ -248,6 +386,9 @@ export async function computeLogitsGPU(
248
386
  let normInputTensor;
249
387
  let normInputOwned = false;
250
388
  let normedTensor;
389
+ let finalNormTensor;
390
+ let lmHeadInputTensor;
391
+ let lmHeadInputOwned = false;
251
392
  let lmHeadBuffer;
252
393
  let lmHeadBufferOwned = false;
253
394
 
@@ -271,21 +412,92 @@ export async function computeLogitsGPU(
271
412
 
272
413
  const inputDtype = isGpuBufferInstance(hiddenStates) ? activationDtype : 'f32';
273
414
  const inputTensor = createTensor(inputBuffer, inputDtype, [numTokens, hiddenSize], 'logits_input');
274
- const forceStableF32Logits = shouldForceStableF32Logits(config, inputDtype);
415
+ const phase = numTokens === 1 ? 'decode' : 'prefill';
416
+ const kernelPath = config.kernelPath ?? null;
417
+ const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
418
+ const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
419
+ await runProbes('pre_final_norm', inputBuffer, {
420
+ numTokens,
421
+ hiddenSize,
422
+ operatorDiagnostics,
423
+ dtype: inputDtype,
424
+ });
425
+ const forceStableF32Logits = !hasExplicitFinalNormPrecision && shouldForceStableF32Logits(config, inputDtype);
426
+ const stableKernelPath = forceStableF32Logits
427
+ ? createStableF32LogitsKernelPath(kernelPath)
428
+ : kernelPath;
275
429
  normInputTensor = inputTensor;
276
430
  if (forceStableF32Logits) {
431
+ assertImplicitDtypeTransitionAllowed({
432
+ executionPolicies: config.executionPolicies ?? null,
433
+ fromDtype: inputTensor.dtype,
434
+ toDtype: 'f32',
435
+ op: 'logits_final_norm',
436
+ detail: 'Stable logits mode would widen activations implicitly before final RMSNorm.',
437
+ });
277
438
  normInputTensor = await castF16ToF32(inputTensor);
278
439
  normInputOwned = true;
440
+ } else {
441
+ const finalNormInputDtype = resolvePostLayerStepDtype('final_norm', phase, stableKernelPath, inputTensor.dtype, 'inputDtype');
442
+ normInputTensor = finalNormInputDtype !== inputTensor.dtype
443
+ ? await coerceTensorDtype(inputTensor, finalNormInputDtype, null, {
444
+ executionPolicies: config.executionPolicies ?? null,
445
+ op: 'final_norm',
446
+ transitionDeclaredBy: 'step_precision',
447
+ })
448
+ : inputTensor;
449
+ normInputOwned = normInputTensor !== inputTensor;
279
450
  }
280
451
  normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
281
452
  batchSize: numTokens,
282
453
  hiddenSize,
283
454
  rmsNormWeightOffset: config.rmsNormWeightOffset,
284
455
  });
456
+ finalNormTensor = normedTensor;
457
+ if (!forceStableF32Logits) {
458
+ const finalNormOutputDtype = resolvePostLayerStepDtype(
459
+ 'final_norm',
460
+ phase,
461
+ stableKernelPath,
462
+ normedTensor.dtype,
463
+ 'outputDtype'
464
+ );
465
+ finalNormTensor = finalNormOutputDtype !== normedTensor.dtype
466
+ ? await coerceTensorDtype(normedTensor, finalNormOutputDtype, null, {
467
+ executionPolicies: config.executionPolicies ?? null,
468
+ op: 'final_norm',
469
+ transitionDeclaredBy: 'step_precision',
470
+ })
471
+ : normedTensor;
472
+ }
473
+ if (finalNormTensor !== normedTensor) {
474
+ releaseBuffer(normedTensor.buffer);
475
+ normedTensor = null;
476
+ }
477
+ await runProbes('final_norm', finalNormTensor.buffer, {
478
+ numTokens,
479
+ hiddenSize,
480
+ operatorDiagnostics,
481
+ dtype: finalNormTensor.dtype,
482
+ });
285
483
  if (normInputOwned) {
286
484
  releaseBuffer(normInputTensor.buffer);
287
485
  normInputOwned = false;
288
486
  }
487
+ const lmHeadInputDtype = forceStableF32Logits
488
+ ? finalNormTensor.dtype
489
+ : resolveMatmulStepDtype('lm_head', phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
490
+ const lmHeadOutputDtype = forceStableF32Logits
491
+ ? finalNormTensor.dtype
492
+ : resolveMatmulStepDtype('lm_head', phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
493
+ lmHeadInputTensor = lmHeadInputDtype !== finalNormTensor.dtype
494
+ ? await coerceTensorDtype(finalNormTensor, lmHeadInputDtype, null, {
495
+ executionPolicies: config.executionPolicies ?? null,
496
+ op: 'lm_head',
497
+ transitionDeclaredBy: 'step_precision',
498
+ })
499
+ : finalNormTensor;
500
+ lmHeadInputOwned = lmHeadInputTensor !== finalNormTensor;
289
501
 
290
502
  // Project to vocab via LM head
291
503
  if (isGpuBufferInstance(lmHead)) {
@@ -303,15 +515,28 @@ export async function computeLogitsGPU(
303
515
  ? embeddingVocabSize
304
516
  : vocabSize;
305
517
 
306
- const logitsTensor = await runMatmul(normedTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
518
+ const logitsTensor = await runMatmul(lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
307
519
  transposeB: 'auto',
308
520
  role: 'lm_head',
309
- kernelPath: config.kernelPath ?? null,
521
+ kernelPath: stableKernelPath,
522
+ outputDtype: lmHeadOutputDtype,
523
+ executionPolicies: config.executionPolicies ?? null,
524
+ });
525
+ await runProbes('logits', logitsTensor.buffer, {
526
+ numTokens,
527
+ hiddenSize: matmulVocabSize,
528
+ operatorDiagnostics,
529
+ dtype: logitsTensor.dtype,
310
530
  });
311
531
 
312
532
  // Cleanup intermediate buffers (but keep logitsBuffer)
313
533
  if (inputBufferOwned) { releaseBuffer(inputBuffer); inputBufferOwned = false; }
314
- releaseBuffer(normedTensor.buffer); normedTensor = null;
534
+ if (lmHeadInputOwned) { releaseBuffer(lmHeadInputTensor.buffer); lmHeadInputOwned = false; }
535
+ if (finalNormTensor) {
536
+ releaseBuffer(finalNormTensor.buffer);
537
+ finalNormTensor = null;
538
+ normedTensor = null;
539
+ }
315
540
  if (normWeightBufferOwned) { releaseBuffer(normWeightBuffer); normWeightBufferOwned = false; }
316
541
  if (lmHeadBufferOwned) { releaseBuffer(isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer); lmHeadBufferOwned = false; }
317
542
 
@@ -319,6 +544,8 @@ export async function computeLogitsGPU(
319
544
  } finally {
320
545
  if (inputBufferOwned && inputBuffer) releaseBuffer(inputBuffer);
321
546
  if (normInputOwned && normInputTensor) releaseBuffer(normInputTensor.buffer);
547
+ if (lmHeadInputOwned && lmHeadInputTensor) releaseBuffer(lmHeadInputTensor.buffer);
548
+ if (finalNormTensor) releaseBuffer(finalNormTensor.buffer);
322
549
  if (normedTensor) releaseBuffer(normedTensor.buffer);
323
550
  if (normWeightBufferOwned && normWeightBuffer) releaseBuffer(normWeightBuffer);
324
551
  if (lmHeadBufferOwned && lmHeadBuffer) releaseBuffer(isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer);
@@ -332,6 +559,7 @@ export async function recordLogitsGPU(
332
559
  numTokens,
333
560
  weights,
334
561
  config,
562
+ operatorDiagnostics = null,
335
563
  ) {
336
564
  const {
337
565
  hiddenSize,
@@ -367,12 +595,43 @@ export async function recordLogitsGPU(
367
595
  const inputDtype = activationDtype;
368
596
  // Wrap input buffer as Tensor for RMSNorm
369
597
  const inputTensor = createTensor(hiddenStates, inputDtype, [numTokens, hiddenSize], 'logits_input');
370
- const forceStableF32Logits = shouldForceStableF32Logits(config, inputDtype);
598
+ const phase = numTokens === 1 ? 'decode' : 'prefill';
599
+ const kernelPath = config.kernelPath ?? null;
600
+ const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
601
+ const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
602
+ await runProbes('pre_final_norm', hiddenStates, {
603
+ numTokens,
604
+ hiddenSize,
605
+ recorder,
606
+ operatorDiagnostics,
607
+ dtype: inputDtype,
608
+ });
609
+ const forceStableF32Logits = !hasExplicitFinalNormPrecision && shouldForceStableF32Logits(config, inputDtype);
610
+ const stableKernelPath = forceStableF32Logits
611
+ ? createStableF32LogitsKernelPath(kernelPath)
612
+ : kernelPath;
371
613
  let normInputTensor = inputTensor;
372
614
  let normInputOwned = false;
373
615
  if (forceStableF32Logits) {
616
+ assertImplicitDtypeTransitionAllowed({
617
+ executionPolicies: config.executionPolicies ?? null,
618
+ fromDtype: inputTensor.dtype,
619
+ toDtype: 'f32',
620
+ op: 'logits_final_norm',
621
+ detail: 'Stable logits mode would widen activations implicitly before final RMSNorm.',
622
+ });
374
623
  normInputTensor = await recordCastF16ToF32(recorder, inputTensor);
375
624
  normInputOwned = true;
625
+ } else {
626
+ const finalNormInputDtype = resolvePostLayerStepDtype('final_norm', phase, stableKernelPath, inputTensor.dtype, 'inputDtype');
627
+ normInputTensor = finalNormInputDtype !== inputTensor.dtype
628
+ ? await coerceTensorDtype(inputTensor, finalNormInputDtype, recorder, {
629
+ executionPolicies: config.executionPolicies ?? null,
630
+ op: 'final_norm',
631
+ transitionDeclaredBy: 'step_precision',
632
+ })
633
+ : inputTensor;
634
+ normInputOwned = normInputTensor !== inputTensor;
376
635
  }
377
636
  // Record RMSNorm (no submit)
378
637
  const normedTensor = await recordRMSNorm(recorder, normInputTensor, normWeightBuffer, rmsNormEps, {
@@ -380,6 +639,43 @@ export async function recordLogitsGPU(
380
639
  hiddenSize,
381
640
  rmsNormWeightOffset: config.rmsNormWeightOffset,
382
641
  });
642
+ let finalNormTensor = normedTensor;
643
+ if (!forceStableF32Logits) {
644
+ const finalNormOutputDtype = resolvePostLayerStepDtype(
645
+ 'final_norm',
646
+ phase,
647
+ stableKernelPath,
648
+ normedTensor.dtype,
649
+ 'outputDtype'
650
+ );
651
+ finalNormTensor = finalNormOutputDtype !== normedTensor.dtype
652
+ ? await coerceTensorDtype(normedTensor, finalNormOutputDtype, recorder, {
653
+ executionPolicies: config.executionPolicies ?? null,
654
+ op: 'final_norm',
655
+ transitionDeclaredBy: 'step_precision',
656
+ })
657
+ : normedTensor;
658
+ }
659
+ await runProbes('final_norm', finalNormTensor.buffer, {
660
+ numTokens,
661
+ hiddenSize,
662
+ recorder,
663
+ operatorDiagnostics,
664
+ dtype: finalNormTensor.dtype,
665
+ });
666
+ const lmHeadInputDtype = forceStableF32Logits
667
+ ? finalNormTensor.dtype
668
+ : resolveMatmulStepDtype('lm_head', phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
669
+ const lmHeadOutputDtype = forceStableF32Logits
670
+ ? finalNormTensor.dtype
671
+ : resolveMatmulStepDtype('lm_head', phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
672
+ const lmHeadInputTensor = lmHeadInputDtype !== finalNormTensor.dtype
673
+ ? await coerceTensorDtype(finalNormTensor, lmHeadInputDtype, recorder, {
674
+ executionPolicies: config.executionPolicies ?? null,
675
+ op: 'lm_head',
676
+ transitionDeclaredBy: 'step_precision',
677
+ })
678
+ : finalNormTensor;
383
679
 
384
680
  // Get LM head buffer
385
681
 
@@ -397,14 +693,37 @@ export async function recordLogitsGPU(
397
693
  }
398
694
 
399
695
  // Record matmul (no submit)
400
- const logitsTensor = await recordMatmul(recorder, normedTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
696
+ const logitsTensor = await recordMatmul(recorder, lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
401
697
  transposeB: 'auto',
402
698
  role: 'lm_head',
403
- kernelPath: config.kernelPath ?? null,
699
+ kernelPath: stableKernelPath,
700
+ outputDtype: lmHeadOutputDtype,
701
+ executionPolicies: config.executionPolicies ?? null,
702
+ });
703
+ await runProbes('logits', logitsTensor.buffer, {
704
+ numTokens,
705
+ hiddenSize: matmulVocabSize,
706
+ recorder,
707
+ operatorDiagnostics,
708
+ dtype: logitsTensor.dtype,
404
709
  });
405
710
 
406
711
  // Track intermediate buffer for cleanup after submit
407
- recorder.trackTemporaryBuffer(normedTensor.buffer);
712
+ const trackedTempBuffers = new Set();
713
+ const trackTempBufferOnce = (buffer) => {
714
+ if (!buffer || trackedTempBuffers.has(buffer)) {
715
+ return;
716
+ }
717
+ trackedTempBuffers.add(buffer);
718
+ recorder.trackTemporaryBuffer(buffer);
719
+ };
720
+ if (finalNormTensor !== normedTensor) {
721
+ trackTempBufferOnce(normedTensor.buffer);
722
+ }
723
+ trackTempBufferOnce(finalNormTensor.buffer);
724
+ if (lmHeadInputTensor !== finalNormTensor) {
725
+ trackTempBufferOnce(lmHeadInputTensor.buffer);
726
+ }
408
727
  if (normWeightOwned) {
409
728
  recorder.trackTemporaryBuffer(normWeightBuffer);
410
729
  }
@@ -29,6 +29,17 @@ export interface ComputeLogitsOptions {
29
29
  lastPositionOnly?: boolean;
30
30
  }
31
31
 
32
+ export interface ResolvedLmHeadMatmulConfig {
33
+ lastPositionOnly: boolean;
34
+ matmulRows: number;
35
+ phaseOverride: 'decode' | null;
36
+ }
37
+
38
+ export function resolveLmHeadMatmulConfig(
39
+ numTokens: number,
40
+ options?: ComputeLogitsOptions | null
41
+ ): ResolvedLmHeadMatmulConfig;
42
+
32
43
  /**
33
44
  * Compute logits from hidden states.
34
45
  *