doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
12
12
  // Imports for computeLogits orchestrator
13
13
  import { getDevice } from '../../../../gpu/device.js';
14
14
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
15
- import { runMatmul, runRMSNorm, castF16ToF32 } from '../../../../gpu/kernel-selector.js';
15
+ import { runMatmul, runRMSNorm, castF16ToF32, castF32ToF16 } from '../../../../gpu/kernel-selector.js';
16
16
  import { createTensor } from '../../../../gpu/tensor.js';
17
17
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../../gpu/weight-buffer.js';
18
18
  import { kernelTrace, traceStep } from '../kernel-trace.js';
@@ -22,16 +22,137 @@ import { rmsNormCPU, matmulCPU, f16BufferToF32 } from './cpu.js';
22
22
  import { resolveCpuWeightDims, computeChunkedLogitsGPU } from './gpu.js';
23
23
  import { finalizeLogits, readBufferWithCleanup } from './utils.js';
24
24
  import { getRuntimeConfig } from '../../../../config/runtime.js';
25
+ import { getKernelPathMatmulPrecision, getKernelPathStepPrecision } from '../../../../config/kernel-path-loader.js';
25
26
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
27
+ import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
26
28
 
27
29
  function shouldForceStableF32Logits(config, inputDtype) {
28
- // Small Gemma-family checkpoints can overflow in pure F16 logits path after RMSNorm offset.
29
- return inputDtype === 'f16'
30
- && config.rmsNormWeightOffset === true
30
+ if (inputDtype !== 'f16') {
31
+ return false;
32
+ }
33
+ // Softcapped output heads are numerically sensitive in pure F16 on the
34
+ // final RMSNorm + LM-head path. Widen only the logits tail so the main
35
+ // layer stack and KV cache can stay on the faster F16 lane.
36
+ if (Number.isFinite(config.finalLogitSoftcapping) && config.finalLogitSoftcapping > 0) {
37
+ return true;
38
+ }
39
+ // Small Gemma-family checkpoints can also overflow in pure F16 logits path
40
+ // after RMSNorm offset even without output softcapping.
41
+ return config.rmsNormWeightOffset === true
31
42
  && Number.isFinite(config.hiddenSize)
32
43
  && config.hiddenSize <= 768;
33
44
  }
34
45
 
46
+ function resolvePrecisionFieldDtype(precision, fallback, field) {
47
+ const requested = precision?.[field] ?? fallback;
48
+ if (requested == null) {
49
+ return fallback;
50
+ }
51
+ return selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: requested });
52
+ }
53
+
54
+ function resolveMatmulStepDtype(role, phase, kernelPath, fallback, field) {
55
+ const precision = getKernelPathMatmulPrecision(role, phase, 0, kernelPath);
56
+ return resolvePrecisionFieldDtype(precision, fallback, field);
57
+ }
58
+
59
+ function resolvePostLayerStepDtype(op, phase, kernelPath, fallback, field) {
60
+ const precision = getKernelPathStepPrecision(op, 'postLayer', phase, 0, kernelPath);
61
+ return resolvePrecisionFieldDtype(precision, fallback, field);
62
+ }
63
+
64
+ async function coerceTensorDtype(tensor, targetDtype, options = {}) {
65
+ if (!targetDtype || tensor.dtype === targetDtype) {
66
+ return tensor;
67
+ }
68
+ assertImplicitDtypeTransitionAllowed({
69
+ executionPolicies: options.executionPolicies ?? null,
70
+ fromDtype: tensor.dtype,
71
+ toDtype: targetDtype,
72
+ op: options.op ?? 'logits',
73
+ detail: 'The execution graph must declare this cast explicitly.',
74
+ transitionDeclaredBy: options.transitionDeclaredBy ?? null,
75
+ });
76
+ if (tensor.dtype === 'f32' && targetDtype === 'f16') {
77
+ return castF32ToF16(tensor);
78
+ }
79
+ if (tensor.dtype === 'f16' && targetDtype === 'f32') {
80
+ return castF16ToF32(tensor);
81
+ }
82
+ throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
83
+ }
84
+
85
+ const STABLE_F32_LOGITS_KERNEL_MAP = new Map([
86
+ ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
87
+ ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
88
+ ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
89
+ ]);
90
+
91
+ function createStableF32LogitsKernelPath(kernelPath) {
92
+ if (!kernelPath?.postLayer) {
93
+ return kernelPath;
94
+ }
95
+ let changed = false;
96
+ const postLayer = kernelPath.postLayer.map((step) => {
97
+ if (step?.op === 'final_norm') {
98
+ const precision = {
99
+ ...(step.precision ?? {}),
100
+ inputDtype: 'f32',
101
+ outputDtype: 'f32',
102
+ };
103
+ if (
104
+ step.precision?.inputDtype === precision.inputDtype
105
+ && step.precision?.outputDtype === precision.outputDtype
106
+ ) {
107
+ return step;
108
+ }
109
+ changed = true;
110
+ return {
111
+ ...step,
112
+ precision,
113
+ };
114
+ }
115
+ if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
116
+ return step;
117
+ }
118
+ const replacement = STABLE_F32_LOGITS_KERNEL_MAP.get(step.kernel) ?? step.kernel;
119
+ const precision = {
120
+ ...(step.precision ?? {}),
121
+ inputDtype: 'f32',
122
+ outputDtype: 'f32',
123
+ };
124
+ if (
125
+ replacement === step.kernel
126
+ && step.precision?.inputDtype === precision.inputDtype
127
+ && step.precision?.outputDtype === precision.outputDtype
128
+ ) {
129
+ return step;
130
+ }
131
+ changed = true;
132
+ return {
133
+ ...step,
134
+ kernel: replacement,
135
+ precision,
136
+ };
137
+ });
138
+ if (!changed) {
139
+ return kernelPath;
140
+ }
141
+ return {
142
+ ...kernelPath,
143
+ postLayer,
144
+ };
145
+ }
146
+
147
+ export function resolveLmHeadMatmulConfig(numTokens, options = null) {
148
+ const lastPositionOnly = options?.lastPositionOnly === true && numTokens > 1;
149
+ return {
150
+ lastPositionOnly,
151
+ matmulRows: lastPositionOnly ? 1 : numTokens,
152
+ phaseOverride: lastPositionOnly ? 'decode' : null,
153
+ };
154
+ }
155
+
35
156
 
36
157
  export async function computeLogits(
37
158
  hiddenStates,
@@ -43,7 +164,8 @@ export async function computeLogits(
43
164
  getNormWeightBuffer,
44
165
  debugCheckBuffer,
45
166
  debugProbes,
46
- options = null
167
+ options = null,
168
+ operatorDiagnostics = null
47
169
  ) {
48
170
  if (isTraceEnabled('logits')) {
49
171
  trace.logits(`LOGITS_ENTRY: numTokens=${numTokens}, useGPU=${useGPU}`);
@@ -138,7 +260,7 @@ export async function computeLogits(
138
260
  cpuWeightLayout === 'column' ? cpuWeightVocabSize : null
139
261
  )
140
262
  : matmulCPU(normed, (lmHead), numTokens, matmulVocabSize, hiddenSize);
141
- return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes);
263
+ return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
142
264
  }
143
265
 
144
266
  // GPU path
@@ -158,6 +280,7 @@ export async function computeLogits(
158
280
  numTokens,
159
281
  hiddenSize,
160
282
  probes: debugProbes,
283
+ operatorDiagnostics,
161
284
  dtype: inputDtype,
162
285
  });
163
286
 
@@ -182,14 +305,38 @@ export async function computeLogits(
182
305
 
183
306
  // Wrap input buffer as Tensor for RMSNorm
184
307
  const inputTensor = createTensor(inputBuffer, inputDtype, [numTokens, hiddenSize], 'logits_input');
185
- const forceStableF32Logits = shouldForceStableF32Logits(config, inputDtype);
308
+ const phase = numTokens === 1 ? 'decode' : 'prefill';
309
+ const kernelPath = config.kernelPath ?? null;
310
+ const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
311
+ const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
312
+ const forceStableF32Logits = !hasExplicitFinalNormPrecision && shouldForceStableF32Logits(config, inputDtype);
313
+ const stableKernelPath = forceStableF32Logits
314
+ ? createStableF32LogitsKernelPath(kernelPath)
315
+ : kernelPath;
186
316
  let normInputTensor = inputTensor;
187
317
  let normInputBufferOwned = false;
188
318
  if (forceStableF32Logits) {
319
+ assertImplicitDtypeTransitionAllowed({
320
+ executionPolicies: config.executionPolicies ?? null,
321
+ fromDtype: inputTensor.dtype,
322
+ toDtype: 'f32',
323
+ op: 'logits_final_norm',
324
+ detail: 'Stable logits mode would widen activations implicitly before final RMSNorm.',
325
+ });
189
326
  normInputTensor = await castF16ToF32(inputTensor);
190
327
  normInputBufferOwned = true;
328
+ } else {
329
+ const finalNormInputDtype = resolvePostLayerStepDtype('final_norm', phase, stableKernelPath, inputTensor.dtype, 'inputDtype');
330
+ normInputTensor = finalNormInputDtype !== inputTensor.dtype
331
+ ? await coerceTensorDtype(inputTensor, finalNormInputDtype, {
332
+ executionPolicies: config.executionPolicies ?? null,
333
+ op: 'final_norm',
334
+ transitionDeclaredBy: 'step_precision',
335
+ })
336
+ : inputTensor;
337
+ normInputBufferOwned = normInputTensor !== inputTensor;
191
338
  }
192
- const normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
339
+ let normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
193
340
  batchSize: numTokens,
194
341
  hiddenSize,
195
342
  rmsNormWeightOffset: config.rmsNormWeightOffset,
@@ -197,22 +344,44 @@ export async function computeLogits(
197
344
  if (normInputBufferOwned) {
198
345
  releaseBuffer(normInputTensor.buffer);
199
346
  }
200
- await runProbes('final_norm', normedTensor.buffer, {
347
+ let finalNormTensor = normedTensor;
348
+ if (!forceStableF32Logits) {
349
+ const finalNormOutputDtype = resolvePostLayerStepDtype(
350
+ 'final_norm',
351
+ phase,
352
+ stableKernelPath,
353
+ normedTensor.dtype,
354
+ 'outputDtype'
355
+ );
356
+ finalNormTensor = finalNormOutputDtype !== normedTensor.dtype
357
+ ? await coerceTensorDtype(normedTensor, finalNormOutputDtype, {
358
+ executionPolicies: config.executionPolicies ?? null,
359
+ op: 'final_norm',
360
+ transitionDeclaredBy: 'step_precision',
361
+ })
362
+ : normedTensor;
363
+ }
364
+ if (finalNormTensor !== normedTensor) {
365
+ releaseBuffer(normedTensor.buffer);
366
+ normedTensor = null;
367
+ }
368
+ await runProbes('final_norm', finalNormTensor.buffer, {
201
369
  numTokens,
202
370
  hiddenSize,
203
371
  probes: debugProbes,
204
- dtype: normedTensor.dtype,
372
+ operatorDiagnostics,
373
+ dtype: finalNormTensor.dtype,
205
374
  });
206
375
 
207
376
  // Trace final norm output
208
377
  if (kernelTrace.enabled) {
209
- await traceStep('rmsnorm', 'final_norm', -1, normedTensor.buffer, [numTokens, hiddenSize]);
378
+ await traceStep('rmsnorm', 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
210
379
  }
211
380
 
212
381
  // Debug: Check hidden state after final norm
213
382
  if (!debugFlags.afterFinalNormDebugDone && debugCheckBuffer) {
214
383
  debugFlags.afterFinalNormDebugDone = true;
215
- await debugCheckBuffer(normedTensor.buffer, 'After final norm', numTokens, hiddenSize);
384
+ await debugCheckBuffer(finalNormTensor.buffer, 'After final norm', numTokens, hiddenSize);
216
385
  }
217
386
 
218
387
  if (isCpuWeightBuffer(lmHead)) {
@@ -220,22 +389,24 @@ export async function computeLogits(
220
389
  throw new Error('LM head CPU weight is missing vocabSize metadata.');
221
390
  }
222
391
  const rawLogits = await computeChunkedLogitsGPU(
223
- normedTensor,
392
+ finalNormTensor,
224
393
  lmHead,
225
394
  numTokens,
226
395
  hiddenSize,
227
396
  matmulVocabSize,
228
397
  cpuWeightVocabSize,
229
398
  debugProbes,
399
+ operatorDiagnostics,
230
400
  largeWeights,
231
- config.kernelPath ?? null
401
+ stableKernelPath,
402
+ config.executionPolicies ?? null
232
403
  );
233
404
 
234
405
  if (inputBufferOwned) releaseBuffer(inputBuffer);
235
- releaseBuffer(normedTensor.buffer);
406
+ releaseBuffer(finalNormTensor.buffer);
236
407
  if (!getNormWeightBuffer && !isGpuBufferInstance(finalNorm)) releaseBuffer(normWeightBuffer);
237
408
 
238
- return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes);
409
+ return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
239
410
  }
240
411
 
241
412
  // 3. Project to vocab via LM head
@@ -253,28 +424,52 @@ export async function computeLogits(
253
424
  lmHeadBufferOwned = true;
254
425
  }
255
426
 
427
+ const lastTokenMatmul = resolveLmHeadMatmulConfig(numTokens, options);
428
+ const { lastPositionOnly, matmulRows } = lastTokenMatmul;
429
+ const matmulPhaseOverride = lastTokenMatmul.phaseOverride;
430
+ const lmHeadPhase = matmulPhaseOverride ?? (matmulRows === 1 ? 'decode' : 'prefill');
431
+
256
432
  // Debug: Log buffer info for lm_head matmul
257
433
  const lmHeadGPU = isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer;
258
- const lmHeadDtype = getWeightDtype(lmHeadBuffer); // dtype from WeightBuffer metadata
259
- const normedDtype = normedTensor.dtype;
434
+ const lmHeadDtype = getWeightDtype(lmHeadBuffer);
435
+ const normedDtype = finalNormTensor.dtype;
260
436
  if (isTraceEnabled('logits')) {
261
- trace.logits(`LM_HEAD_MATMUL: M=${numTokens}, N=${matmulVocabSize}, K=${hiddenSize}, lmHeadDtype=${lmHeadDtype}, normedDtype=${normedDtype}, size=${lmHeadGPU.size}, bufLabel=${lmHeadGPU.label}`);
437
+ trace.logits(
438
+ `LM_HEAD_MATMUL: M=${matmulRows}, N=${matmulVocabSize}, K=${hiddenSize}, ` +
439
+ `phase=${matmulPhaseOverride ?? 'auto'}, lmHeadDtype=${lmHeadDtype}, ` +
440
+ `normedDtype=${normedDtype}, size=${lmHeadGPU.size}, bufLabel=${lmHeadGPU.label}`
441
+ );
262
442
  }
263
443
 
264
- const lastPositionOnly = options?.lastPositionOnly === true && numTokens > 1;
265
- const matmulRows = lastPositionOnly ? 1 : numTokens;
266
- const matmulPhaseOverride = lastPositionOnly ? 'prefill' : null;
267
- let matmulInputTensor = normedTensor;
444
+ let matmulInputTensor = finalNormTensor;
268
445
  let matmulInputOwned = false;
269
446
  if (lastPositionOnly) {
270
- const inputBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: normedTensor.dtype });
447
+ const inputBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: finalNormTensor.dtype });
271
448
  const rowSize = hiddenSize * inputBytes;
272
449
  const rowOffset = (numTokens - 1) * rowSize;
273
450
  const lastInputBuffer = acquireBuffer(rowSize, undefined, 'logits_input_last');
274
451
  const encoder = device.createCommandEncoder();
275
- encoder.copyBufferToBuffer(normedTensor.buffer, rowOffset, lastInputBuffer, 0, rowSize);
452
+ encoder.copyBufferToBuffer(finalNormTensor.buffer, rowOffset, lastInputBuffer, 0, rowSize);
276
453
  device.queue.submit([encoder.finish()]);
277
- matmulInputTensor = createTensor(lastInputBuffer, normedTensor.dtype, [1, hiddenSize], 'logits_input_last');
454
+ matmulInputTensor = createTensor(lastInputBuffer, finalNormTensor.dtype, [1, hiddenSize], 'logits_input_last');
455
+ matmulInputOwned = true;
456
+ }
457
+ const lmHeadInputDtype = forceStableF32Logits
458
+ ? matmulInputTensor.dtype
459
+ : resolveMatmulStepDtype('lm_head', lmHeadPhase, stableKernelPath, matmulInputTensor.dtype, 'inputDtype');
460
+ const lmHeadOutputDtype = forceStableF32Logits
461
+ ? matmulInputTensor.dtype
462
+ : resolveMatmulStepDtype('lm_head', lmHeadPhase, stableKernelPath, matmulInputTensor.dtype, 'outputDtype');
463
+ if (lmHeadInputDtype !== matmulInputTensor.dtype) {
464
+ const coercedInput = await coerceTensorDtype(matmulInputTensor, lmHeadInputDtype, {
465
+ executionPolicies: config.executionPolicies ?? null,
466
+ op: 'lm_head',
467
+ transitionDeclaredBy: 'step_precision',
468
+ });
469
+ if (matmulInputOwned) {
470
+ releaseBuffer(matmulInputTensor.buffer);
471
+ }
472
+ matmulInputTensor = coercedInput;
278
473
  matmulInputOwned = true;
279
474
  }
280
475
 
@@ -283,12 +478,15 @@ export async function computeLogits(
283
478
  transposeB: 'auto',
284
479
  role: 'lm_head',
285
480
  phaseOverride: matmulPhaseOverride,
286
- kernelPath: config.kernelPath ?? null,
481
+ kernelPath: stableKernelPath,
482
+ outputDtype: lmHeadOutputDtype,
483
+ executionPolicies: config.executionPolicies ?? null,
287
484
  });
288
485
  await runProbes('logits', logitsTensor.buffer, {
289
486
  numTokens: matmulRows,
290
487
  hiddenSize: matmulVocabSize,
291
488
  probes: debugProbes,
489
+ operatorDiagnostics,
292
490
  dtype: logitsTensor.dtype,
293
491
  });
294
492
 
@@ -302,7 +500,7 @@ export async function computeLogits(
302
500
  const logitsReadSize = matmulRows * matmulVocabSize * logitsBytes;
303
501
  const logitsData = await readBufferWithCleanup(logitsTensor.buffer, logitsReadSize, () => {
304
502
  if (inputBufferOwned) releaseBuffer(inputBuffer);
305
- releaseBuffer(normedTensor.buffer);
503
+ releaseBuffer(finalNormTensor.buffer);
306
504
  if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
307
505
  releaseBuffer(logitsTensor.buffer);
308
506
  if (!getNormWeightBuffer && !isGpuBufferInstance(finalNorm)) releaseBuffer(normWeightBuffer);
@@ -312,5 +510,5 @@ export async function computeLogits(
312
510
  const rawLogits = logitsTensor.dtype === 'f16'
313
511
  ? f16BufferToF32(logitsData)
314
512
  : new Float32Array(logitsData);
315
- return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes);
513
+ return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
316
514
  }
@@ -8,6 +8,7 @@
8
8
  */
9
9
 
10
10
  import type { LargeWeightConfigSchema, KernelPathSchema } from '../../../../config/schema/index.js';
11
+ import type { ExecutionV1PoliciesSchema } from '../../../../config/schema/execution-v1.schema.js';
11
12
  import type { WeightBuffer, CpuWeightBuffer } from '../../../../gpu/weight-buffer.js';
12
13
 
13
14
  /**
@@ -25,6 +26,8 @@ export interface LogitsConfig {
25
26
  activationDtype?: 'f16' | 'f32';
26
27
  /** Explicit kernel-path context for variant selection. */
27
28
  kernelPath?: KernelPathSchema | null;
29
+ /** Execution-v1 fail-fast policies for dtype transitions. */
30
+ executionPolicies?: ExecutionV1PoliciesSchema | null;
28
31
  /** Gemma 2 RMS scaling: (1+w)*x */
29
32
  rmsNormWeightOffset?: boolean;
30
33
  }
@@ -35,7 +35,8 @@ export async function finalizeLogits(
35
35
  matmulVocabSize,
36
36
  vocabSize,
37
37
  config,
38
- debugProbes
38
+ debugProbes,
39
+ operatorDiagnostics = null
39
40
  ) {
40
41
  let logits = rawLogits;
41
42
 
@@ -62,6 +63,7 @@ export async function finalizeLogits(
62
63
  numTokens,
63
64
  hiddenSize: vocabSize,
64
65
  probes: debugProbes,
66
+ operatorDiagnostics,
65
67
  });
66
68
 
67
69
  return logits;
@@ -1,7 +1,7 @@
1
1
 
2
2
 
3
3
  import { releaseBuffer } from '../../../memory/buffer-pool.js';
4
- import { isWeightBuffer } from '../../../gpu/weight-buffer.js';
4
+ import { isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
5
5
  import { runMatmul, recordMatmul } from '../../../gpu/kernel-selector.js';
6
6
  import { runResidualAdd, recordResidualAdd } from '../../../gpu/kernels/residual.js';
7
7
  import { runScale, recordScale } from '../../../gpu/kernels/scale.js';
@@ -17,8 +17,8 @@ export async function applyLoRA(input, baseOutput, lora, dims, getWeightBuffer,
17
17
 
18
18
  const aBuf = getWeightBuffer(lora.a, 'lora_a');
19
19
  const bBuf = getWeightBuffer(lora.b, 'lora_b');
20
- const ownsA = !(typeof GPUBuffer !== 'undefined' && lora.a instanceof GPUBuffer) && !isWeightBuffer(lora.a);
21
- const ownsB = !(typeof GPUBuffer !== 'undefined' && lora.b instanceof GPUBuffer) && !isWeightBuffer(lora.b);
20
+ const ownsA = !isGpuBufferInstance(lora.a) && !isWeightBuffer(lora.a);
21
+ const ownsB = !isGpuBufferInstance(lora.b) && !isWeightBuffer(lora.b);
22
22
  // Extract underlying GPUBuffer for WeightBuffers
23
23
  const aBufGPU = isWeightBuffer(aBuf) ? aBuf.buffer : aBuf;
24
24
  const bBufGPU = isWeightBuffer(bBuf) ? bBuf.buffer : bBuf;
@@ -36,7 +36,8 @@ export function runKernelWarmup(options: KernelWarmupOptions): Promise<void>;
36
36
  export function applyModelBatchingRuntimeDefaults(
37
37
  runtimeConfig: RuntimeConfigSchema,
38
38
  manifest: Manifest,
39
- modelConfig: ParsedModelConfig | null | undefined
39
+ modelConfig: ParsedModelConfig | null | undefined,
40
+ runtimeOverrides?: Record<string, unknown> | null
40
41
  ): RuntimeConfigSchema;
41
42
 
42
43
  export function resolveKernelPathState(