doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -5,6 +5,7 @@ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js'
5
5
  import { runMatmul, runRMSNorm } from '../../../../gpu/kernel-selector.js';
6
6
  import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
7
7
  import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
8
+ import { recordLmHeadArgmaxF16 } from '../../../../gpu/kernels/lm-head-argmax.js';
8
9
  import { createTensor } from '../../../../gpu/tensor.js';
9
10
  import {
10
11
  castF16ToF32,
@@ -12,7 +13,7 @@ import {
12
13
  recordCastF16ToF32,
13
14
  recordCastF32ToF16,
14
15
  } from '../../../../gpu/kernels/cast.js';
15
- import { createWeightBuffer, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../../gpu/weight-buffer.js';
16
+ import { createWeightBuffer, createSplitWeightBuffer, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../../gpu/weight-buffer.js';
16
17
  import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
17
18
  import { getRuntimeConfig } from '../../../../config/runtime.js';
18
19
  import { getKernelPathMatmulPrecision, getKernelPathStepPrecision } from '../../../../config/kernel-path-loader.js';
@@ -58,6 +59,10 @@ function resolvePostLayerStepDtype(op, phase, kernelPath, fallback, field) {
58
59
  return resolvePrecisionFieldDtype(precision, fallback, field);
59
60
  }
60
61
 
62
+ function resolveLmHeadMatmulRole(phase) {
63
+ return phase === 'prefill' ? 'lm_head_prefill' : 'lm_head';
64
+ }
65
+
61
66
  async function coerceTensorDtype(tensor, targetDtype, recorder = null, options = {}) {
62
67
  if (!targetDtype || tensor.dtype === targetDtype) {
63
68
  return tensor;
@@ -143,6 +148,7 @@ function createStableF32LogitsKernelPath(kernelPath) {
143
148
 
144
149
  const bf16ScratchU32 = new Uint32Array(1);
145
150
  const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
151
+ const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
146
152
 
147
153
  function bf16ToF32(value) {
148
154
  bf16ScratchU32[0] = (value & 0xffff) << 16;
@@ -158,6 +164,17 @@ function isRangeBackedCpuWeightSource(value) {
158
164
  );
159
165
  }
160
166
 
167
+ function alignByteLength(byteLength) {
168
+ return Math.ceil(byteLength / 4) * 4;
169
+ }
170
+
171
+ function writeBufferInChunks(queue, buffer, bytes) {
172
+ for (let offset = 0; offset < bytes.byteLength; offset += SPLIT_UPLOAD_CHUNK_BYTES) {
173
+ const end = Math.min(offset + SPLIT_UPLOAD_CHUNK_BYTES, bytes.byteLength);
174
+ queue.writeBuffer(buffer, offset, bytes, offset, end - offset);
175
+ }
176
+ }
177
+
161
178
  function normalizeRangeBytes(value, label) {
162
179
  if (value instanceof Uint8Array) return value;
163
180
  if (value instanceof ArrayBuffer) return new Uint8Array(value);
@@ -357,6 +374,141 @@ export function writeChunkLogits(
357
374
  }
358
375
 
359
376
 
377
+ function resolveSplitLmHeadRows(device, hiddenSize, largeWeightConfig) {
378
+ if (largeWeightConfig.safetyRatio == null) {
379
+ throw new Error('runtime.inference.largeWeights.safetyRatio is required.');
380
+ }
381
+ const safety = Math.min(Math.max(largeWeightConfig.safetyRatio, 0.1), 1);
382
+ const maxBinding = Math.min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize);
383
+ const maxBytes = Math.floor(maxBinding * safety);
384
+ const rowBytes = hiddenSize * 2;
385
+ const rowsByBinding = Math.floor(maxBytes / rowBytes);
386
+ const requested = largeWeightConfig.lmHeadChunkRows;
387
+ const rows = Number.isInteger(requested) && requested > 0
388
+ ? Math.min(requested, rowsByBinding)
389
+ : rowsByBinding;
390
+ if (!Number.isFinite(rows) || rows <= 0) {
391
+ throw new Error(
392
+ `[Logits] split LM head row size underflow (maxBytes=${maxBytes}, hiddenSize=${hiddenSize}).`
393
+ );
394
+ }
395
+ return rows;
396
+ }
397
+
398
+ export function shouldMaterializeSplitLmHeadGPU(lmHead, largeWeightConfig) {
399
+ const overrides = largeWeightConfig?.gpuResidentOverrides;
400
+ if (!Array.isArray(overrides) || overrides.length === 0) {
401
+ return false;
402
+ }
403
+ const label = lmHead?.label;
404
+ return typeof label === 'string' && overrides.includes(label);
405
+ }
406
+
407
+ function destroySplitWeightBuffer(splitWeight) {
408
+ if (!splitWeight) {
409
+ return;
410
+ }
411
+ for (const section of splitWeight.sections ?? []) {
412
+ try {
413
+ section.buffer.destroy();
414
+ } catch {}
415
+ }
416
+ }
417
+
418
+ async function readSplitLmHeadSectionBytes(lmHead, hiddenSize, rowStart, rowCount) {
419
+ const byteOffset = rowStart * hiddenSize * 2;
420
+ const byteLength = rowCount * hiddenSize * 2;
421
+ const data = lmHead.data;
422
+ if (isRangeBackedCpuWeightSource(data)) {
423
+ const bytes = normalizeRangeBytes(
424
+ await data.loadRange(byteOffset, byteLength),
425
+ 'CPU LM head split range source'
426
+ );
427
+ if (bytes.byteLength !== byteLength) {
428
+ throw new Error(
429
+ `[Logits] CPU LM head split source returned ${bytes.byteLength} bytes, expected ${byteLength}.`
430
+ );
431
+ }
432
+ return bytes;
433
+ }
434
+ if (data instanceof Uint16Array) {
435
+ return new Uint8Array(data.buffer, data.byteOffset + byteOffset, byteLength);
436
+ }
437
+ return null;
438
+ }
439
+
440
+ async function materializeSplitLmHeadGPU(lmHead, hiddenSize, weightVocabSize, largeWeightConfig) {
441
+ if (!shouldMaterializeSplitLmHeadGPU(lmHead, largeWeightConfig)) {
442
+ if (lmHead.gpuSplitWeight) {
443
+ destroySplitWeightBuffer(lmHead.gpuSplitWeight);
444
+ lmHead.gpuSplitWeight = null;
445
+ }
446
+ return null;
447
+ }
448
+ if (lmHead.gpuSplitWeight) {
449
+ return lmHead.gpuSplitWeight;
450
+ }
451
+ if (lmHead.layout !== 'row' || lmHead.dtype !== 'f16') {
452
+ return null;
453
+ }
454
+
455
+ const device = getDevice();
456
+ if (!device) {
457
+ throw new Error('[Logits] GPU device not available for split LM head materialization.');
458
+ }
459
+
460
+ const rowsPerSection = resolveSplitLmHeadRows(device, hiddenSize, largeWeightConfig);
461
+ const createdBuffers = [];
462
+ try {
463
+ const sections = [];
464
+ for (let rowStart = 0; rowStart < weightVocabSize; rowStart += rowsPerSection) {
465
+ const rowCount = Math.min(rowsPerSection, weightVocabSize - rowStart);
466
+ const bytes = await readSplitLmHeadSectionBytes(lmHead, hiddenSize, rowStart, rowCount);
467
+ if (!bytes) {
468
+ for (const buffer of createdBuffers) {
469
+ buffer.destroy();
470
+ }
471
+ return null;
472
+ }
473
+ const buffer = device.createBuffer({
474
+ label: `${lmHead.label ?? 'lm_head'}:lazy_split:${sections.length}`,
475
+ size: alignByteLength(bytes.byteLength),
476
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
477
+ });
478
+ createdBuffers.push(buffer);
479
+ writeBufferInChunks(device.queue, buffer, bytes);
480
+ sections.push({ buffer, rowStart, rowCount });
481
+ }
482
+ const splitWeight = createSplitWeightBuffer(
483
+ sections,
484
+ lmHead.dtype,
485
+ lmHead.layout,
486
+ lmHead.shape,
487
+ lmHead.label
488
+ );
489
+ log.warn(
490
+ 'Logits',
491
+ `LM head "${lmHead.label ?? 'lm_head'}" materialized as lazy split GPU sections ` +
492
+ `(${sections.length} sections, dtype=${lmHead.dtype}, layout=${lmHead.layout}).`
493
+ );
494
+ Object.defineProperty(lmHead, 'gpuSplitWeight', {
495
+ value: splitWeight,
496
+ configurable: true,
497
+ enumerable: false,
498
+ writable: true,
499
+ });
500
+ return splitWeight;
501
+ } catch (error) {
502
+ for (const buffer of createdBuffers) {
503
+ try {
504
+ buffer.destroy();
505
+ } catch {}
506
+ }
507
+ throw error;
508
+ }
509
+ }
510
+
511
+
360
512
  export async function computeChunkedLogitsGPU(
361
513
  normedTensor,
362
514
  lmHead,
@@ -378,10 +530,27 @@ export async function computeChunkedLogitsGPU(
378
530
  throw new Error('[Logits] largeWeights config is required for chunked LM head.');
379
531
  }
380
532
 
533
+ const splitLmHead = await materializeSplitLmHeadGPU(lmHead, hiddenSize, weightVocabSize, largeWeightConfig);
534
+ if (splitLmHead) {
535
+ return computeSplitLogitsGPU(
536
+ normedTensor,
537
+ splitLmHead,
538
+ numTokens,
539
+ hiddenSize,
540
+ vocabSize,
541
+ weightVocabSize,
542
+ debugProbes,
543
+ operatorDiagnostics,
544
+ kernelPath,
545
+ executionPolicies
546
+ );
547
+ }
548
+
381
549
  const chunkRows = resolveLmHeadChunkRows(device, numTokens, hiddenSize, largeWeightConfig);
382
550
  const phase = numTokens === 1 ? 'decode' : 'prefill';
383
- const lmHeadInputDtype = resolveMatmulStepDtype('lm_head', phase, kernelPath, normedTensor.dtype, 'inputDtype');
384
- const lmHeadOutputDtype = resolveMatmulStepDtype('lm_head', phase, kernelPath, normedTensor.dtype, 'outputDtype');
551
+ const lmHeadRole = resolveLmHeadMatmulRole(phase);
552
+ const lmHeadInputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'inputDtype');
553
+ const lmHeadOutputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'outputDtype');
385
554
  const caps = getKernelCapabilities();
386
555
  const weightDtype = selectRuleValue('inference', 'dtype', 'lmHeadChunkWeightDtype', {
387
556
  preferF16: largeWeightConfig.preferF16,
@@ -405,41 +574,52 @@ export async function computeChunkedLogitsGPU(
405
574
 
406
575
  for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
407
576
  const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
408
- const chunkData = await extractLmHeadChunk(
409
- lmHead.data,
410
- lmHead.layout,
411
- hiddenSize,
412
- weightVocabSize,
413
- rowOffset,
414
- rowCount,
415
- lmHead.dtype
416
- );
417
-
418
- const f32Buffer = acquireBuffer(chunkData.byteLength, undefined, 'lm_head_chunk_f32');
419
- device.queue.writeBuffer(
420
- f32Buffer,
421
- 0,
422
- chunkData.buffer,
423
- chunkData.byteOffset,
424
- chunkData.byteLength
425
- );
426
-
427
577
  const chunkShape = lmHead.layout === 'column'
428
578
  ? [hiddenSize, rowCount]
429
579
  : [rowCount, hiddenSize];
430
580
 
431
- let weightBuffer = createWeightBuffer(f32Buffer, 'f32', lmHead.layout, chunkShape, 'lm_head_chunk_f32');
581
+ let weightBuffer;
582
+ if (preferF16 && lmHead.layout === 'row' && lmHead.dtype === 'f16') {
583
+ const chunkBytes = await readSplitLmHeadSectionBytes(lmHead, hiddenSize, rowOffset, rowCount);
584
+ if (!chunkBytes) {
585
+ throw new Error('[Logits] F16 LM head chunk source is not range-readable.');
586
+ }
587
+ const f16Buffer = acquireBuffer(alignByteLength(chunkBytes.byteLength), undefined, 'lm_head_chunk_f16');
588
+ writeBufferInChunks(device.queue, f16Buffer, chunkBytes);
589
+ weightBuffer = createWeightBuffer(f16Buffer, 'f16', lmHead.layout, chunkShape, 'lm_head_chunk_f16');
590
+ } else {
591
+ const chunkData = await extractLmHeadChunk(
592
+ lmHead.data,
593
+ lmHead.layout,
594
+ hiddenSize,
595
+ weightVocabSize,
596
+ rowOffset,
597
+ rowCount,
598
+ lmHead.dtype
599
+ );
600
+
601
+ const f32Buffer = acquireBuffer(chunkData.byteLength, undefined, 'lm_head_chunk_f32');
602
+ device.queue.writeBuffer(
603
+ f32Buffer,
604
+ 0,
605
+ chunkData.buffer,
606
+ chunkData.byteOffset,
607
+ chunkData.byteLength
608
+ );
609
+
610
+ weightBuffer = createWeightBuffer(f32Buffer, 'f32', lmHead.layout, chunkShape, 'lm_head_chunk_f32');
432
611
 
433
- if (preferF16) {
434
- const f32Tensor = createTensor(f32Buffer, 'f32', chunkShape, 'lm_head_chunk_f32');
435
- const f16Tensor = await castF32ToF16(f32Tensor);
436
- releaseBuffer(f32Buffer);
437
- weightBuffer = createWeightBuffer(f16Tensor.buffer, 'f16', lmHead.layout, chunkShape, 'lm_head_chunk_f16');
612
+ if (preferF16) {
613
+ const f32Tensor = createTensor(f32Buffer, 'f32', chunkShape, 'lm_head_chunk_f32');
614
+ const f16Tensor = await castF32ToF16(f32Tensor);
615
+ releaseBuffer(f32Buffer);
616
+ weightBuffer = createWeightBuffer(f16Tensor.buffer, 'f16', lmHead.layout, chunkShape, 'lm_head_chunk_f16');
617
+ }
438
618
  }
439
619
 
440
620
  const logitsTensor = await runMatmul(matmulInput, weightBuffer, numTokens, rowCount, hiddenSize, {
441
621
  transposeB: 'auto',
442
- role: 'lm_head',
622
+ role: lmHeadRole,
443
623
  kernelPath,
444
624
  outputDtype: lmHeadOutputDtype,
445
625
  executionPolicies,
@@ -478,6 +658,104 @@ export async function computeChunkedLogitsGPU(
478
658
  }
479
659
 
480
660
 
661
+ export async function computeSplitLogitsGPU(
662
+ normedTensor,
663
+ lmHead,
664
+ numTokens,
665
+ hiddenSize,
666
+ vocabSize,
667
+ weightVocabSize,
668
+ debugProbes,
669
+ operatorDiagnostics,
670
+ kernelPath = null,
671
+ executionPolicies = null
672
+ ) {
673
+ const device = getDevice();
674
+ if (!device) {
675
+ throw new Error('[Logits] GPU device not available for split LM head.');
676
+ }
677
+ if (lmHead.layout !== 'row') {
678
+ throw new Error(`[Logits] split LM head requires row layout, got "${lmHead.layout}".`);
679
+ }
680
+
681
+ const phase = numTokens === 1 ? 'decode' : 'prefill';
682
+ const lmHeadRole = resolveLmHeadMatmulRole(phase);
683
+ const lmHeadInputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'inputDtype');
684
+ const lmHeadOutputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'outputDtype');
685
+ const logits = new Float32Array(numTokens * vocabSize);
686
+ let matmulInput = normedTensor;
687
+ let matmulInputOwned = false;
688
+
689
+ try {
690
+ matmulInput = lmHeadInputDtype !== normedTensor.dtype
691
+ ? await coerceTensorDtype(normedTensor, lmHeadInputDtype, null, {
692
+ executionPolicies,
693
+ op: 'lm_head',
694
+ transitionDeclaredBy: 'step_precision',
695
+ })
696
+ : normedTensor;
697
+ matmulInputOwned = matmulInput !== normedTensor;
698
+
699
+ for (const section of lmHead.sections) {
700
+ if (section.rowStart >= vocabSize) {
701
+ continue;
702
+ }
703
+ if (section.rowStart + section.rowCount > weightVocabSize) {
704
+ throw new Error(
705
+ `[Logits] split LM head section exceeds weight vocab: rowStart=${section.rowStart}, ` +
706
+ `rowCount=${section.rowCount}, weightVocabSize=${weightVocabSize}.`
707
+ );
708
+ }
709
+
710
+ const rowCount = Math.min(section.rowCount, vocabSize - section.rowStart);
711
+ const weightBuffer = createWeightBuffer(
712
+ section.buffer,
713
+ lmHead.dtype,
714
+ lmHead.layout,
715
+ [section.rowCount, hiddenSize],
716
+ `${lmHead.label ?? 'lm_head'}:split:${section.rowStart}`
717
+ );
718
+ const logitsTensor = await runMatmul(matmulInput, weightBuffer, numTokens, rowCount, hiddenSize, {
719
+ transposeB: 'auto',
720
+ role: lmHeadRole,
721
+ kernelPath,
722
+ outputDtype: lmHeadOutputDtype,
723
+ executionPolicies,
724
+ });
725
+
726
+ if (debugProbes?.length || operatorDiagnostics?.enabled) {
727
+ await runProbes('logits', logitsTensor.buffer, {
728
+ numTokens,
729
+ hiddenSize: rowCount,
730
+ probes: debugProbes,
731
+ operatorDiagnostics,
732
+ dtype: logitsTensor.dtype,
733
+ });
734
+ }
735
+
736
+ const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsTensor.dtype });
737
+ const chunkLogitsData = await readBufferWithCleanup(
738
+ logitsTensor.buffer,
739
+ numTokens * rowCount * logitsBytes,
740
+ () => {
741
+ releaseBuffer(logitsTensor.buffer);
742
+ }
743
+ );
744
+ const chunkLogits = logitsTensor.dtype === 'f16'
745
+ ? f16BufferToF32(chunkLogitsData)
746
+ : new Float32Array(chunkLogitsData);
747
+ writeChunkLogits(logits, chunkLogits, numTokens, vocabSize, section.rowStart, rowCount);
748
+ }
749
+ } finally {
750
+ if (matmulInputOwned) {
751
+ releaseBuffer(matmulInput.buffer);
752
+ }
753
+ }
754
+
755
+ return logits;
756
+ }
757
+
758
+
481
759
  export async function computeLogitsGPU(
482
760
  hiddenStates,
483
761
  numTokens,
@@ -508,7 +786,7 @@ export async function computeLogitsGPU(
508
786
  log.warn('Pipeline', 'Final norm or LM head not loaded');
509
787
  return null;
510
788
  }
511
- if (isCpuWeightBuffer(lmHead)) {
789
+ if (isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead)) {
512
790
  return null;
513
791
  }
514
792
 
@@ -621,12 +899,13 @@ export async function computeLogitsGPU(
621
899
  releaseBuffer(normInputTensor.buffer);
622
900
  normInputOwned = false;
623
901
  }
902
+ const lmHeadRole = resolveLmHeadMatmulRole(phase);
624
903
  const lmHeadInputDtype = forceStableF32Logits
625
904
  ? finalNormTensor.dtype
626
- : resolveMatmulStepDtype('lm_head', phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
905
+ : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
627
906
  const lmHeadOutputDtype = forceStableF32Logits
628
907
  ? finalNormTensor.dtype
629
- : resolveMatmulStepDtype('lm_head', phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
908
+ : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
630
909
  lmHeadInputTensor = lmHeadInputDtype !== finalNormTensor.dtype
631
910
  ? await coerceTensorDtype(finalNormTensor, lmHeadInputDtype, null, {
632
911
  executionPolicies: config.executionPolicies ?? null,
@@ -654,7 +933,7 @@ export async function computeLogitsGPU(
654
933
 
655
934
  const logitsTensor = await runMatmul(lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
656
935
  transposeB: 'auto',
657
- role: 'lm_head',
936
+ role: lmHeadRole,
658
937
  kernelPath: stableKernelPath,
659
938
  outputDtype: lmHeadOutputDtype,
660
939
  executionPolicies: config.executionPolicies ?? null,
@@ -690,7 +969,7 @@ export async function computeLogitsGPU(
690
969
  }
691
970
 
692
971
 
693
- export async function recordLogitsGPU(
972
+ async function recordLogitsTailGPU(
694
973
  recorder,
695
974
  hiddenStates,
696
975
  numTokens,
@@ -712,8 +991,8 @@ export async function recordLogitsGPU(
712
991
  if (!finalNorm || !lmHead) {
713
992
  throw new Error('[recordLogitsGPU] Final norm or LM head not loaded');
714
993
  }
715
- if (isCpuWeightBuffer(lmHead)) {
716
- throw new Error('[recordLogitsGPU] CPU-resident LM head not supported in recorded path');
994
+ if (isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead)) {
995
+ throw new Error('[recordLogitsGPU] CPU-resident or split LM head not supported in recorded path');
717
996
  }
718
997
 
719
998
  // Get norm weight buffer
@@ -775,6 +1054,7 @@ export async function recordLogitsGPU(
775
1054
  batchSize: numTokens,
776
1055
  hiddenSize,
777
1056
  rmsNormWeightOffset: config.rmsNormWeightOffset,
1057
+ label: 'final_norm',
778
1058
  });
779
1059
  let finalNormTensor = normedTensor;
780
1060
  if (!forceStableF32Logits) {
@@ -800,12 +1080,13 @@ export async function recordLogitsGPU(
800
1080
  operatorDiagnostics,
801
1081
  dtype: finalNormTensor.dtype,
802
1082
  });
1083
+ const lmHeadRole = resolveLmHeadMatmulRole(phase);
803
1084
  const lmHeadInputDtype = forceStableF32Logits
804
1085
  ? finalNormTensor.dtype
805
- : resolveMatmulStepDtype('lm_head', phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
1086
+ : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
806
1087
  const lmHeadOutputDtype = forceStableF32Logits
807
1088
  ? finalNormTensor.dtype
808
- : resolveMatmulStepDtype('lm_head', phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
1089
+ : resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
809
1090
  const lmHeadInputTensor = lmHeadInputDtype !== finalNormTensor.dtype
810
1091
  ? await coerceTensorDtype(finalNormTensor, lmHeadInputDtype, recorder, {
811
1092
  executionPolicies: config.executionPolicies ?? null,
@@ -829,23 +1110,26 @@ export async function recordLogitsGPU(
829
1110
  lmHeadBufferOwned = true;
830
1111
  }
831
1112
 
832
- // Record matmul (no submit)
833
- const logitsTensor = await recordMatmul(recorder, lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
834
- transposeB: 'auto',
835
- role: 'lm_head',
836
- kernelPath: stableKernelPath,
837
- outputDtype: lmHeadOutputDtype,
838
- executionPolicies: config.executionPolicies ?? null,
839
- });
840
- await runProbes('logits', logitsTensor.buffer, {
841
- numTokens,
842
- hiddenSize: matmulVocabSize,
843
- recorder,
844
- operatorDiagnostics,
845
- dtype: logitsTensor.dtype,
846
- });
1113
+ return {
1114
+ hiddenSize,
1115
+ matmulVocabSize,
1116
+ phase,
1117
+ stableKernelPath,
1118
+ lmHeadRole,
1119
+ lmHeadOutputDtype,
1120
+ normedTensor,
1121
+ finalNormTensor,
1122
+ normInputTensor,
1123
+ normInputOwned,
1124
+ normWeightBuffer,
1125
+ normWeightOwned,
1126
+ lmHeadInputTensor,
1127
+ lmHeadBuffer,
1128
+ lmHeadBufferOwned,
1129
+ };
1130
+ }
847
1131
 
848
- // Track intermediate buffer for cleanup after submit
1132
+ function trackRecordedLogitsTail(recorder, tail) {
849
1133
  const trackedTempBuffers = new Set();
850
1134
  const trackTempBufferOnce = (buffer) => {
851
1135
  if (!buffer || trackedTempBuffers.has(buffer)) {
@@ -854,22 +1138,89 @@ export async function recordLogitsGPU(
854
1138
  trackedTempBuffers.add(buffer);
855
1139
  recorder.trackTemporaryBuffer(buffer);
856
1140
  };
857
- if (finalNormTensor !== normedTensor) {
858
- trackTempBufferOnce(normedTensor.buffer);
1141
+ if (tail.finalNormTensor !== tail.normedTensor) {
1142
+ trackTempBufferOnce(tail.normedTensor.buffer);
859
1143
  }
860
- trackTempBufferOnce(finalNormTensor.buffer);
861
- if (lmHeadInputTensor !== finalNormTensor) {
862
- trackTempBufferOnce(lmHeadInputTensor.buffer);
1144
+ trackTempBufferOnce(tail.finalNormTensor.buffer);
1145
+ if (tail.lmHeadInputTensor !== tail.finalNormTensor) {
1146
+ trackTempBufferOnce(tail.lmHeadInputTensor.buffer);
863
1147
  }
864
- if (normWeightOwned) {
865
- recorder.trackTemporaryBuffer(normWeightBuffer);
1148
+ if (tail.normWeightOwned) {
1149
+ recorder.trackTemporaryBuffer(tail.normWeightBuffer);
866
1150
  }
867
- if (normInputOwned) {
868
- recorder.trackTemporaryBuffer(normInputTensor.buffer);
1151
+ if (tail.normInputOwned) {
1152
+ recorder.trackTemporaryBuffer(tail.normInputTensor.buffer);
869
1153
  }
870
- if (lmHeadBufferOwned) {
871
- recorder.trackTemporaryBuffer(isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer);
1154
+ if (tail.lmHeadBufferOwned) {
1155
+ recorder.trackTemporaryBuffer(isWeightBuffer(tail.lmHeadBuffer) ? tail.lmHeadBuffer.buffer : tail.lmHeadBuffer);
872
1156
  }
1157
+ }
873
1158
 
874
- return { logitsBuffer: logitsTensor.buffer, vocabSize: matmulVocabSize, logitsDtype: logitsTensor.dtype };
1159
+ export async function recordLogitsGPU(
1160
+ recorder,
1161
+ hiddenStates,
1162
+ numTokens,
1163
+ weights,
1164
+ config,
1165
+ operatorDiagnostics = null,
1166
+ ) {
1167
+ const tail = await recordLogitsTailGPU(
1168
+ recorder,
1169
+ hiddenStates,
1170
+ numTokens,
1171
+ weights,
1172
+ config,
1173
+ operatorDiagnostics
1174
+ );
1175
+
1176
+ const logitsTensor = await recordMatmul(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, numTokens, tail.matmulVocabSize, tail.hiddenSize, {
1177
+ transposeB: 'auto',
1178
+ role: tail.lmHeadRole,
1179
+ kernelPath: tail.stableKernelPath,
1180
+ outputDtype: tail.lmHeadOutputDtype,
1181
+ executionPolicies: config.executionPolicies ?? null,
1182
+ });
1183
+ await runProbes('logits', logitsTensor.buffer, {
1184
+ numTokens,
1185
+ hiddenSize: tail.matmulVocabSize,
1186
+ recorder,
1187
+ operatorDiagnostics,
1188
+ dtype: logitsTensor.dtype,
1189
+ });
1190
+
1191
+ trackRecordedLogitsTail(recorder, tail);
1192
+
1193
+ return { logitsBuffer: logitsTensor.buffer, vocabSize: tail.matmulVocabSize, logitsDtype: logitsTensor.dtype };
1194
+ }
1195
+
1196
+ export async function recordGreedyLmHeadArgmaxGPU(
1197
+ recorder,
1198
+ hiddenStates,
1199
+ numTokens,
1200
+ weights,
1201
+ config,
1202
+ options,
1203
+ operatorDiagnostics = null,
1204
+ ) {
1205
+ if (numTokens !== 1) {
1206
+ throw new Error(`[recordGreedyLmHeadArgmaxGPU] expected numTokens=1, got ${numTokens}.`);
1207
+ }
1208
+ const tail = await recordLogitsTailGPU(
1209
+ recorder,
1210
+ hiddenStates,
1211
+ numTokens,
1212
+ weights,
1213
+ config,
1214
+ operatorDiagnostics
1215
+ );
1216
+ const outputBuffer = await recordLmHeadArgmaxF16(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
1217
+ vocabSize: tail.matmulVocabSize,
1218
+ hiddenSize: tail.hiddenSize,
1219
+ padTokenId: options.padTokenId,
1220
+ logitSoftcap: options.logitSoftcap,
1221
+ outputBuffer: options.outputBuffer,
1222
+ outputIndex: options.outputIndex,
1223
+ });
1224
+ trackRecordedLogitsTail(recorder, tail);
1225
+ return outputBuffer;
875
1226
  }
@@ -20,7 +20,7 @@ export type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
20
20
  export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
21
21
 
22
22
  // Re-export GPU functions
23
- export { computeLogitsGPU, recordLogitsGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
23
+ export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
24
24
 
25
25
  // Re-export utilities
26
26
  export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from './utils.js';
@@ -32,7 +32,7 @@ export interface ComputeLogitsOptions {
32
32
  export interface ResolvedLmHeadMatmulConfig {
33
33
  lastPositionOnly: boolean;
34
34
  matmulRows: number;
35
- phaseOverride: 'decode' | null;
35
+ phaseOverride: 'decode' | 'prefill' | null;
36
36
  }
37
37
 
38
38
  export function resolveLmHeadMatmulConfig(