doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -21,8 +21,8 @@ import { isStopToken } from './init.js';
21
21
  import { embed } from './embed.js';
22
22
  import { resolvePerLayerInputsSession } from './generator-helpers.js';
23
23
  import { processLayer } from './layer.js';
24
- import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
25
- import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
24
+ import { computeLogits, computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
25
+ import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype, getWeightMetadata } from '../../../gpu/weight-buffer.js';
26
26
  import { decodeReadback } from './debug-utils/index.js';
27
27
  import { getFinalNormWeights, extractEmbeddingFromHidden } from './generator-runtime.js';
28
28
  import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
@@ -33,10 +33,13 @@ import {
33
33
  createPleBufferCache,
34
34
  prefetchPerLayerRow,
35
35
  hasRangeBackedPerLayerInputEmbeddings,
36
+ hasGpuSplitPerLayerInputEmbeddings,
36
37
  getPleHotVocabularyRuntime,
37
38
  } from './per-layer-inputs.js';
38
39
 
39
40
  const UNKNOWN_TOKEN_TEXT = '<unknown>';
41
+ const FINITENESS_RESET_WORDS = new Uint32Array(4);
42
+ const FINITENESS_STATUS_BYTES = FINITENESS_RESET_WORDS.byteLength;
40
43
 
41
44
  export function sumProfileTimings(timings) {
42
45
  if (!timings || Object.keys(timings).length === 0) return null;
@@ -49,6 +52,23 @@ export function sumProfileTimings(timings) {
49
52
  return total;
50
53
  }
51
54
 
55
+ function mergeRecordOpLabelCounts(target, source) {
56
+ const merged = target && typeof target === 'object' && !Array.isArray(target)
57
+ ? target
58
+ : {};
59
+ if (!source || typeof source !== 'object' || Array.isArray(source)) {
60
+ return merged;
61
+ }
62
+ for (const [label, rawCount] of Object.entries(source)) {
63
+ const count = Number(rawCount);
64
+ if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
65
+ continue;
66
+ }
67
+ merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
68
+ }
69
+ return merged;
70
+ }
71
+
52
72
  function getEffectiveActivationDtype(state, opts) {
53
73
  const executionPlanDtype = opts?.executionPlan?.activationDtype;
54
74
  if (executionPlanDtype !== undefined && executionPlanDtype !== null) {
@@ -174,7 +194,53 @@ export function shouldUseFusedDecodeSampling(config) {
174
194
  && !hasConvLayers(config.layerTypes ?? []);
175
195
  }
176
196
 
177
- function resolveBatchStop(tokens, stopFlags, stopTokenIds, eosTokenId) {
197
+ function shouldUseGreedyLmHeadArgmaxFusion(state, opts, samplingDefaults, repetitionPenalty) {
198
+ const probes = state.runtimeConfig?.shared?.debug?.probes;
199
+ return state.runtimeConfig?.inference?.session?.useGreedyLmHeadArgmaxFusion === true
200
+ && opts.temperature < samplingDefaults.greedyThreshold
201
+ && repetitionPenalty === 1.0
202
+ && state.operatorDiagnostics == null
203
+ && !(Array.isArray(probes) && probes.length > 0);
204
+ }
205
+
206
+ export function createStopTokenLookup(stopTokenIds, eosTokenId) {
207
+ if (!Array.isArray(stopTokenIds)) {
208
+ throw new Error('[Pipeline] stopTokenIds must be an array.');
209
+ }
210
+
211
+ const tokenIds = [];
212
+ for (const tokenId of stopTokenIds) {
213
+ if (!tokenIds.includes(tokenId)) {
214
+ tokenIds.push(tokenId);
215
+ }
216
+ }
217
+ if (typeof eosTokenId === 'number' && !tokenIds.includes(eosTokenId)) {
218
+ tokenIds.push(eosTokenId);
219
+ }
220
+
221
+ if (tokenIds.length === 0) {
222
+ return { firstTokenId: null, secondTokenId: null, tokenSet: null };
223
+ }
224
+ if (tokenIds.length === 1) {
225
+ return { firstTokenId: tokenIds[0], secondTokenId: null, tokenSet: null };
226
+ }
227
+ if (tokenIds.length === 2) {
228
+ return { firstTokenId: tokenIds[0], secondTokenId: tokenIds[1], tokenSet: null };
229
+ }
230
+ return { firstTokenId: null, secondTokenId: null, tokenSet: new Set(tokenIds) };
231
+ }
232
+
233
+ function lookupHasStopToken(token, stopTokenLookup) {
234
+ if (stopTokenLookup.tokenSet) {
235
+ return stopTokenLookup.tokenSet.has(token);
236
+ }
237
+ if (stopTokenLookup.firstTokenId !== null && token === stopTokenLookup.firstTokenId) {
238
+ return true;
239
+ }
240
+ return stopTokenLookup.secondTokenId !== null && token === stopTokenLookup.secondTokenId;
241
+ }
242
+
243
+ export function resolveBatchStop(tokens, stopFlags, stopTokenLookup) {
178
244
  let actualCount = tokens.length;
179
245
  if (stopFlags) {
180
246
  const maxFlags = Math.min(stopFlags.length, tokens.length);
@@ -187,7 +253,7 @@ function resolveBatchStop(tokens, stopFlags, stopTokenIds, eosTokenId) {
187
253
  }
188
254
 
189
255
  for (let i = 0; i < actualCount; i++) {
190
- if (isStopToken(tokens[i], stopTokenIds, eosTokenId)) {
256
+ if (lookupHasStopToken(tokens[i], stopTokenLookup)) {
191
257
  actualCount = i + 1;
192
258
  break;
193
259
  }
@@ -215,21 +281,34 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
215
281
  const hasFinitenessBuffer = options.hasFinitenessBuffer === true;
216
282
  const ring = options.ring ?? null;
217
283
  const cleanupRecorder = options.cleanupRecorder ?? null;
284
+ const timing = {
285
+ mapWaitMs: 0,
286
+ cleanupMs: 0,
287
+ copyMs: 0,
288
+ };
218
289
  let mapped = false;
219
290
  let cleanupCompleted = false;
220
291
 
221
292
  try {
293
+ const mapStart = performance.now();
222
294
  await stagingBuffer.mapAsync(GPUMapMode.READ);
295
+ timing.mapWaitMs = performance.now() - mapStart;
223
296
  mapped = true;
297
+ const cleanupStart = performance.now();
224
298
  await cleanupRecorder?.completeDeferredCleanup();
299
+ timing.cleanupMs = performance.now() - cleanupStart;
225
300
  cleanupCompleted = true;
301
+ const copyStart = performance.now();
226
302
  const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
227
- return {
303
+ const result = {
228
304
  nextToken: mappedWords[0],
229
305
  finitenessStatus: hasFinitenessBuffer
230
306
  ? parseFinitenessStatusWords(mappedWords, 1)
231
307
  : parseFinitenessStatusWords(mappedWords, 0),
308
+ timing,
232
309
  };
310
+ timing.copyMs = performance.now() - copyStart;
311
+ return result;
233
312
  } finally {
234
313
  if (mapped) {
235
314
  stagingBuffer.unmap();
@@ -268,6 +347,7 @@ export async function readBatchTokensFromStagingBuffers(options) {
268
347
  tokensStagingBuffer,
269
348
  stopStagingBuffer = null,
270
349
  finitenessStagingBuffer = null,
350
+ finitenessOffsetBytes = null,
271
351
  tokenCount,
272
352
  ownsTokensStaging = false,
273
353
  ownsStopStaging = false,
@@ -279,41 +359,60 @@ export async function readBatchTokensFromStagingBuffers(options) {
279
359
  let stopMapped = false;
280
360
  let finitenessMapped = false;
281
361
  let cleanupCompleted = false;
362
+ const hasPackedFiniteness = Number.isFinite(finitenessOffsetBytes) && finitenessOffsetBytes >= 0;
363
+ const timing = {
364
+ mapWaitMs: 0,
365
+ cleanupMs: 0,
366
+ copyMs: 0,
367
+ };
282
368
 
283
369
  try {
370
+ const mapStart = performance.now();
284
371
  const mapPromises = [tokensStagingBuffer.mapAsync(GPUMapMode.READ)];
285
372
  if (stopStagingBuffer) {
286
373
  mapPromises.push(stopStagingBuffer.mapAsync(GPUMapMode.READ));
287
374
  }
288
- if (finitenessStagingBuffer) {
375
+ if (finitenessStagingBuffer && !hasPackedFiniteness) {
289
376
  mapPromises.push(finitenessStagingBuffer.mapAsync(GPUMapMode.READ));
290
377
  }
291
378
  const mapResults = await Promise.allSettled(mapPromises);
379
+ timing.mapWaitMs = performance.now() - mapStart;
292
380
  tokensMapped = mapResults[0]?.status === 'fulfilled';
293
381
  stopMapped = Boolean(stopStagingBuffer) && mapResults[1]?.status === 'fulfilled';
294
- finitenessMapped = Boolean(finitenessStagingBuffer)
382
+ finitenessMapped = Boolean(finitenessStagingBuffer) && !hasPackedFiniteness
295
383
  && mapResults[stopStagingBuffer ? 2 : 1]?.status === 'fulfilled';
296
384
  const mapFailure = mapResults.find((result) => result.status === 'rejected');
297
385
  if (mapFailure) {
298
386
  throw mapFailure.reason;
299
387
  }
388
+ const cleanupStart = performance.now();
300
389
  await cleanupRecorder?.completeDeferredCleanup();
390
+ timing.cleanupMs = performance.now() - cleanupStart;
301
391
  cleanupCompleted = true;
302
392
 
303
- const tokens = Array.from(
304
- new Uint32Array(tokensStagingBuffer.getMappedRange()).subarray(0, tokenCount)
305
- );
306
- const stopFlags = stopStagingBuffer
307
- ? new Uint32Array(stopStagingBuffer.getMappedRange().slice(0, tokenCount * 4))
308
- : null;
309
- const finitenessStatus = finitenessStagingBuffer
393
+ const copyStart = performance.now();
394
+ const tokensRange = tokensStagingBuffer.getMappedRange();
395
+ const tokenWords = new Uint32Array(tokensRange, 0, tokenCount);
396
+ const tokens = new Uint32Array(tokenWords.length);
397
+ tokens.set(tokenWords);
398
+ let stopFlags = null;
399
+ if (stopStagingBuffer) {
400
+ const stopWords = new Uint32Array(stopStagingBuffer.getMappedRange()).subarray(0, tokenCount);
401
+ stopFlags = new Uint32Array(stopWords.length);
402
+ stopFlags.set(stopWords);
403
+ }
404
+ const finitenessStatus = hasPackedFiniteness
405
+ ? parseFinitenessStatusWords(new Uint32Array(tokensRange, finitenessOffsetBytes, 4), 0)
406
+ : finitenessStagingBuffer
310
407
  ? parseFinitenessStatusWords(new Uint32Array(finitenessStagingBuffer.getMappedRange()), 0)
311
408
  : { triggered: false, metadata: '' };
409
+ timing.copyMs = performance.now() - copyStart;
312
410
 
313
411
  return {
314
412
  tokens,
315
413
  stopFlags,
316
414
  finitenessStatus,
415
+ timing,
317
416
  };
318
417
  } finally {
319
418
  if (finitenessMapped) {
@@ -354,13 +453,14 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
354
453
  const decodeAltBuffer = state.decodeBuffers.getOutputHiddenBuffer();
355
454
 
356
455
  const embedBufferRaw = state.weights.get('embed');
357
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
456
+ if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
358
457
  throw new Error('Embed buffer not found or not a supported buffer type');
359
458
  }
360
459
  const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
361
460
  const embedDtype = isCpuWeightBuffer(embedBufferRaw)
362
461
  ? embedBufferRaw.dtype
363
462
  : getWeightDtype(embedBufferRaw);
463
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
364
464
  const activationDtype = getEffectiveActivationDtype(state, opts);
365
465
 
366
466
  const embedTensor = await embed([tokenId], embedBuffer, {
@@ -372,7 +472,8 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
372
472
  debugProbes: state.runtimeConfig.shared.debug.probes,
373
473
  operatorDiagnostics: state.operatorDiagnostics,
374
474
  activationDtype,
375
- embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
475
+ embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
476
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
376
477
  executionPolicies: state.executionV1State?.policies ?? null,
377
478
  });
378
479
 
@@ -437,8 +538,8 @@ function createDecodeRecorder(state, opts) {
437
538
  let recorder;
438
539
  if (recorderEnabled) {
439
540
  recorder = opts.profile
440
- ? createProfilingRecorder('decode')
441
- : createCommandRecorder('decode');
541
+ ? createProfilingRecorder('decode', device)
542
+ : createCommandRecorder('decode', { recordLabels: opts.debug === true }, device);
442
543
  }
443
544
  if (state.decodeStepCount === 1) {
444
545
  const path = selectRuleValue('inference', 'config', 'tracePath', { useRecorder: Boolean(recorder) });
@@ -491,7 +592,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
491
592
  const recorder = createDecodeRecorder(state, opts);
492
593
 
493
594
  if (state.finitenessBuffer && device) {
494
- device.queue.writeBuffer(state.finitenessBuffer, 0, new Uint32Array([0, 0, 0, 0]));
595
+ device.queue.writeBuffer(state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
495
596
  }
496
597
 
497
598
  const context = helpers.buildLayerContext(recorder, true, opts.debugLayers, executionPlan);
@@ -503,13 +604,14 @@ export async function decodeStep(state, currentIds, opts, helpers) {
503
604
  const decodeAltBuffer = state.decodeBuffers.getOutputHiddenBuffer();
504
605
 
505
606
  const embedBufferRaw = state.weights.get('embed');
506
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
607
+ if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
507
608
  throw new Error('Embed buffer not found or not a supported buffer type');
508
609
  }
509
610
  const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
510
611
  const embedDtype = isCpuWeightBuffer(embedBufferRaw)
511
612
  ? embedBufferRaw.dtype
512
613
  : getWeightDtype(embedBufferRaw);
614
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
513
615
  const activationDtype = getEffectiveActivationDtype(state, opts);
514
616
  const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
515
617
 
@@ -523,7 +625,8 @@ export async function decodeStep(state, currentIds, opts, helpers) {
523
625
  debugProbes: state.runtimeConfig.shared.debug.probes,
524
626
  operatorDiagnostics: state.operatorDiagnostics,
525
627
  activationDtype,
526
- embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
628
+ embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
629
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
527
630
  executionPolicies: state.executionV1State?.policies ?? null,
528
631
  });
529
632
 
@@ -708,6 +811,14 @@ export async function decodeStep(state, currentIds, opts, helpers) {
708
811
 
709
812
  state.stats.singleTokenSubmitWaitMs = (state.stats.singleTokenSubmitWaitMs ?? 0) + submitWaitMs;
710
813
  state.stats.singleTokenReadbackWaitMs = (state.stats.singleTokenReadbackWaitMs ?? 0) + readbackWaitMs;
814
+ if (readbackResult.timing) {
815
+ state.stats.singleTokenReadbackMapWaitMs = (state.stats.singleTokenReadbackMapWaitMs ?? 0)
816
+ + readbackResult.timing.mapWaitMs;
817
+ state.stats.singleTokenReadbackCleanupMs = (state.stats.singleTokenReadbackCleanupMs ?? 0)
818
+ + readbackResult.timing.cleanupMs;
819
+ state.stats.singleTokenReadbackCopyMs = (state.stats.singleTokenReadbackCopyMs ?? 0)
820
+ + readbackResult.timing.copyMs;
821
+ }
711
822
 
712
823
  const { nextToken: fusedNextToken, finitenessStatus } = readbackResult;
713
824
 
@@ -1209,8 +1320,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1209
1320
  state.batchingStats.batchedForwardCalls += 1;
1210
1321
  const tokensPerInterval = batchSize * readbackInterval;
1211
1322
  const recorder = opts.profile
1212
- ? createProfilingRecorder('batch_decode')
1213
- : createCommandRecorder('batch_decode');
1323
+ ? createProfilingRecorder('batch_decode', device)
1324
+ : createCommandRecorder('batch_decode', { recordLabels: opts.debug === true }, device);
1214
1325
  const lmHead = state.weights.get('lm_head');
1215
1326
  if (lmHead && isCpuWeightBuffer(lmHead)) {
1216
1327
  throw new Error('[Pipeline] GPU-only decode not supported with CPU-resident LM head.');
@@ -1227,6 +1338,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1227
1338
  config,
1228
1339
  weights: state.weights,
1229
1340
  });
1341
+ const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
1342
+ config,
1343
+ weights: state.weights,
1344
+ });
1230
1345
  const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: state.weights });
1231
1346
  const hotStartTokenIndex = pleHotVocabularyRuntime?.hotTokenIndexMap?.[startToken] ?? null;
1232
1347
  const canUseHotVocabularyBatchDecode = hasRangeBackedPerLayerInputs
@@ -1260,6 +1375,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1260
1375
  if (eosTokenId == null) {
1261
1376
  throw new Error('[Pipeline] Missing EOS token. Ensure tokenizer or manifest provides stop tokens.');
1262
1377
  }
1378
+ const stopTokenLookup = createStopTokenLookup(stopTokenIds, eosToken);
1263
1379
  const maxTokens = executionPlan?.maxTokens
1264
1380
  ?? opts.maxTokens
1265
1381
  ?? state.runtimeConfig.inference.generation.maxTokens;
@@ -1304,8 +1420,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1304
1420
  })
1305
1421
  : null;
1306
1422
 
1423
+ const tokenReadbackBytes = N * 4;
1424
+ const tokenStagingBytes = tokenReadbackBytes + (state.finitenessBuffer ? FINITENESS_STATUS_BYTES : 0);
1307
1425
  const tokensStagingBuffer = ringSlot?.stagingTokens ?? device.createBuffer({
1308
- size: N * 4,
1426
+ size: tokenStagingBytes,
1309
1427
  usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
1310
1428
  });
1311
1429
  const ownsTokensStaging = !ringSlot?.stagingTokens;
@@ -1317,24 +1435,33 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1317
1435
  })
1318
1436
  : null;
1319
1437
  const ownsStopStaging = useGpuStopFlags && !ringSlot?.stagingStop;
1320
- const finitenessStagingBuffer = state.finitenessBuffer
1438
+ const packedFinitenessOffsetBytes = state.finitenessBuffer
1439
+ && tokensStagingBuffer.size >= tokenReadbackBytes + FINITENESS_STATUS_BYTES
1440
+ ? tokenReadbackBytes
1441
+ : null;
1442
+ const finitenessStagingBuffer = state.finitenessBuffer && packedFinitenessOffsetBytes == null
1321
1443
  ? ringSlot?.stagingFiniteness ?? device.createBuffer({
1322
- size: 16,
1444
+ size: FINITENESS_STATUS_BYTES,
1323
1445
  usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
1324
1446
  })
1325
1447
  : null;
1326
- const ownsFinitenessStaging = Boolean(state.finitenessBuffer) && !ringSlot?.stagingFiniteness;
1448
+ const ownsFinitenessStaging = Boolean(state.finitenessBuffer)
1449
+ && packedFinitenessOffsetBytes == null
1450
+ && !ringSlot?.stagingFiniteness;
1327
1451
  let readbackCleanupDelegated = false;
1328
1452
  let repHistoryBuffer = null;
1329
1453
  let repHistoryCount = 0;
1330
1454
  try {
1331
1455
  if (state.finitenessBuffer) {
1332
- device.queue.writeBuffer(state.finitenessBuffer, 0, new Uint32Array([0, 0, 0, 0]));
1456
+ device.queue.writeBuffer(state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
1333
1457
  }
1334
1458
 
1335
- device.queue.writeBuffer(tokensBuffer, 0, new Uint32Array([startToken]));
1459
+ const singleWordUpload = new Uint32Array(1);
1460
+ singleWordUpload[0] = startToken;
1461
+ device.queue.writeBuffer(tokensBuffer, 0, singleWordUpload);
1336
1462
  if (pleInputTokensBuffer) {
1337
- device.queue.writeBuffer(pleInputTokensBuffer, 0, new Uint32Array([hotStartTokenIndex]));
1463
+ singleWordUpload[0] = hotStartTokenIndex;
1464
+ device.queue.writeBuffer(pleInputTokensBuffer, 0, singleWordUpload);
1338
1465
  }
1339
1466
  if (stopBuffer) {
1340
1467
  const stopElements = stopBuffer.size / 4;
@@ -1350,11 +1477,12 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1350
1477
  if (isCpuWeightBuffer(embedBufferRaw)) {
1351
1478
  throw new Error('[Pipeline] GPU-only decode not supported with CPU-resident embeddings.');
1352
1479
  }
1353
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw)) {
1480
+ if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw)) {
1354
1481
  throw new Error('Embed buffer not found or not a GPUBuffer/WeightBuffer');
1355
1482
  }
1356
1483
  const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
1357
1484
  const embedDtype = getWeightDtype(embedBufferRaw);
1485
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
1358
1486
  const activationDtype = getEffectiveActivationDtype(state, opts);
1359
1487
 
1360
1488
  // GPU-side repetition penalty: upload deduplicated history before batch
@@ -1373,9 +1501,15 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1373
1501
  }
1374
1502
 
1375
1503
  // Hoist loop-invariant values to avoid repeated rule lookups and allocations.
1376
- const embeddingDtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype });
1504
+ const embeddingDtype = selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype });
1377
1505
  const debugProbes = state.runtimeConfig.shared.debug.probes;
1378
1506
  const currentTokenIdsArray = [startToken];
1507
+ const useGreedyLmHeadArgmaxFusion = shouldUseGreedyLmHeadArgmaxFusion(
1508
+ state,
1509
+ opts,
1510
+ samplingDefaults,
1511
+ repetitionPenalty
1512
+ );
1379
1513
 
1380
1514
  for (let i = 0; i < N; i++) {
1381
1515
  // In the GPU batch path, only the start token (i=0) is known on the CPU.
@@ -1394,11 +1528,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1394
1528
  vocabSize: config.vocabSize,
1395
1529
  scaleEmbeddings: config.scaleEmbeddings,
1396
1530
  recorder,
1531
+ outputBuffer: context.decodeBuffers?.getHiddenBuffer() ?? undefined,
1397
1532
  transpose: state.embeddingTranspose,
1398
1533
  debugProbes,
1399
1534
  operatorDiagnostics: state.operatorDiagnostics,
1400
1535
  activationDtype,
1401
1536
  embeddingDtype,
1537
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
1402
1538
  executionPolicies: state.executionV1State?.policies ?? null,
1403
1539
  numTokens: 1,
1404
1540
  indexOffset: i,
@@ -1408,7 +1544,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1408
1544
  const perLayerInputs = await preparePerLayerInputs(tokensBuffer, hiddenTensor, context, {
1409
1545
  numTokens: 1,
1410
1546
  indexOffset: i,
1411
- perLayerTokenIds: pleInputTokensBuffer,
1547
+ perLayerTokenIds: pleInputTokensBuffer ?? (hasGpuSplitPerLayerInputs ? tokensBuffer : null),
1412
1548
  perLayerIndexOffset: i,
1413
1549
  tokenIdHint: i === 0 ? startToken : null,
1414
1550
  pleCache: state.pleCache ?? null,
@@ -1451,48 +1587,67 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1451
1587
  helpers.releaseSharedAttentionState?.(context.sharedAttentionState, recorder);
1452
1588
  }
1453
1589
 
1454
- const logits = await recordLogitsGPU(
1455
- recorder,
1456
- hiddenStatesBuffer,
1457
- 1,
1458
- helpers.getLogitsWeights(),
1459
- helpers.getLogitsConfig(),
1460
- state.operatorDiagnostics
1461
- );
1462
- const { logitsBuffer, vocabSize, logitsDtype } = logits;
1463
-
1464
- // Apply GPU-side repetition penalty before sampling
1465
- if (repHistoryBuffer && repetitionPenalty !== 1.0) {
1466
- await recordRepPenalty(recorder, logitsBuffer, repHistoryBuffer, tokensBuffer, {
1467
- vocabSize,
1468
- historyCount: repHistoryCount,
1469
- penalty: repetitionPenalty,
1470
- batchCount: i,
1471
- batchOffset: 1,
1472
- logitsDtype,
1473
- });
1474
- }
1475
-
1476
1590
  const outputIndex = i + 1;
1477
- if (opts.temperature < samplingDefaults.greedyThreshold) {
1478
- await recordArgmax(recorder, logitsBuffer, vocabSize, {
1479
- padTokenId,
1480
- logitSoftcap,
1481
- logitsDtype,
1482
- outputBuffer: tokensBuffer,
1483
- outputIndex,
1484
- });
1591
+ let logitsBuffer = null;
1592
+ if (useGreedyLmHeadArgmaxFusion) {
1593
+ await recordGreedyLmHeadArgmaxGPU(
1594
+ recorder,
1595
+ hiddenStatesBuffer,
1596
+ 1,
1597
+ helpers.getLogitsWeights(),
1598
+ helpers.getLogitsConfig(),
1599
+ {
1600
+ padTokenId,
1601
+ logitSoftcap,
1602
+ outputBuffer: tokensBuffer,
1603
+ outputIndex,
1604
+ },
1605
+ state.operatorDiagnostics
1606
+ );
1485
1607
  } else {
1486
- await recordGPUSample(recorder, logitsBuffer, vocabSize, {
1487
- temperature: opts.temperature,
1488
- topK: opts.topK,
1489
- padTokenId,
1490
- logitSoftcap,
1491
- logitsDtype,
1492
- outputBuffer: tokensBuffer,
1493
- outputIndex,
1494
- greedyThreshold: samplingDefaults.greedyThreshold,
1495
- });
1608
+ const logits = await recordLogitsGPU(
1609
+ recorder,
1610
+ hiddenStatesBuffer,
1611
+ 1,
1612
+ helpers.getLogitsWeights(),
1613
+ helpers.getLogitsConfig(),
1614
+ state.operatorDiagnostics
1615
+ );
1616
+ const { vocabSize, logitsDtype } = logits;
1617
+ logitsBuffer = logits.logitsBuffer;
1618
+
1619
+ // Apply GPU-side repetition penalty before sampling
1620
+ if (repHistoryBuffer && repetitionPenalty !== 1.0) {
1621
+ await recordRepPenalty(recorder, logitsBuffer, repHistoryBuffer, tokensBuffer, {
1622
+ vocabSize,
1623
+ historyCount: repHistoryCount,
1624
+ penalty: repetitionPenalty,
1625
+ batchCount: i,
1626
+ batchOffset: 1,
1627
+ logitsDtype,
1628
+ });
1629
+ }
1630
+
1631
+ if (opts.temperature < samplingDefaults.greedyThreshold) {
1632
+ await recordArgmax(recorder, logitsBuffer, vocabSize, {
1633
+ padTokenId,
1634
+ logitSoftcap,
1635
+ logitsDtype,
1636
+ outputBuffer: tokensBuffer,
1637
+ outputIndex,
1638
+ });
1639
+ } else {
1640
+ await recordGPUSample(recorder, logitsBuffer, vocabSize, {
1641
+ temperature: opts.temperature,
1642
+ topK: opts.topK,
1643
+ padTokenId,
1644
+ logitSoftcap,
1645
+ logitsDtype,
1646
+ outputBuffer: tokensBuffer,
1647
+ outputIndex,
1648
+ greedyThreshold: samplingDefaults.greedyThreshold,
1649
+ });
1650
+ }
1496
1651
  }
1497
1652
 
1498
1653
  const stopCheck = canUseHotVocabularyBatchDecode
@@ -1530,7 +1685,16 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1530
1685
  }
1531
1686
 
1532
1687
  const recordMs = performance.now() - recordStart;
1688
+ const recordStats = recorder.getStats();
1689
+ const recordOps = recordStats.opCount;
1690
+ const recordPasses = recordStats.computePassCount;
1533
1691
  state.stats.decodeRecordMs = (state.stats.decodeRecordMs ?? 0) + recordMs;
1692
+ state.stats.decodeRecordOps = (state.stats.decodeRecordOps ?? 0) + recordOps;
1693
+ state.stats.decodeRecordPasses = (state.stats.decodeRecordPasses ?? 0) + recordPasses;
1694
+ state.stats.decodeRecordOpLabels = mergeRecordOpLabelCounts(
1695
+ state.stats.decodeRecordOpLabels,
1696
+ recordStats.opLabelCounts
1697
+ );
1534
1698
 
1535
1699
  const encoder = recorder.getEncoder();
1536
1700
  encoder.copyBufferToBuffer(tokensBuffer, 4, tokensStagingBuffer, 0, N * 4);
@@ -1538,8 +1702,22 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1538
1702
  encoder.copyBufferToBuffer(stopBuffer, 4, stopStagingBuffer, 0, N * 4);
1539
1703
  }
1540
1704
 
1541
- if (state.finitenessBuffer && finitenessStagingBuffer) {
1542
- encoder.copyBufferToBuffer(state.finitenessBuffer, 0, finitenessStagingBuffer, 0, 16);
1705
+ if (state.finitenessBuffer && packedFinitenessOffsetBytes != null) {
1706
+ encoder.copyBufferToBuffer(
1707
+ state.finitenessBuffer,
1708
+ 0,
1709
+ tokensStagingBuffer,
1710
+ packedFinitenessOffsetBytes,
1711
+ FINITENESS_STATUS_BYTES
1712
+ );
1713
+ } else if (state.finitenessBuffer && finitenessStagingBuffer) {
1714
+ encoder.copyBufferToBuffer(
1715
+ state.finitenessBuffer,
1716
+ 0,
1717
+ finitenessStagingBuffer,
1718
+ 0,
1719
+ FINITENESS_STATUS_BYTES
1720
+ );
1543
1721
  }
1544
1722
 
1545
1723
  if (!allowReadback('pipeline.decode.sample')) {
@@ -1554,6 +1732,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1554
1732
  tokensStagingBuffer,
1555
1733
  stopStagingBuffer,
1556
1734
  finitenessStagingBuffer,
1735
+ finitenessOffsetBytes: packedFinitenessOffsetBytes,
1557
1736
  tokenCount: N,
1558
1737
  ownsTokensStaging,
1559
1738
  ownsStopStaging,
@@ -1563,6 +1742,14 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1563
1742
  });
1564
1743
  const readbackWaitMs = performance.now() - readbackStart;
1565
1744
  state.stats.decodeReadbackWaitMs = (state.stats.decodeReadbackWaitMs ?? 0) + readbackWaitMs;
1745
+ if (readback.timing) {
1746
+ state.stats.decodeReadbackMapWaitMs = (state.stats.decodeReadbackMapWaitMs ?? 0)
1747
+ + readback.timing.mapWaitMs;
1748
+ state.stats.decodeReadbackCleanupMs = (state.stats.decodeReadbackCleanupMs ?? 0)
1749
+ + readback.timing.cleanupMs;
1750
+ state.stats.decodeReadbackCopyMs = (state.stats.decodeReadbackCopyMs ?? 0)
1751
+ + readback.timing.copyMs;
1752
+ }
1566
1753
 
1567
1754
  const isInfinite = readback.finitenessStatus.triggered;
1568
1755
  const metadata = readback.finitenessStatus.metadata;
@@ -1581,8 +1768,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1581
1768
  log.debug('Pipeline', `[STOP] N=${N} flags=[${Array.from(stopFlags).join(',')}] tokens=[${tokens.join(',')}] eos=${eosTokenId}`);
1582
1769
  }
1583
1770
 
1584
- const actualCount = resolveBatchStop(tokens, stopFlags, stopTokenIds, eosToken);
1585
- const generatedTokens = tokens.slice(0, actualCount);
1771
+ const actualCount = resolveBatchStop(tokens, stopFlags, stopTokenLookup);
1772
+ const generatedTokens = actualCount === tokens.length
1773
+ ? tokens
1774
+ : tokens.subarray(0, actualCount);
1586
1775
  const invalidToken = findInvalidGeneratedToken(generatedTokens, config.vocabSize, padTokenId);
1587
1776
 
1588
1777
  if (isInfinite) {
@@ -1595,6 +1784,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1595
1784
  `at batch index ${invalidToken.index} (vocabSize=${config.vocabSize}, padTokenId=${padTokenId ?? 'none'}).`
1596
1785
  );
1597
1786
  }
1787
+ state.batchingStats.executedBatchTokens = (state.batchingStats.executedBatchTokens ?? 0) + N;
1788
+ state.batchingStats.resolvedBatchTokens = (state.batchingStats.resolvedBatchTokens ?? 0) + actualCount;
1598
1789
 
1599
1790
  if (opts.profile && recorder.isProfilingEnabled()) {
1600
1791
  const timings = await recorder.resolveProfileTimings();