doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -21,8 +21,8 @@ import { isStopToken } from './init.js';
21
21
  import { embed } from './embed.js';
22
22
  import { resolvePerLayerInputsSession } from './generator-helpers.js';
23
23
  import { processLayer } from './layer.js';
24
- import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
25
- import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
24
+ import { computeLogits, computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
25
+ import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype, getWeightMetadata } from '../../../gpu/weight-buffer.js';
26
26
  import { decodeReadback } from './debug-utils/index.js';
27
27
  import { getFinalNormWeights, extractEmbeddingFromHidden } from './generator-runtime.js';
28
28
  import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
@@ -33,10 +33,13 @@ import {
33
33
  createPleBufferCache,
34
34
  prefetchPerLayerRow,
35
35
  hasRangeBackedPerLayerInputEmbeddings,
36
+ hasGpuSplitPerLayerInputEmbeddings,
36
37
  getPleHotVocabularyRuntime,
37
38
  } from './per-layer-inputs.js';
38
39
 
39
40
  const UNKNOWN_TOKEN_TEXT = '<unknown>';
41
+ const FINITENESS_RESET_WORDS = new Uint32Array(4);
42
+ const FINITENESS_STATUS_BYTES = FINITENESS_RESET_WORDS.byteLength;
40
43
 
41
44
  export function sumProfileTimings(timings) {
42
45
  if (!timings || Object.keys(timings).length === 0) return null;
@@ -49,6 +52,23 @@ export function sumProfileTimings(timings) {
49
52
  return total;
50
53
  }
51
54
 
55
+ function mergeRecordOpLabelCounts(target, source) {
56
+ const merged = target && typeof target === 'object' && !Array.isArray(target)
57
+ ? target
58
+ : {};
59
+ if (!source || typeof source !== 'object' || Array.isArray(source)) {
60
+ return merged;
61
+ }
62
+ for (const [label, rawCount] of Object.entries(source)) {
63
+ const count = Number(rawCount);
64
+ if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
65
+ continue;
66
+ }
67
+ merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
68
+ }
69
+ return merged;
70
+ }
71
+
52
72
  function getEffectiveActivationDtype(state, opts) {
53
73
  const executionPlanDtype = opts?.executionPlan?.activationDtype;
54
74
  if (executionPlanDtype !== undefined && executionPlanDtype !== null) {
@@ -174,7 +194,53 @@ export function shouldUseFusedDecodeSampling(config) {
174
194
  && !hasConvLayers(config.layerTypes ?? []);
175
195
  }
176
196
 
177
- export function resolveBatchStop(tokens, stopFlags, stopTokenIds, eosTokenId) {
197
+ function shouldUseGreedyLmHeadArgmaxFusion(state, opts, samplingDefaults, repetitionPenalty) {
198
+ const probes = state.runtimeConfig?.shared?.debug?.probes;
199
+ return state.runtimeConfig?.inference?.session?.useGreedyLmHeadArgmaxFusion === true
200
+ && opts.temperature < samplingDefaults.greedyThreshold
201
+ && repetitionPenalty === 1.0
202
+ && state.operatorDiagnostics == null
203
+ && !(Array.isArray(probes) && probes.length > 0);
204
+ }
205
+
206
+ export function createStopTokenLookup(stopTokenIds, eosTokenId) {
207
+ if (!Array.isArray(stopTokenIds)) {
208
+ throw new Error('[Pipeline] stopTokenIds must be an array.');
209
+ }
210
+
211
+ const tokenIds = [];
212
+ for (const tokenId of stopTokenIds) {
213
+ if (!tokenIds.includes(tokenId)) {
214
+ tokenIds.push(tokenId);
215
+ }
216
+ }
217
+ if (typeof eosTokenId === 'number' && !tokenIds.includes(eosTokenId)) {
218
+ tokenIds.push(eosTokenId);
219
+ }
220
+
221
+ if (tokenIds.length === 0) {
222
+ return { firstTokenId: null, secondTokenId: null, tokenSet: null };
223
+ }
224
+ if (tokenIds.length === 1) {
225
+ return { firstTokenId: tokenIds[0], secondTokenId: null, tokenSet: null };
226
+ }
227
+ if (tokenIds.length === 2) {
228
+ return { firstTokenId: tokenIds[0], secondTokenId: tokenIds[1], tokenSet: null };
229
+ }
230
+ return { firstTokenId: null, secondTokenId: null, tokenSet: new Set(tokenIds) };
231
+ }
232
+
233
+ function lookupHasStopToken(token, stopTokenLookup) {
234
+ if (stopTokenLookup.tokenSet) {
235
+ return stopTokenLookup.tokenSet.has(token);
236
+ }
237
+ if (stopTokenLookup.firstTokenId !== null && token === stopTokenLookup.firstTokenId) {
238
+ return true;
239
+ }
240
+ return stopTokenLookup.secondTokenId !== null && token === stopTokenLookup.secondTokenId;
241
+ }
242
+
243
+ export function resolveBatchStop(tokens, stopFlags, stopTokenLookup) {
178
244
  let actualCount = tokens.length;
179
245
  if (stopFlags) {
180
246
  const maxFlags = Math.min(stopFlags.length, tokens.length);
@@ -187,7 +253,7 @@ export function resolveBatchStop(tokens, stopFlags, stopTokenIds, eosTokenId) {
187
253
  }
188
254
 
189
255
  for (let i = 0; i < actualCount; i++) {
190
- if (isStopToken(tokens[i], stopTokenIds, eosTokenId)) {
256
+ if (lookupHasStopToken(tokens[i], stopTokenLookup)) {
191
257
  actualCount = i + 1;
192
258
  break;
193
259
  }
@@ -215,21 +281,34 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
215
281
  const hasFinitenessBuffer = options.hasFinitenessBuffer === true;
216
282
  const ring = options.ring ?? null;
217
283
  const cleanupRecorder = options.cleanupRecorder ?? null;
284
+ const timing = {
285
+ mapWaitMs: 0,
286
+ cleanupMs: 0,
287
+ copyMs: 0,
288
+ };
218
289
  let mapped = false;
219
290
  let cleanupCompleted = false;
220
291
 
221
292
  try {
293
+ const mapStart = performance.now();
222
294
  await stagingBuffer.mapAsync(GPUMapMode.READ);
295
+ timing.mapWaitMs = performance.now() - mapStart;
223
296
  mapped = true;
297
+ const cleanupStart = performance.now();
224
298
  await cleanupRecorder?.completeDeferredCleanup();
299
+ timing.cleanupMs = performance.now() - cleanupStart;
225
300
  cleanupCompleted = true;
301
+ const copyStart = performance.now();
226
302
  const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
227
- return {
303
+ const result = {
228
304
  nextToken: mappedWords[0],
229
305
  finitenessStatus: hasFinitenessBuffer
230
306
  ? parseFinitenessStatusWords(mappedWords, 1)
231
307
  : parseFinitenessStatusWords(mappedWords, 0),
308
+ timing,
232
309
  };
310
+ timing.copyMs = performance.now() - copyStart;
311
+ return result;
233
312
  } finally {
234
313
  if (mapped) {
235
314
  stagingBuffer.unmap();
@@ -268,6 +347,7 @@ export async function readBatchTokensFromStagingBuffers(options) {
268
347
  tokensStagingBuffer,
269
348
  stopStagingBuffer = null,
270
349
  finitenessStagingBuffer = null,
350
+ finitenessOffsetBytes = null,
271
351
  tokenCount,
272
352
  ownsTokensStaging = false,
273
353
  ownsStopStaging = false,
@@ -279,41 +359,60 @@ export async function readBatchTokensFromStagingBuffers(options) {
279
359
  let stopMapped = false;
280
360
  let finitenessMapped = false;
281
361
  let cleanupCompleted = false;
362
+ const hasPackedFiniteness = Number.isFinite(finitenessOffsetBytes) && finitenessOffsetBytes >= 0;
363
+ const timing = {
364
+ mapWaitMs: 0,
365
+ cleanupMs: 0,
366
+ copyMs: 0,
367
+ };
282
368
 
283
369
  try {
370
+ const mapStart = performance.now();
284
371
  const mapPromises = [tokensStagingBuffer.mapAsync(GPUMapMode.READ)];
285
372
  if (stopStagingBuffer) {
286
373
  mapPromises.push(stopStagingBuffer.mapAsync(GPUMapMode.READ));
287
374
  }
288
- if (finitenessStagingBuffer) {
375
+ if (finitenessStagingBuffer && !hasPackedFiniteness) {
289
376
  mapPromises.push(finitenessStagingBuffer.mapAsync(GPUMapMode.READ));
290
377
  }
291
378
  const mapResults = await Promise.allSettled(mapPromises);
379
+ timing.mapWaitMs = performance.now() - mapStart;
292
380
  tokensMapped = mapResults[0]?.status === 'fulfilled';
293
381
  stopMapped = Boolean(stopStagingBuffer) && mapResults[1]?.status === 'fulfilled';
294
- finitenessMapped = Boolean(finitenessStagingBuffer)
382
+ finitenessMapped = Boolean(finitenessStagingBuffer) && !hasPackedFiniteness
295
383
  && mapResults[stopStagingBuffer ? 2 : 1]?.status === 'fulfilled';
296
384
  const mapFailure = mapResults.find((result) => result.status === 'rejected');
297
385
  if (mapFailure) {
298
386
  throw mapFailure.reason;
299
387
  }
388
+ const cleanupStart = performance.now();
300
389
  await cleanupRecorder?.completeDeferredCleanup();
390
+ timing.cleanupMs = performance.now() - cleanupStart;
301
391
  cleanupCompleted = true;
302
392
 
303
- const tokens = Array.from(
304
- new Uint32Array(tokensStagingBuffer.getMappedRange()).subarray(0, tokenCount)
305
- );
306
- const stopFlags = stopStagingBuffer
307
- ? new Uint32Array(stopStagingBuffer.getMappedRange().slice(0, tokenCount * 4))
308
- : null;
309
- const finitenessStatus = finitenessStagingBuffer
393
+ const copyStart = performance.now();
394
+ const tokensRange = tokensStagingBuffer.getMappedRange();
395
+ const tokenWords = new Uint32Array(tokensRange, 0, tokenCount);
396
+ const tokens = new Uint32Array(tokenWords.length);
397
+ tokens.set(tokenWords);
398
+ let stopFlags = null;
399
+ if (stopStagingBuffer) {
400
+ const stopWords = new Uint32Array(stopStagingBuffer.getMappedRange()).subarray(0, tokenCount);
401
+ stopFlags = new Uint32Array(stopWords.length);
402
+ stopFlags.set(stopWords);
403
+ }
404
+ const finitenessStatus = hasPackedFiniteness
405
+ ? parseFinitenessStatusWords(new Uint32Array(tokensRange, finitenessOffsetBytes, 4), 0)
406
+ : finitenessStagingBuffer
310
407
  ? parseFinitenessStatusWords(new Uint32Array(finitenessStagingBuffer.getMappedRange()), 0)
311
408
  : { triggered: false, metadata: '' };
409
+ timing.copyMs = performance.now() - copyStart;
312
410
 
313
411
  return {
314
412
  tokens,
315
413
  stopFlags,
316
414
  finitenessStatus,
415
+ timing,
317
416
  };
318
417
  } finally {
319
418
  if (finitenessMapped) {
@@ -354,13 +453,14 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
354
453
  const decodeAltBuffer = state.decodeBuffers.getOutputHiddenBuffer();
355
454
 
356
455
  const embedBufferRaw = state.weights.get('embed');
357
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
456
+ if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
358
457
  throw new Error('Embed buffer not found or not a supported buffer type');
359
458
  }
360
459
  const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
361
460
  const embedDtype = isCpuWeightBuffer(embedBufferRaw)
362
461
  ? embedBufferRaw.dtype
363
462
  : getWeightDtype(embedBufferRaw);
463
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
364
464
  const activationDtype = getEffectiveActivationDtype(state, opts);
365
465
 
366
466
  const embedTensor = await embed([tokenId], embedBuffer, {
@@ -372,7 +472,8 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
372
472
  debugProbes: state.runtimeConfig.shared.debug.probes,
373
473
  operatorDiagnostics: state.operatorDiagnostics,
374
474
  activationDtype,
375
- embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
475
+ embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
476
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
376
477
  executionPolicies: state.executionV1State?.policies ?? null,
377
478
  });
378
479
 
@@ -437,8 +538,8 @@ function createDecodeRecorder(state, opts) {
437
538
  let recorder;
438
539
  if (recorderEnabled) {
439
540
  recorder = opts.profile
440
- ? createProfilingRecorder('decode')
441
- : createCommandRecorder('decode');
541
+ ? createProfilingRecorder('decode', device)
542
+ : createCommandRecorder('decode', { recordLabels: opts.debug === true }, device);
442
543
  }
443
544
  if (state.decodeStepCount === 1) {
444
545
  const path = selectRuleValue('inference', 'config', 'tracePath', { useRecorder: Boolean(recorder) });
@@ -491,7 +592,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
491
592
  const recorder = createDecodeRecorder(state, opts);
492
593
 
493
594
  if (state.finitenessBuffer && device) {
494
- device.queue.writeBuffer(state.finitenessBuffer, 0, new Uint32Array([0, 0, 0, 0]));
595
+ device.queue.writeBuffer(state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
495
596
  }
496
597
 
497
598
  const context = helpers.buildLayerContext(recorder, true, opts.debugLayers, executionPlan);
@@ -503,13 +604,14 @@ export async function decodeStep(state, currentIds, opts, helpers) {
503
604
  const decodeAltBuffer = state.decodeBuffers.getOutputHiddenBuffer();
504
605
 
505
606
  const embedBufferRaw = state.weights.get('embed');
506
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
607
+ if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
507
608
  throw new Error('Embed buffer not found or not a supported buffer type');
508
609
  }
509
610
  const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
510
611
  const embedDtype = isCpuWeightBuffer(embedBufferRaw)
511
612
  ? embedBufferRaw.dtype
512
613
  : getWeightDtype(embedBufferRaw);
614
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
513
615
  const activationDtype = getEffectiveActivationDtype(state, opts);
514
616
  const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
515
617
 
@@ -523,7 +625,8 @@ export async function decodeStep(state, currentIds, opts, helpers) {
523
625
  debugProbes: state.runtimeConfig.shared.debug.probes,
524
626
  operatorDiagnostics: state.operatorDiagnostics,
525
627
  activationDtype,
526
- embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
628
+ embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
629
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
527
630
  executionPolicies: state.executionV1State?.policies ?? null,
528
631
  });
529
632
 
@@ -708,6 +811,14 @@ export async function decodeStep(state, currentIds, opts, helpers) {
708
811
 
709
812
  state.stats.singleTokenSubmitWaitMs = (state.stats.singleTokenSubmitWaitMs ?? 0) + submitWaitMs;
710
813
  state.stats.singleTokenReadbackWaitMs = (state.stats.singleTokenReadbackWaitMs ?? 0) + readbackWaitMs;
814
+ if (readbackResult.timing) {
815
+ state.stats.singleTokenReadbackMapWaitMs = (state.stats.singleTokenReadbackMapWaitMs ?? 0)
816
+ + readbackResult.timing.mapWaitMs;
817
+ state.stats.singleTokenReadbackCleanupMs = (state.stats.singleTokenReadbackCleanupMs ?? 0)
818
+ + readbackResult.timing.cleanupMs;
819
+ state.stats.singleTokenReadbackCopyMs = (state.stats.singleTokenReadbackCopyMs ?? 0)
820
+ + readbackResult.timing.copyMs;
821
+ }
711
822
 
712
823
  const { nextToken: fusedNextToken, finitenessStatus } = readbackResult;
713
824
 
@@ -1209,8 +1320,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1209
1320
  state.batchingStats.batchedForwardCalls += 1;
1210
1321
  const tokensPerInterval = batchSize * readbackInterval;
1211
1322
  const recorder = opts.profile
1212
- ? createProfilingRecorder('batch_decode')
1213
- : createCommandRecorder('batch_decode');
1323
+ ? createProfilingRecorder('batch_decode', device)
1324
+ : createCommandRecorder('batch_decode', { recordLabels: opts.debug === true }, device);
1214
1325
  const lmHead = state.weights.get('lm_head');
1215
1326
  if (lmHead && isCpuWeightBuffer(lmHead)) {
1216
1327
  throw new Error('[Pipeline] GPU-only decode not supported with CPU-resident LM head.');
@@ -1227,6 +1338,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1227
1338
  config,
1228
1339
  weights: state.weights,
1229
1340
  });
1341
+ const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
1342
+ config,
1343
+ weights: state.weights,
1344
+ });
1230
1345
  const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: state.weights });
1231
1346
  const hotStartTokenIndex = pleHotVocabularyRuntime?.hotTokenIndexMap?.[startToken] ?? null;
1232
1347
  const canUseHotVocabularyBatchDecode = hasRangeBackedPerLayerInputs
@@ -1260,6 +1375,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1260
1375
  if (eosTokenId == null) {
1261
1376
  throw new Error('[Pipeline] Missing EOS token. Ensure tokenizer or manifest provides stop tokens.');
1262
1377
  }
1378
+ const stopTokenLookup = createStopTokenLookup(stopTokenIds, eosToken);
1263
1379
  const maxTokens = executionPlan?.maxTokens
1264
1380
  ?? opts.maxTokens
1265
1381
  ?? state.runtimeConfig.inference.generation.maxTokens;
@@ -1304,8 +1420,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1304
1420
  })
1305
1421
  : null;
1306
1422
 
1423
+ const tokenReadbackBytes = N * 4;
1424
+ const tokenStagingBytes = tokenReadbackBytes + (state.finitenessBuffer ? FINITENESS_STATUS_BYTES : 0);
1307
1425
  const tokensStagingBuffer = ringSlot?.stagingTokens ?? device.createBuffer({
1308
- size: N * 4,
1426
+ size: tokenStagingBytes,
1309
1427
  usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
1310
1428
  });
1311
1429
  const ownsTokensStaging = !ringSlot?.stagingTokens;
@@ -1317,24 +1435,33 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1317
1435
  })
1318
1436
  : null;
1319
1437
  const ownsStopStaging = useGpuStopFlags && !ringSlot?.stagingStop;
1320
- const finitenessStagingBuffer = state.finitenessBuffer
1438
+ const packedFinitenessOffsetBytes = state.finitenessBuffer
1439
+ && tokensStagingBuffer.size >= tokenReadbackBytes + FINITENESS_STATUS_BYTES
1440
+ ? tokenReadbackBytes
1441
+ : null;
1442
+ const finitenessStagingBuffer = state.finitenessBuffer && packedFinitenessOffsetBytes == null
1321
1443
  ? ringSlot?.stagingFiniteness ?? device.createBuffer({
1322
- size: 16,
1444
+ size: FINITENESS_STATUS_BYTES,
1323
1445
  usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
1324
1446
  })
1325
1447
  : null;
1326
- const ownsFinitenessStaging = Boolean(state.finitenessBuffer) && !ringSlot?.stagingFiniteness;
1448
+ const ownsFinitenessStaging = Boolean(state.finitenessBuffer)
1449
+ && packedFinitenessOffsetBytes == null
1450
+ && !ringSlot?.stagingFiniteness;
1327
1451
  let readbackCleanupDelegated = false;
1328
1452
  let repHistoryBuffer = null;
1329
1453
  let repHistoryCount = 0;
1330
1454
  try {
1331
1455
  if (state.finitenessBuffer) {
1332
- device.queue.writeBuffer(state.finitenessBuffer, 0, new Uint32Array([0, 0, 0, 0]));
1456
+ device.queue.writeBuffer(state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
1333
1457
  }
1334
1458
 
1335
- device.queue.writeBuffer(tokensBuffer, 0, new Uint32Array([startToken]));
1459
+ const singleWordUpload = new Uint32Array(1);
1460
+ singleWordUpload[0] = startToken;
1461
+ device.queue.writeBuffer(tokensBuffer, 0, singleWordUpload);
1336
1462
  if (pleInputTokensBuffer) {
1337
- device.queue.writeBuffer(pleInputTokensBuffer, 0, new Uint32Array([hotStartTokenIndex]));
1463
+ singleWordUpload[0] = hotStartTokenIndex;
1464
+ device.queue.writeBuffer(pleInputTokensBuffer, 0, singleWordUpload);
1338
1465
  }
1339
1466
  if (stopBuffer) {
1340
1467
  const stopElements = stopBuffer.size / 4;
@@ -1350,11 +1477,12 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1350
1477
  if (isCpuWeightBuffer(embedBufferRaw)) {
1351
1478
  throw new Error('[Pipeline] GPU-only decode not supported with CPU-resident embeddings.');
1352
1479
  }
1353
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw)) {
1480
+ if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw)) {
1354
1481
  throw new Error('Embed buffer not found or not a GPUBuffer/WeightBuffer');
1355
1482
  }
1356
1483
  const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
1357
1484
  const embedDtype = getWeightDtype(embedBufferRaw);
1485
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
1358
1486
  const activationDtype = getEffectiveActivationDtype(state, opts);
1359
1487
 
1360
1488
  // GPU-side repetition penalty: upload deduplicated history before batch
@@ -1373,12 +1501,23 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1373
1501
  }
1374
1502
 
1375
1503
  // Hoist loop-invariant values to avoid repeated rule lookups and allocations.
1376
- const embeddingDtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype });
1504
+ const embeddingDtype = selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype });
1377
1505
  const debugProbes = state.runtimeConfig.shared.debug.probes;
1378
1506
  const currentTokenIdsArray = [startToken];
1507
+ const useGreedyLmHeadArgmaxFusion = shouldUseGreedyLmHeadArgmaxFusion(
1508
+ state,
1509
+ opts,
1510
+ samplingDefaults,
1511
+ repetitionPenalty
1512
+ );
1379
1513
 
1380
1514
  for (let i = 0; i < N; i++) {
1381
- currentTokenIdsArray[0] = rollingIds[i];
1515
+ // In the GPU batch path, only the start token (i=0) is known on the CPU.
1516
+ // Subsequent tokens (i>0) are sampled on the GPU and not read back until
1517
+ // after the full batch completes. Set currentTokenIds to null for those
1518
+ // iterations so downstream code (PLE cache, KV-cache update) gracefully
1519
+ // skips CPU-side token-dependent optimizations.
1520
+ currentTokenIdsArray[0] = i === 0 ? startToken : null;
1382
1521
  const currentPos = state.currentSeqLen + i;
1383
1522
  context.currentSeqLen = currentPos;
1384
1523
  context.currentTokenIds = currentTokenIdsArray;
@@ -1389,11 +1528,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1389
1528
  vocabSize: config.vocabSize,
1390
1529
  scaleEmbeddings: config.scaleEmbeddings,
1391
1530
  recorder,
1531
+ outputBuffer: context.decodeBuffers?.getHiddenBuffer() ?? undefined,
1392
1532
  transpose: state.embeddingTranspose,
1393
1533
  debugProbes,
1394
1534
  operatorDiagnostics: state.operatorDiagnostics,
1395
1535
  activationDtype,
1396
1536
  embeddingDtype,
1537
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
1397
1538
  executionPolicies: state.executionV1State?.policies ?? null,
1398
1539
  numTokens: 1,
1399
1540
  indexOffset: i,
@@ -1403,9 +1544,9 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1403
1544
  const perLayerInputs = await preparePerLayerInputs(tokensBuffer, hiddenTensor, context, {
1404
1545
  numTokens: 1,
1405
1546
  indexOffset: i,
1406
- perLayerTokenIds: pleInputTokensBuffer,
1547
+ perLayerTokenIds: pleInputTokensBuffer ?? (hasGpuSplitPerLayerInputs ? tokensBuffer : null),
1407
1548
  perLayerIndexOffset: i,
1408
- tokenIdHint: Number.isInteger(rollingIds[i]) ? rollingIds[i] : null,
1549
+ tokenIdHint: i === 0 ? startToken : null,
1409
1550
  pleCache: state.pleCache ?? null,
1410
1551
  });
1411
1552
  try {
@@ -1446,48 +1587,67 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1446
1587
  helpers.releaseSharedAttentionState?.(context.sharedAttentionState, recorder);
1447
1588
  }
1448
1589
 
1449
- const logits = await recordLogitsGPU(
1450
- recorder,
1451
- hiddenStatesBuffer,
1452
- 1,
1453
- helpers.getLogitsWeights(),
1454
- helpers.getLogitsConfig(),
1455
- state.operatorDiagnostics
1456
- );
1457
- const { logitsBuffer, vocabSize, logitsDtype } = logits;
1458
-
1459
- // Apply GPU-side repetition penalty before sampling
1460
- if (repHistoryBuffer && repetitionPenalty !== 1.0) {
1461
- await recordRepPenalty(recorder, logitsBuffer, repHistoryBuffer, tokensBuffer, {
1462
- vocabSize,
1463
- historyCount: repHistoryCount,
1464
- penalty: repetitionPenalty,
1465
- batchCount: i,
1466
- batchOffset: 1,
1467
- logitsDtype,
1468
- });
1469
- }
1470
-
1471
1590
  const outputIndex = i + 1;
1472
- if (opts.temperature < samplingDefaults.greedyThreshold) {
1473
- await recordArgmax(recorder, logitsBuffer, vocabSize, {
1474
- padTokenId,
1475
- logitSoftcap,
1476
- logitsDtype,
1477
- outputBuffer: tokensBuffer,
1478
- outputIndex,
1479
- });
1591
+ let logitsBuffer = null;
1592
+ if (useGreedyLmHeadArgmaxFusion) {
1593
+ await recordGreedyLmHeadArgmaxGPU(
1594
+ recorder,
1595
+ hiddenStatesBuffer,
1596
+ 1,
1597
+ helpers.getLogitsWeights(),
1598
+ helpers.getLogitsConfig(),
1599
+ {
1600
+ padTokenId,
1601
+ logitSoftcap,
1602
+ outputBuffer: tokensBuffer,
1603
+ outputIndex,
1604
+ },
1605
+ state.operatorDiagnostics
1606
+ );
1480
1607
  } else {
1481
- await recordGPUSample(recorder, logitsBuffer, vocabSize, {
1482
- temperature: opts.temperature,
1483
- topK: opts.topK,
1484
- padTokenId,
1485
- logitSoftcap,
1486
- logitsDtype,
1487
- outputBuffer: tokensBuffer,
1488
- outputIndex,
1489
- greedyThreshold: samplingDefaults.greedyThreshold,
1490
- });
1608
+ const logits = await recordLogitsGPU(
1609
+ recorder,
1610
+ hiddenStatesBuffer,
1611
+ 1,
1612
+ helpers.getLogitsWeights(),
1613
+ helpers.getLogitsConfig(),
1614
+ state.operatorDiagnostics
1615
+ );
1616
+ const { vocabSize, logitsDtype } = logits;
1617
+ logitsBuffer = logits.logitsBuffer;
1618
+
1619
+ // Apply GPU-side repetition penalty before sampling
1620
+ if (repHistoryBuffer && repetitionPenalty !== 1.0) {
1621
+ await recordRepPenalty(recorder, logitsBuffer, repHistoryBuffer, tokensBuffer, {
1622
+ vocabSize,
1623
+ historyCount: repHistoryCount,
1624
+ penalty: repetitionPenalty,
1625
+ batchCount: i,
1626
+ batchOffset: 1,
1627
+ logitsDtype,
1628
+ });
1629
+ }
1630
+
1631
+ if (opts.temperature < samplingDefaults.greedyThreshold) {
1632
+ await recordArgmax(recorder, logitsBuffer, vocabSize, {
1633
+ padTokenId,
1634
+ logitSoftcap,
1635
+ logitsDtype,
1636
+ outputBuffer: tokensBuffer,
1637
+ outputIndex,
1638
+ });
1639
+ } else {
1640
+ await recordGPUSample(recorder, logitsBuffer, vocabSize, {
1641
+ temperature: opts.temperature,
1642
+ topK: opts.topK,
1643
+ padTokenId,
1644
+ logitSoftcap,
1645
+ logitsDtype,
1646
+ outputBuffer: tokensBuffer,
1647
+ outputIndex,
1648
+ greedyThreshold: samplingDefaults.greedyThreshold,
1649
+ });
1650
+ }
1491
1651
  }
1492
1652
 
1493
1653
  const stopCheck = canUseHotVocabularyBatchDecode
@@ -1525,7 +1685,16 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1525
1685
  }
1526
1686
 
1527
1687
  const recordMs = performance.now() - recordStart;
1688
+ const recordStats = recorder.getStats();
1689
+ const recordOps = recordStats.opCount;
1690
+ const recordPasses = recordStats.computePassCount;
1528
1691
  state.stats.decodeRecordMs = (state.stats.decodeRecordMs ?? 0) + recordMs;
1692
+ state.stats.decodeRecordOps = (state.stats.decodeRecordOps ?? 0) + recordOps;
1693
+ state.stats.decodeRecordPasses = (state.stats.decodeRecordPasses ?? 0) + recordPasses;
1694
+ state.stats.decodeRecordOpLabels = mergeRecordOpLabelCounts(
1695
+ state.stats.decodeRecordOpLabels,
1696
+ recordStats.opLabelCounts
1697
+ );
1529
1698
 
1530
1699
  const encoder = recorder.getEncoder();
1531
1700
  encoder.copyBufferToBuffer(tokensBuffer, 4, tokensStagingBuffer, 0, N * 4);
@@ -1533,8 +1702,22 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1533
1702
  encoder.copyBufferToBuffer(stopBuffer, 4, stopStagingBuffer, 0, N * 4);
1534
1703
  }
1535
1704
 
1536
- if (state.finitenessBuffer && finitenessStagingBuffer) {
1537
- encoder.copyBufferToBuffer(state.finitenessBuffer, 0, finitenessStagingBuffer, 0, 16);
1705
+ if (state.finitenessBuffer && packedFinitenessOffsetBytes != null) {
1706
+ encoder.copyBufferToBuffer(
1707
+ state.finitenessBuffer,
1708
+ 0,
1709
+ tokensStagingBuffer,
1710
+ packedFinitenessOffsetBytes,
1711
+ FINITENESS_STATUS_BYTES
1712
+ );
1713
+ } else if (state.finitenessBuffer && finitenessStagingBuffer) {
1714
+ encoder.copyBufferToBuffer(
1715
+ state.finitenessBuffer,
1716
+ 0,
1717
+ finitenessStagingBuffer,
1718
+ 0,
1719
+ FINITENESS_STATUS_BYTES
1720
+ );
1538
1721
  }
1539
1722
 
1540
1723
  if (!allowReadback('pipeline.decode.sample')) {
@@ -1549,6 +1732,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1549
1732
  tokensStagingBuffer,
1550
1733
  stopStagingBuffer,
1551
1734
  finitenessStagingBuffer,
1735
+ finitenessOffsetBytes: packedFinitenessOffsetBytes,
1552
1736
  tokenCount: N,
1553
1737
  ownsTokensStaging,
1554
1738
  ownsStopStaging,
@@ -1558,6 +1742,14 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1558
1742
  });
1559
1743
  const readbackWaitMs = performance.now() - readbackStart;
1560
1744
  state.stats.decodeReadbackWaitMs = (state.stats.decodeReadbackWaitMs ?? 0) + readbackWaitMs;
1745
+ if (readback.timing) {
1746
+ state.stats.decodeReadbackMapWaitMs = (state.stats.decodeReadbackMapWaitMs ?? 0)
1747
+ + readback.timing.mapWaitMs;
1748
+ state.stats.decodeReadbackCleanupMs = (state.stats.decodeReadbackCleanupMs ?? 0)
1749
+ + readback.timing.cleanupMs;
1750
+ state.stats.decodeReadbackCopyMs = (state.stats.decodeReadbackCopyMs ?? 0)
1751
+ + readback.timing.copyMs;
1752
+ }
1561
1753
 
1562
1754
  const isInfinite = readback.finitenessStatus.triggered;
1563
1755
  const metadata = readback.finitenessStatus.metadata;
@@ -1576,8 +1768,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1576
1768
  log.debug('Pipeline', `[STOP] N=${N} flags=[${Array.from(stopFlags).join(',')}] tokens=[${tokens.join(',')}] eos=${eosTokenId}`);
1577
1769
  }
1578
1770
 
1579
- const actualCount = resolveBatchStop(tokens, stopFlags, stopTokenIds, eosToken);
1580
- const generatedTokens = tokens.slice(0, actualCount);
1771
+ const actualCount = resolveBatchStop(tokens, stopFlags, stopTokenLookup);
1772
+ const generatedTokens = actualCount === tokens.length
1773
+ ? tokens
1774
+ : tokens.subarray(0, actualCount);
1581
1775
  const invalidToken = findInvalidGeneratedToken(generatedTokens, config.vocabSize, padTokenId);
1582
1776
 
1583
1777
  if (isInfinite) {
@@ -1590,6 +1784,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1590
1784
  `at batch index ${invalidToken.index} (vocabSize=${config.vocabSize}, padTokenId=${padTokenId ?? 'none'}).`
1591
1785
  );
1592
1786
  }
1787
+ state.batchingStats.executedBatchTokens = (state.batchingStats.executedBatchTokens ?? 0) + N;
1788
+ state.batchingStats.resolvedBatchTokens = (state.batchingStats.resolvedBatchTokens ?? 0) + actualCount;
1593
1789
 
1594
1790
  if (opts.profile && recorder.isProfilingEnabled()) {
1595
1791
  const timings = await recorder.resolveProfileTimings();