doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -0,0 +1,81 @@
1
+ export interface ExecutionPlanSummary {
2
+ id: string;
3
+ kernelPathId: string | null;
4
+ kernelPathSource: string;
5
+ activationDtype: string;
6
+ readbackInterval: number | null;
7
+ readbackMode: string | null;
8
+ batchSize: number | undefined;
9
+ stopCheckMode: string | undefined;
10
+ ringTokens: number | null;
11
+ ringStop: number | null;
12
+ ringStaging: number | null;
13
+ }
14
+
15
+ export declare function shouldDisableBatchDecodeAfterShortBatch(input: {
16
+ hitStop?: boolean;
17
+ actualCount?: number;
18
+ requestedCount?: number;
19
+ }): boolean;
20
+
21
+ export declare function resolveHotVocabularyBatchDecodeAvailability(input: {
22
+ hasRangeBackedPerLayerInputs?: boolean;
23
+ pleHotVocabularyRuntime?: Record<string, unknown> | null;
24
+ tokenId?: number;
25
+ }): boolean;
26
+
27
+ export declare function primePleDecodeRuntimeCache(
28
+ state: Record<string, unknown>,
29
+ seedTokenIds?: number[] | null
30
+ ): Promise<void>;
31
+
32
+ export declare function recordPrefillProfileStep(
33
+ state: Record<string, unknown>,
34
+ entry: Record<string, unknown>
35
+ ): void;
36
+
37
+ /**
38
+ * Resolve display text for token IDs. Custom primary and fallback renderers
39
+ * are tried around tokenizer.decode, preserving empty-string special-token
40
+ * filtering when fallback text looks like a special token.
41
+ */
42
+ export declare function resolveTokenText(
43
+ tokenizer: { decode?(ids: readonly number[], skipSpecialTokens?: boolean): string },
44
+ tokenIds: readonly number[],
45
+ fallbackText?: string,
46
+ renderTokenText?: ((ids: readonly number[]) => string) | null,
47
+ renderFallbackTokenText?: ((ids: readonly number[]) => string) | null
48
+ ): string;
49
+
50
+ export declare function usesReplayPrefillDecode(state: Record<string, unknown> | null | undefined): boolean;
51
+
52
+ export declare function assertIncrementalDecodeSupport(
53
+ state: Record<string, unknown>,
54
+ operation: string
55
+ ): void;
56
+
57
+ export declare function summarizeExecutionPlan(plan: Record<string, unknown> | null | undefined): ExecutionPlanSummary | null;
58
+
59
+ export declare function shouldRetryWithFinitenessFallback(error: unknown): boolean;
60
+
61
+ export declare function createUnhandledFinitenessPolicyError(
62
+ state: Record<string, unknown>,
63
+ contextLabel: string,
64
+ error: unknown
65
+ ): Error;
66
+
67
+ export declare function resolveTargetPlanKVDtype(
68
+ plan: Record<string, unknown> | null | undefined,
69
+ contextLabel: string
70
+ ): 'f16' | 'f32';
71
+
72
+ export declare function resolveCurrentKVCacheDtype(
73
+ state: Record<string, unknown>,
74
+ plan: Record<string, unknown> | null | undefined,
75
+ contextLabel: string
76
+ ): 'f16' | 'f32';
77
+
78
+ export declare function cloneRuntimeInferenceWithKVDtype(
79
+ state: Record<string, unknown>,
80
+ kvDtype: 'f16' | 'f32'
81
+ ): Record<string, unknown>;
@@ -86,18 +86,6 @@ export function recordPrefillProfileStep(state, entry) {
86
86
  state.stats.prefillProfileSteps.push(entry);
87
87
  }
88
88
 
89
- /**
90
- * Resolve display text for a token ID sequence.
91
- *
92
- * Preference order:
93
- * 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
94
- * 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
95
- * 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
96
- * unless primary returned empty string and fallback looks like a special token (<...>),
97
- * in which case empty string is preserved to keep skip-special behavior deterministic.
98
- * 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
99
- * 5. fallbackText — static fallback string (default '?').
100
- */
101
89
  export function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
102
90
  const renderPrimary = typeof renderTokenText === 'function'
103
91
  ? renderTokenText
@@ -162,8 +150,10 @@ export function summarizeExecutionPlan(plan) {
162
150
  kernelPathSource: plan.kernelPathSource ?? 'none',
163
151
  activationDtype: plan.activationDtype,
164
152
  readbackInterval: plan.readbackInterval ?? null,
153
+ readbackMode: plan.readbackMode ?? null,
165
154
  batchSize: plan.defaultBatchSize,
166
155
  stopCheckMode: plan.defaultStopCheckMode,
156
+ disableCommandBatching: plan.defaultDisableCommandBatching === true,
167
157
  ringTokens: plan.ringTokens ?? null,
168
158
  ringStop: plan.ringStop ?? null,
169
159
  ringStaging: plan.ringStaging ?? null,
@@ -5,7 +5,7 @@ import { log } from '../../../debug/index.js';
5
5
  import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
6
6
  import { selectRuleValue } from '../../../rules/rule-registry.js';
7
7
  import { decodeReadback } from './debug-utils/index.js';
8
- import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
8
+ import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../gpu/weight-buffer.js';
9
9
  import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
10
10
  import { resolveActiveExecutionPlan } from './execution-plan.js';
11
11
  import { isPlainObject } from '../../../utils/plain-object.js';
@@ -123,6 +123,9 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
123
123
  kvCache: state.kvCache,
124
124
  currentSeqLen: state.currentSeqLen,
125
125
  currentTokenIds: state.currentTokenIds ?? null,
126
+ diffusionGemmaDecoder: false,
127
+ __precomputedInputNorm: null,
128
+ __postFfnNextInputNorm: null,
126
129
  useGPU: state.useGPU,
127
130
  debug: state.debug,
128
131
  stats: state.stats,
@@ -202,7 +205,7 @@ export function getLogitsWeights(state) {
202
205
  if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array)) {
203
206
  throw new Error('Final norm not found or invalid type');
204
207
  }
205
- if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead))) {
208
+ if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
206
209
  throw new Error('LM head not found or invalid type');
207
210
  }
208
211
  return { finalNorm, lmHead };
@@ -0,0 +1,84 @@
1
+ import type { Tensor } from '../../../gpu/tensor.js';
2
+
3
+ export interface StructuredPromptRequest {
4
+ messages: unknown[];
5
+ }
6
+
7
+ export interface PrefixEmbeddingOverride {
8
+ prefixLength: number;
9
+ offset: number;
10
+ embeddings: Float32Array | GPUBuffer;
11
+ expectedLength: number;
12
+ byteLength: number;
13
+ byteOffset: number;
14
+ }
15
+
16
+ export interface InputSpan {
17
+ offset: number;
18
+ length: number;
19
+ }
20
+
21
+ export type PrefixEmbeddingOverrideTransitionDeclaration =
22
+ | 'step_precision'
23
+ | 'explicit_cast_step'
24
+ | null;
25
+
26
+ export interface PrefixEmbeddingOverrideExecutionOptions {
27
+ executionPolicies?: Record<string, unknown> | null;
28
+ transitionDeclaredBy?: PrefixEmbeddingOverrideTransitionDeclaration;
29
+ }
30
+
31
+ export declare function resolvePromptInput(
32
+ state: Record<string, unknown>,
33
+ prompt: string | unknown[] | StructuredPromptRequest,
34
+ useChatTemplate: boolean,
35
+ contextLabel: string
36
+ ): string;
37
+
38
+ export declare function resolveEffectivePrefillTokenChunkSize(
39
+ state: Record<string, unknown>
40
+ ): number | null | undefined;
41
+
42
+ export declare function releasePerLayerInputBuffer(
43
+ buffer: GPUBuffer | null | undefined,
44
+ recorder: { trackTemporaryBuffer(buffer: GPUBuffer): void } | null | undefined,
45
+ decodeBuffers: { ownsBuffer(buffer: GPUBuffer): boolean } | null | undefined,
46
+ pleCache?: { ownedBuffers?: Set<GPUBuffer> } | null
47
+ ): void;
48
+
49
+ export declare function normalizePrefixEmbeddingOverride(
50
+ override: Record<string, unknown> | null | undefined,
51
+ hiddenSize: number,
52
+ numTokens: number,
53
+ contextLabel: string
54
+ ): PrefixEmbeddingOverride | null;
55
+
56
+ export declare function resolvePrefillEmbeddingInputIds(
57
+ inputIds: readonly number[],
58
+ embeddingInputSpan: InputSpan & { tokenId: number } | null | undefined,
59
+ contextLabel: string
60
+ ): number[] | readonly number[];
61
+
62
+ export declare function resolvePrefillMultimodalBidirectionalSpan(
63
+ inputIds: readonly number[],
64
+ bidirectionalSpan: InputSpan | null | undefined,
65
+ contextLabel: string
66
+ ): InputSpan | null;
67
+
68
+ export declare function resolvePrefixEmbeddingOverrideTransitionDeclaredBy(
69
+ executionV1State: Record<string, unknown> | null | undefined
70
+ ): PrefixEmbeddingOverrideTransitionDeclaration;
71
+
72
+ export declare function applyPrefixEmbeddingOverride(
73
+ baseTensor: Tensor,
74
+ override: PrefixEmbeddingOverride | null,
75
+ hiddenSize: number,
76
+ contextLabel: string,
77
+ executionOptions?: Record<string, unknown> | PrefixEmbeddingOverrideExecutionOptions | null
78
+ ): Promise<Tensor>;
79
+
80
+ export declare function shouldDisablePrefillCommandBatching(
81
+ state: Record<string, unknown>,
82
+ opts: Record<string, unknown>,
83
+ multimodalBidirectionalSpan: InputSpan | null
84
+ ): boolean;
@@ -16,6 +16,50 @@ function isStructuredChatRequest(prompt) {
16
16
  && Array.isArray(prompt.messages);
17
17
  }
18
18
 
19
+ function normalizePrefixEmbeddingOverrideExecutionOptions(options) {
20
+ if (
21
+ options != null
22
+ && typeof options === 'object'
23
+ && !Array.isArray(options)
24
+ && (
25
+ Object.prototype.hasOwnProperty.call(options, 'executionPolicies')
26
+ || Object.prototype.hasOwnProperty.call(options, 'transitionDeclaredBy')
27
+ )
28
+ ) {
29
+ return {
30
+ executionPolicies: options.executionPolicies ?? null,
31
+ transitionDeclaredBy: options.transitionDeclaredBy ?? null,
32
+ };
33
+ }
34
+ return {
35
+ executionPolicies: options ?? null,
36
+ transitionDeclaredBy: null,
37
+ };
38
+ }
39
+
40
+ function normalizeExecutionDtype(value) {
41
+ const normalized = typeof value === 'string' ? value.trim().toLowerCase() : '';
42
+ return normalized === 'f16' || normalized === 'f32' ? normalized : null;
43
+ }
44
+
45
+ export function resolvePrefixEmbeddingOverrideTransitionDeclaredBy(executionV1State) {
46
+ const steps = executionV1State?.resolvedSteps?.all;
47
+ if (!Array.isArray(steps)) {
48
+ return null;
49
+ }
50
+ for (const step of steps) {
51
+ if (step?.op !== 'cast' || step?.section !== 'preLayer') {
52
+ continue;
53
+ }
54
+ const fromDtype = normalizeExecutionDtype(step.fromDtype ?? step.precision?.inputDtype);
55
+ const toDtype = normalizeExecutionDtype(step.toDtype ?? step.precision?.outputDtype);
56
+ if (fromDtype === 'f32' && toDtype === 'f16') {
57
+ return 'explicit_cast_step';
58
+ }
59
+ }
60
+ return null;
61
+ }
62
+
19
63
  export function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
20
64
  const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
21
65
  if (typeof prompt === 'string') {
@@ -49,6 +93,19 @@ export function resolvePromptInput(state, prompt, useChatTemplate, contextLabel)
49
93
  return formatChatMessages(messages, templateType, chatOptions);
50
94
  }
51
95
 
96
+ export function resolveEffectivePrefillTokenChunkSize(state) {
97
+ const runtimeSession = state?.runtimeConfig?.inference?.session;
98
+ const runtimeChunkSize = runtimeSession?.prefillTokenChunkSize;
99
+ if (runtimeChunkSize !== undefined && runtimeChunkSize !== null) {
100
+ return runtimeChunkSize;
101
+ }
102
+ const modelSession = state?.modelConfig?.sessionSettings;
103
+ if (modelSession?.prefillTokenChunkSize !== undefined) {
104
+ return modelSession.prefillTokenChunkSize;
105
+ }
106
+ return runtimeChunkSize;
107
+ }
108
+
52
109
  export function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
53
110
  if (!buffer) {
54
111
  return;
@@ -191,10 +248,14 @@ export function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectiona
191
248
  return { offset, length };
192
249
  }
193
250
 
194
- export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
251
+ export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionOptions = null) {
195
252
  if (!override) {
196
253
  return baseTensor;
197
254
  }
255
+ const {
256
+ executionPolicies,
257
+ transitionDeclaredBy,
258
+ } = normalizePrefixEmbeddingOverrideExecutionOptions(executionOptions);
198
259
  if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
199
260
  throw new Error(
200
261
  `[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
@@ -236,6 +297,7 @@ export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenS
236
297
  toDtype: 'f16',
237
298
  op: 'prefill_embedding_override',
238
299
  detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
300
+ transitionDeclaredBy,
239
301
  });
240
302
  const castedOverride = await castF32ToF16(overrideTensor);
241
303
  try {
@@ -257,6 +319,7 @@ export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenS
257
319
  toDtype: 'f16',
258
320
  op: 'prefill_embedding_override',
259
321
  detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
322
+ transitionDeclaredBy,
260
323
  });
261
324
  const packedOverride = f32ToF16Array(override.embeddings);
262
325
  device.queue.submit([encoder.finish()]);
@@ -280,6 +343,9 @@ export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidir
280
343
  if (state?.kvCache?.layout === 'bdpa_paged') {
281
344
  return true;
282
345
  }
346
+ if (resolveEffectivePrefillTokenChunkSize(state) != null) {
347
+ return true;
348
+ }
283
349
  if (multimodalBidirectionalSpan == null) {
284
350
  return false;
285
351
  }
@@ -9,6 +9,7 @@ export interface StepOptionsResolved {
9
9
  topP: number;
10
10
  topK: number;
11
11
  repetitionPenalty: number;
12
+ suppressTokenIds?: number[];
12
13
  debug: boolean;
13
14
  debugLayers?: number[];
14
15
  profile: boolean;
@@ -43,8 +43,25 @@ export interface FusedDecodeSamplingConfig {
43
43
 
44
44
  export declare function shouldUseFusedDecodeSampling(config: FusedDecodeSamplingConfig): boolean;
45
45
 
46
+ export interface StopTokenLookup {
47
+ firstTokenId: number | null;
48
+ secondTokenId: number | null;
49
+ tokenSet: Set<number> | null;
50
+ }
51
+
52
+ export declare function createStopTokenLookup(
53
+ stopTokenIds: number[],
54
+ eosTokenId?: number | null
55
+ ): StopTokenLookup;
56
+
57
+ export declare function resolveBatchStop(
58
+ tokens: ArrayLike<number>,
59
+ stopFlags: ArrayLike<number> | null,
60
+ stopTokenLookup: StopTokenLookup
61
+ ): number;
62
+
46
63
  export declare function findInvalidGeneratedToken(
47
- tokens: number[],
64
+ tokens: ArrayLike<number>,
48
65
  vocabSize: number,
49
66
  padTokenId?: number | null
50
67
  ): { index: number; tokenId: number } | null;
@@ -56,12 +73,17 @@ export interface SampledTokenStagingBuffer {
56
73
  destroy(): void;
57
74
  }
58
75
 
76
+ export interface DeferredCleanupRecorder {
77
+ completeDeferredCleanup(options?: { discardPooled?: boolean }): Promise<void> | void;
78
+ }
79
+
59
80
  export declare function readSampledTokenFromStagingBuffer(
60
81
  stagingBuffer: SampledTokenStagingBuffer,
61
82
  options?: {
62
83
  ownsStagingBuffer?: boolean;
63
84
  hasFinitenessBuffer?: boolean;
64
85
  ring?: { advance(): void } | null;
86
+ cleanupRecorder?: DeferredCleanupRecorder | null;
65
87
  }
66
88
  ): Promise<{
67
89
  nextToken: number;
@@ -69,6 +91,11 @@ export declare function readSampledTokenFromStagingBuffer(
69
91
  triggered: boolean;
70
92
  metadata: string;
71
93
  };
94
+ timing: {
95
+ mapWaitMs: number;
96
+ cleanupMs: number;
97
+ copyMs: number;
98
+ };
72
99
  }>;
73
100
 
74
101
  export declare function readMappedBufferCopy(
@@ -82,17 +109,25 @@ export declare function readBatchTokensFromStagingBuffers(options: {
82
109
  tokensStagingBuffer: SampledTokenStagingBuffer;
83
110
  stopStagingBuffer?: SampledTokenStagingBuffer | null;
84
111
  finitenessStagingBuffer?: SampledTokenStagingBuffer | null;
112
+ finitenessOffsetBytes?: number | null;
85
113
  tokenCount: number;
86
114
  ownsTokensStaging?: boolean;
87
115
  ownsStopStaging?: boolean;
116
+ ownsFinitenessStaging?: boolean;
88
117
  ring?: { advance(): void } | null;
118
+ cleanupRecorder?: DeferredCleanupRecorder | null;
89
119
  }): Promise<{
90
- tokens: number[];
120
+ tokens: Uint32Array;
91
121
  stopFlags: Uint32Array | null;
92
122
  finitenessStatus: {
93
123
  triggered: boolean;
94
124
  metadata: string;
95
125
  };
126
+ timing: {
127
+ mapWaitMs: number;
128
+ cleanupMs: number;
129
+ copyMs: number;
130
+ };
96
131
  }>;
97
132
 
98
133
  export declare function decodeStep(
@@ -148,4 +183,4 @@ export declare function generateNTokensGPU(
148
183
  getLogitsWeights: () => unknown;
149
184
  getLogitsConfig: () => unknown;
150
185
  }
151
- ): Promise<{ tokens: number[]; actualCount: number }>;
186
+ ): Promise<{ tokens: number[] | Uint32Array; actualCount: number }>;