doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -12,6 +12,7 @@ import {
12
12
  resolveWeightBufferMaterialization,
13
13
  } from '../../../../gpu/weight-buffer.js';
14
14
  import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
15
+ import { getRuntimeConfig } from '../../../../config/runtime.js';
15
16
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
16
17
  import {
17
18
  runFusedFFN,
@@ -27,6 +28,7 @@ import { isKernelDebugEnabled, dumpTokenVector } from '../debug-utils.js';
27
28
  import { applyLoRA } from '../lora-apply.js';
28
29
  import { getLoRAModule } from '../lora.js';
29
30
  import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
31
+ import { runProbes } from '../probes.js';
30
32
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
31
33
  import {
32
34
  getKernelPathMatmulPrecision,
@@ -145,8 +147,20 @@ export function resolveDenseFFNFusedPathDtypes(options = {}) {
145
147
  }
146
148
 
147
149
  function hasExplicitMatmulPrecision(role, phase, layerIdx, kernelPath) {
150
+ // Only treat precision as "explicit split" when declared on the role's OWN
151
+ // step. The FUSED_FFN_PRECISION_FALLBACK_ROLES fallback that resolves via
152
+ // the aggregate `ffn` step's precision applies identically to gate/up/down
153
+ // and must NOT force the split path — otherwise a manifest-declared `ffn`
154
+ // precision permanently blocks the fused gate_up_activation kernel.
148
155
  const precision = getKernelPathMatmulPrecision(role, phase, layerIdx, kernelPath);
149
- return precision?.inputDtype != null || precision?.outputDtype != null;
156
+ if (!precision) return false;
157
+ const fallbackPrecision = getKernelPathMatmulPrecision('ffn', phase, layerIdx, kernelPath);
158
+ const inheritedFromFfn = fallbackPrecision
159
+ && fallbackPrecision.inputDtype === precision.inputDtype
160
+ && fallbackPrecision.outputDtype === precision.outputDtype
161
+ && fallbackPrecision.activationDtype === precision.activationDtype;
162
+ if (inheritedFromFfn) return false;
163
+ return precision.inputDtype != null || precision.outputDtype != null;
150
164
  }
151
165
 
152
166
  export function canUseNativeF16FusedGateUp(options = {}) {
@@ -325,7 +339,7 @@ export async function runDenseFFNGPU(
325
339
  if (!device) throw new Error('No GPU device');
326
340
 
327
341
  const { config, recorder } = context;
328
- const { hiddenSize, hiddenActivation, swigluLimit } = config;
342
+ const { hiddenSize, hiddenActivation, swigluLimit, useDoubleWideMlp } = config;
329
343
  const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
330
344
  const lastTokenIdx = Math.max(0, numTokens - 1);
331
345
  const lora = context.lora || null;
@@ -434,6 +448,22 @@ export async function runDenseFFNGPU(
434
448
  releaseBuffer(gateUpOutput.buffer);
435
449
  }
436
450
 
451
+ // Opt-in WideTile+residual fusion: if the caller (processFFNStandard)
452
+ // staged a residual tensor on the context AND no LoRA is present on
453
+ // down_proj (LoRA requires the pre-residual output for its add), route
454
+ // this matmul to the q4_fused_widetile_residual variant which produces
455
+ // (ffn_down_result + residual) in one dispatch. Tell the caller via a
456
+ // context flag so processFFNStandard skips its downstream doResidualAdd.
457
+ const pendingResidual = context.__pendingFfnResidualTensor;
458
+ const downLoraProbe = getLoRAModule(lora, layerIdx, 'down_proj');
459
+ const mergedSession = getRuntimeConfig().inference?.session;
460
+ const tryFuseDownResidual = pendingResidual != null
461
+ && !downLoraProbe
462
+ && numTokens > 1
463
+ && activatedOutput.dtype === 'f32'
464
+ && downOutputDtype === 'f32'
465
+ && pendingResidual.dtype === 'f32';
466
+ let residualFusedHere = false;
437
467
  let output = await doMatmul(
438
468
  activatedOutput, downWeight,
439
469
  numTokens, hiddenSize, intermediateSize,
@@ -445,9 +475,26 @@ export async function runDenseFFNGPU(
445
475
  outputDtype: downOutputDtype,
446
476
  role: 'ffn_down',
447
477
  executionPolicies: context.executionPolicies ?? null,
478
+ residualTensor: tryFuseDownResidual ? pendingResidual : null,
448
479
  },
449
480
  recorder
450
481
  );
482
+ // Detect whether the fusion fired: if the selected variant for ffn_down
483
+ // matmul was q4_fused_widetile_residual, then output IS post-residual.
484
+ // We infer this cheaply by re-checking the conditions the selector uses.
485
+ // (A cleaner signal would require a return-shape change across all
486
+ // dense.js paths; this local signal is enough for correctness.)
487
+ {
488
+ if (tryFuseDownResidual
489
+ && getKernelCapabilities().hasF16 === true
490
+ && mergedSession?.useWideTileResidualFusion === true
491
+ && mergedSession?.useWideTileQ4KPrefill === true
492
+ && mergedSession?.retainQ4KMaterialization === true
493
+ ) {
494
+ residualFusedHere = true;
495
+ context.__ffnResidualFusedFired = true;
496
+ }
497
+ }
451
498
 
452
499
  const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
453
500
  if (loraDown) {
@@ -537,6 +584,7 @@ export async function runDenseFFNGPU(
537
584
  f16BatchSupported,
538
585
  batchSize: numTokens,
539
586
  hiddenSizeAligned32,
587
+ useDoubleWideMlp: Boolean(useDoubleWideMlp),
540
588
  });
541
589
  const useFusedGateUp = gateUpPathMode === 'split'
542
590
  ? false
@@ -608,6 +656,16 @@ export async function runDenseFFNGPU(
608
656
  releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight);
609
657
  }
610
658
 
659
+ // Opt-in WideTile+residual fusion (fused-gate-up path).
660
+ const pendingResidualFused = context.__pendingFfnResidualTensor;
661
+ const downLoraProbeFused = getLoRAModule(lora, layerIdx, 'down_proj');
662
+ const mergedSessionFused = getRuntimeConfig().inference?.session;
663
+ const tryFuseDownResidualFused = pendingResidualFused != null
664
+ && !downLoraProbeFused
665
+ && numTokens > 1
666
+ && downInput.dtype === 'f32'
667
+ && fusedDownOutputDtype === 'f32'
668
+ && pendingResidualFused.dtype === 'f32';
611
669
  let output = await doMatmul(
612
670
  downInput,
613
671
  downWeight,
@@ -622,9 +680,20 @@ export async function runDenseFFNGPU(
622
680
  outputDtype: fusedDownOutputDtype,
623
681
  role: 'ffn_down',
624
682
  executionPolicies: context.executionPolicies ?? null,
683
+ residualTensor: tryFuseDownResidualFused ? pendingResidualFused : null,
625
684
  },
626
685
  recorder
627
686
  );
687
+ {
688
+ if (tryFuseDownResidualFused
689
+ && getKernelCapabilities().hasF16 === true
690
+ && mergedSessionFused?.useWideTileResidualFusion === true
691
+ && mergedSessionFused?.useWideTileQ4KPrefill === true
692
+ && mergedSessionFused?.retainQ4KMaterialization === true
693
+ ) {
694
+ context.__ffnResidualFusedFired = true;
695
+ }
696
+ }
628
697
 
629
698
  const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
630
699
  if (loraDown) {
@@ -748,6 +817,84 @@ export async function runDenseFFNGPU(
748
817
  });
749
818
  sharedInputOwned = sharedInputTensor !== inputTensor;
750
819
  }
820
+ // Opt-in fused gate + up + GeGLU path. Replaces 3 separate dispatches
821
+ // (gate_proj + up_proj + gelu activation) with a single fused kernel when
822
+ // preconditions match: prefill (M>1), f16-materialisable weights + f16
823
+ // activations, gelu activation, no LoRA on gate/up. Gated by
824
+ // runtime.inference.session.useFusedGateUpGelu (default false).
825
+ const gateF16 = resolveWeightBufferMaterialization(layerWeights.gate, 'f16');
826
+ const upF16 = resolveWeightBufferMaterialization(layerWeights.up, 'f16');
827
+ const gateF16Dtype = getWeightDtype(gateF16);
828
+ const upF16Dtype = getWeightDtype(upF16);
829
+ const earlyLoraGate = getLoRAModule(lora, layerIdx, 'gate_proj');
830
+ const earlyLoraUp = getLoRAModule(lora, layerIdx, 'up_proj');
831
+ const fusedGateUpGeluCandidate = context.useFusedGateUpGelu === true
832
+ && numTokens > 1
833
+ && hiddenActivation === 'gelu'
834
+ && !earlyLoraGate && !earlyLoraUp
835
+ && sharedInputTensor.dtype === 'f16'
836
+ && gateF16Dtype === 'f16'
837
+ && upF16Dtype === 'f16';
838
+ if (fusedGateUpGeluCandidate) {
839
+ const { runFusedGateUpGelu, recordFusedGateUpGelu } =
840
+ await import('../../../../gpu/kernels/fused-gate-up-gelu.js');
841
+ const fused = recorder
842
+ ? await recordFusedGateUpGelu(recorder, sharedInputTensor, gateF16, upF16, {
843
+ M: numTokens,
844
+ hiddenSize,
845
+ intermediateSize,
846
+ transposeB: true,
847
+ })
848
+ : await runFusedGateUpGelu(sharedInputTensor, gateF16, upF16, {
849
+ M: numTokens,
850
+ hiddenSize,
851
+ intermediateSize,
852
+ transposeB: true,
853
+ });
854
+ if (sharedInputOwned) {
855
+ if (recorder) { recorder.trackTemporaryBuffer(sharedInputTensor.buffer); }
856
+ else { releaseBuffer(sharedInputTensor.buffer); }
857
+ }
858
+ // Proceed directly to down_proj with the fused activation output.
859
+ const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
860
+ let downInputTensor = fused;
861
+ let downInputOwned = false;
862
+ if (downInputDtype && fused.dtype !== downInputDtype) {
863
+ downInputTensor = await coerceTensorDtype(fused, downInputDtype, recorder, {
864
+ executionPolicies: context.executionPolicies ?? null,
865
+ op: 'ffn_down_input',
866
+ transitionDeclaredBy: 'step_precision',
867
+ });
868
+ downInputOwned = downInputTensor !== fused;
869
+ }
870
+ let outFused = await doMatmul(
871
+ downInputTensor,
872
+ downWeight,
873
+ numTokens,
874
+ hiddenSize,
875
+ intermediateSize,
876
+ {
877
+ transposeB: 'auto',
878
+ label: `L${layerIdx}.ffn_down`,
879
+ layerIdx,
880
+ kernelPath,
881
+ outputDtype: downOutputDtype,
882
+ role: 'ffn_down',
883
+ executionPolicies: context.executionPolicies ?? null,
884
+ },
885
+ recorder
886
+ );
887
+ if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
888
+ releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
889
+ }
890
+ if (downInputOwned) {
891
+ if (recorder) { recorder.trackTemporaryBuffer(downInputTensor.buffer); }
892
+ else { releaseBuffer(downInputTensor.buffer); }
893
+ }
894
+ if (recorder) { recorder.trackTemporaryBuffer(fused.buffer); }
895
+ else { releaseBuffer(fused.buffer); }
896
+ return outFused;
897
+ }
751
898
  const gateWeight = getWeightBuffer(layerWeights.gate, 'ffn_gate');
752
899
  let gateOutput = await doMatmul(
753
900
  gateInputDtype === sharedInputTensor.dtype ? sharedInputTensor : inputTensor,
@@ -849,6 +996,25 @@ export async function runDenseFFNGPU(
849
996
  });
850
997
  }
851
998
 
999
+ await runProbes('ffn_gate', gateOutput.buffer, {
1000
+ layerIdx,
1001
+ numTokens,
1002
+ hiddenSize: intermediateSize,
1003
+ probes: context.debugProbes,
1004
+ recorder,
1005
+ operatorDiagnostics: context.operatorDiagnostics,
1006
+ dtype: gateOutput.dtype,
1007
+ });
1008
+ await runProbes('ffn_up', upOutput.buffer, {
1009
+ layerIdx,
1010
+ numTokens,
1011
+ hiddenSize: intermediateSize,
1012
+ probes: context.debugProbes,
1013
+ recorder,
1014
+ operatorDiagnostics: context.operatorDiagnostics,
1015
+ dtype: upOutput.dtype,
1016
+ });
1017
+
852
1018
  const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
853
1019
  size: numTokens * intermediateSize,
854
1020
  gate: gateOutput,
@@ -866,6 +1032,16 @@ export async function runDenseFFNGPU(
866
1032
  });
867
1033
  }
868
1034
 
1035
+ await runProbes('ffn_act', activatedOutput.buffer, {
1036
+ layerIdx,
1037
+ numTokens,
1038
+ hiddenSize: intermediateSize,
1039
+ probes: context.debugProbes,
1040
+ recorder,
1041
+ operatorDiagnostics: context.operatorDiagnostics,
1042
+ dtype: activatedOutput.dtype,
1043
+ });
1044
+
869
1045
  if (recorder) {
870
1046
  recorder.trackTemporaryBuffer(gateOutput.buffer);
871
1047
  recorder.trackTemporaryBuffer(upOutput.buffer);
@@ -975,7 +1151,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
975
1151
  if (!device) throw new Error('No GPU device');
976
1152
 
977
1153
  const { config, weightConfig, debugFlags, recorder } = context;
978
- const { hiddenSize, hiddenActivation, swigluLimit } = config;
1154
+ const { hiddenSize, hiddenActivation, swigluLimit, useDoubleWideMlp } = config;
979
1155
  const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
980
1156
  const lora = context.lora || null;
981
1157
  const ffnStepPrecision = context.ffnStepPrecision ?? null;
@@ -1124,6 +1300,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
1124
1300
  f16BatchSupported: getKernelCapabilities().hasF16,
1125
1301
  batchSize: numTokens,
1126
1302
  hiddenSizeAligned32,
1303
+ useDoubleWideMlp: Boolean(useDoubleWideMlp),
1127
1304
  });
1128
1305
  const canUseFusedGateUp = gateUpPathMode === 'split'
1129
1306
  ? false
@@ -77,6 +77,18 @@ export async function processFFNStandard(
77
77
  await debugFFNBuffer(context, layerIdx, 'FFN input', normedTensor, numTokens, hiddenSize);
78
78
 
79
79
  // 2. FFN
80
+ // Stage the residual tensor so that runDenseFFNGPU's ffn_down matmul can
81
+ // opportunistically fuse with it (WideTile+residual kernel). The flag
82
+ // `context.__ffnResidualFusedFired` will be set to true by dense.js if
83
+ // the fusion actually fired; we read it below to skip the separate add.
84
+ // If the pre-norm residual sum was captured (fused RMSNorm path), use that
85
+ // buffer directly — it IS the correct residual for ffn_residual because
86
+ // ffn_residual adds to the pre-rmsnorm tensor.
87
+ const pendingResidualForFfn = prenormSumBuffer
88
+ ? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
89
+ : postAttn;
90
+ context.__pendingFfnResidualTensor = pendingResidualForFfn;
91
+ context.__ffnResidualFusedFired = false;
80
92
 
81
93
  let ffnOutput;
82
94
  if (config.useMoE && isMoELayerLocal(layerIdx, config, layerWeights)) {
@@ -84,6 +96,9 @@ export async function processFFNStandard(
84
96
  } else {
85
97
  ffnOutput = await runDenseFFNGPU(layerIdx, normedTensor, numTokens, context, layerWeights);
86
98
  }
99
+ const ffnResidualFused = context.__ffnResidualFusedFired === true;
100
+ context.__pendingFfnResidualTensor = null;
101
+ context.__ffnResidualFusedFired = false;
87
102
  await runProbes('ffn_out', ffnOutput.buffer, {
88
103
  layerIdx,
89
104
  numTokens,
@@ -96,20 +111,28 @@ export async function processFFNStandard(
96
111
  await debugFFNBuffer(context, layerIdx, 'FFN output', ffnOutput, numTokens, hiddenSize);
97
112
 
98
113
  // 3. Residual add (uses prenorm sum when fused, otherwise postAttn)
99
- const residualTensor = prenormSumBuffer
100
- ? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
101
- : postAttn;
114
+ // When WideTile+residual fusion fired inside runDenseFFNGPU, ffnOutput
115
+ // IS already post-residual — skip the separate add entirely.
116
+ let output;
102
117
  let residualInput = ffnOutput;
103
- const residualInputOwned = ffnOutput.dtype !== residualTensor.dtype;
104
- if (residualInputOwned) {
105
- residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
118
+ let residualInputOwned = false;
119
+ if (ffnResidualFused) {
120
+ output = ffnOutput;
121
+ } else {
122
+ const residualTensor = prenormSumBuffer
123
+ ? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
124
+ : postAttn;
125
+ residualInputOwned = ffnOutput.dtype !== residualTensor.dtype;
126
+ if (residualInputOwned) {
127
+ residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
128
+ }
129
+ output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
130
+ label: `L${layerIdx}.ffn_residual`,
131
+ layerIdx,
132
+ outputBuffer: decodeOutputBuffer,
133
+ executionPolicies: context.executionPolicies ?? null,
134
+ });
106
135
  }
107
- const output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
108
- label: `L${layerIdx}.ffn_residual`,
109
- layerIdx,
110
- outputBuffer: decodeOutputBuffer,
111
- executionPolicies: context.executionPolicies ?? null,
112
- });
113
136
  await runProbes('layer_out', output.buffer, {
114
137
  layerIdx,
115
138
  numTokens,
@@ -131,7 +154,11 @@ export async function processFFNStandard(
131
154
  if (residualInputOwned) {
132
155
  releaseOrTrack(recorder, residualInput.buffer, decodeBuffers);
133
156
  }
134
- releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
157
+ // When fusion fired, `output === ffnOutput` and we must NOT release it
158
+ // here — it's our return value.
159
+ if (!ffnResidualFused) {
160
+ releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
161
+ }
135
162
 
136
163
  return output;
137
164
  }
@@ -0,0 +1,243 @@
1
+
2
+ import { log } from '../../../debug/index.js';
3
+ import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
4
+ import { resolveActiveExecutionPlan } from './execution-plan.js';
5
+ import { resolvePerLayerInputsSession } from './generator-helpers.js';
6
+ import {
7
+ ensurePleGpuHotVocabularyRuntime,
8
+ ensurePleGpuSplitTablesRuntime,
9
+ ensurePleScaledProjectionNormWeight,
10
+ } from './per-layer-inputs.js';
11
+
12
+ export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
13
+ return hitStop !== true
14
+ && Number.isInteger(actualCount)
15
+ && Number.isInteger(requestedCount)
16
+ && actualCount > 0
17
+ && actualCount < requestedCount;
18
+ }
19
+
20
+ export function resolveHotVocabularyBatchDecodeAvailability({
21
+ hasRangeBackedPerLayerInputs,
22
+ pleHotVocabularyRuntime,
23
+ tokenId,
24
+ }) {
25
+ if (hasRangeBackedPerLayerInputs !== true) {
26
+ return false;
27
+ }
28
+ if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
29
+ return false;
30
+ }
31
+ const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
32
+ if (!Number.isInteger(sentinelIndex)) {
33
+ return false;
34
+ }
35
+ if (!Number.isInteger(tokenId)) {
36
+ return false;
37
+ }
38
+ const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
39
+ return hotTokenIndex !== sentinelIndex;
40
+ }
41
+
42
+ export async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
43
+ const perLayerInputsSession = resolvePerLayerInputsSession(
44
+ state.modelConfig?.perLayerInputsSession ?? null,
45
+ state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
46
+ );
47
+ const materialization = perLayerInputsSession?.materialization;
48
+ if (state.debug) {
49
+ log.debug(
50
+ 'Pipeline',
51
+ `PLE materialization session=${materialization ?? 'null'} ` +
52
+ `modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
53
+ `runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
54
+ `manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
55
+ );
56
+ }
57
+ await ensurePleGpuHotVocabularyRuntime({
58
+ config: state.modelConfig,
59
+ weights: state.weights,
60
+ perLayerInputsSession,
61
+ debugFlags: state.debugFlags,
62
+ tokenizer: state.tokenizer ?? null,
63
+ seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
64
+ });
65
+ await ensurePleGpuSplitTablesRuntime({
66
+ config: state.modelConfig,
67
+ weights: state.weights,
68
+ perLayerInputsSession,
69
+ debugFlags: state.debugFlags,
70
+ });
71
+ await ensurePleScaledProjectionNormWeight({
72
+ config: state.modelConfig,
73
+ weights: state.weights,
74
+ weightConfig: {
75
+ rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
76
+ },
77
+ debugFlags: state.debugFlags,
78
+ });
79
+ }
80
+
81
+ export function recordPrefillProfileStep(state, entry) {
82
+ if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
83
+ if (!state.stats.prefillProfileSteps) {
84
+ state.stats.prefillProfileSteps = [];
85
+ }
86
+ state.stats.prefillProfileSteps.push(entry);
87
+ }
88
+
89
+ /**
90
+ * Resolve display text for a token ID sequence.
91
+ *
92
+ * Preference order:
93
+ * 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
94
+ * 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
95
+ * 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
96
+ * unless primary returned empty string and fallback looks like a special token (<...>),
97
+ * in which case empty string is preserved to keep skip-special behavior deterministic.
98
+ * 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
99
+ * 5. fallbackText — static fallback string (default '?').
100
+ */
101
+ export function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
102
+ const renderPrimary = typeof renderTokenText === 'function'
103
+ ? renderTokenText
104
+ : (ids) => tokenizer?.decode?.(ids);
105
+ const renderFallback = typeof renderFallbackTokenText === 'function'
106
+ ? renderFallbackTokenText
107
+ : (ids) => tokenizer?.decode?.(ids, false);
108
+
109
+ const primaryText = renderPrimary(tokenIds);
110
+ if (typeof primaryText === 'string' && primaryText.length > 0) {
111
+ return primaryText;
112
+ }
113
+
114
+ const fallback = renderFallback(tokenIds);
115
+ if (typeof fallback === 'string' && fallback.length > 0) {
116
+ // Keep skip-special behavior deterministic: if primary decoding filtered this
117
+ // token to empty, do not reintroduce obvious special-token text via fallback.
118
+ if (
119
+ primaryText === ''
120
+ && /^<[^>\n]{1,80}>$/.test(fallback.trim())
121
+ ) {
122
+ return '';
123
+ }
124
+ return fallback;
125
+ }
126
+
127
+ return fallbackText;
128
+ }
129
+
130
+ export function usesReplayPrefillDecode(state) {
131
+ return state?.modelConfig?.decodeStrategy === 'replay_prefill';
132
+ }
133
+
134
+ export function assertIncrementalDecodeSupport(state, operation) {
135
+ if (!usesReplayPrefillDecode(state)) {
136
+ return;
137
+ }
138
+ throw new Error(
139
+ `[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
140
+ 'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
141
+ 'explicit layerTypes for mixed-geometry/shared-KV decode.'
142
+ );
143
+ }
144
+
145
+ export function summarizeExecutionPlan(plan) {
146
+ if (!plan) {
147
+ return null;
148
+ }
149
+ if (typeof plan !== 'object') {
150
+ log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
151
+ return null;
152
+ }
153
+ if (typeof plan.id !== 'string') {
154
+ log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
155
+ }
156
+ if (typeof plan.activationDtype !== 'string') {
157
+ log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
158
+ }
159
+ return {
160
+ id: plan.id,
161
+ kernelPathId: plan.kernelPathId ?? null,
162
+ kernelPathSource: plan.kernelPathSource ?? 'none',
163
+ activationDtype: plan.activationDtype,
164
+ readbackInterval: plan.readbackInterval ?? null,
165
+ batchSize: plan.defaultBatchSize,
166
+ stopCheckMode: plan.defaultStopCheckMode,
167
+ ringTokens: plan.ringTokens ?? null,
168
+ ringStop: plan.ringStop ?? null,
169
+ ringStaging: plan.ringStaging ?? null,
170
+ };
171
+ }
172
+
173
+ export function shouldRetryWithFinitenessFallback(error) {
174
+ if (error?.name === 'FinitenessError') {
175
+ return true;
176
+ }
177
+ const message = typeof error?.message === 'string'
178
+ ? error.message
179
+ : (typeof error === 'string' ? error : '');
180
+ if (!message.startsWith('[Sampling]')) {
181
+ return false;
182
+ }
183
+ return message.includes('no finite candidate logits after masking the pad token')
184
+ || message.includes('Softmax produced no finite candidate probabilities');
185
+ }
186
+
187
+ export function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
188
+ const activePlan = resolveActiveExecutionPlan(state);
189
+ const wrapped = new Error(
190
+ `[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
191
+ `"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
192
+ 'Resolve the unstable path with an explicit capability-aware execution override, ' +
193
+ 'or opt into alternate-plan recovery with ' +
194
+ 'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
195
+ error instanceof Error ? { cause: error } : undefined
196
+ );
197
+ wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
198
+ return wrapped;
199
+ }
200
+
201
+ function assertResolvedKVDtype(kvDtype, contextLabel) {
202
+ if (kvDtype === 'f16' || kvDtype === 'f32') {
203
+ return kvDtype;
204
+ }
205
+ throw new Error(
206
+ `[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
207
+ `got ${JSON.stringify(kvDtype)}.`
208
+ );
209
+ }
210
+
211
+ export function resolveTargetPlanKVDtype(plan, contextLabel) {
212
+ return assertResolvedKVDtype(
213
+ plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
214
+ contextLabel
215
+ );
216
+ }
217
+
218
+ export function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
219
+ return assertResolvedKVDtype(
220
+ state?.kvCache?.kvDtype
221
+ ?? plan?.kernelPath?.kvDtype
222
+ ?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
223
+ ?? plan?.activationDtype
224
+ ?? null,
225
+ contextLabel
226
+ );
227
+ }
228
+
229
+ export function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
230
+ const runtimeInference = state?.runtimeConfig?.inference;
231
+ if (!runtimeInference?.session?.kvcache) {
232
+ throw new Error(
233
+ '[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
234
+ );
235
+ }
236
+ return mergeRuntimeValues(runtimeInference, {
237
+ session: {
238
+ kvcache: {
239
+ kvDtype,
240
+ },
241
+ },
242
+ });
243
+ }
@@ -40,8 +40,6 @@ export declare function releaseSharedAttentionState(
40
40
  } | null
41
41
  ): void;
42
42
 
43
- export declare function getWeightBufferConfig(state: PipelineState): WeightBufferConfig;
44
-
45
43
  export declare function getLogitsWeights(state: PipelineState): LogitsWeights;
46
44
 
47
45
  export declare function getLogitsConfig(state: PipelineState): LogitsConfig;
@@ -8,10 +8,7 @@ import { decodeReadback } from './debug-utils/index.js';
8
8
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
9
9
  import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
10
10
  import { resolveActiveExecutionPlan } from './execution-plan.js';
11
-
12
- function isPlainObject(value) {
13
- return value != null && typeof value === 'object' && !Array.isArray(value);
14
- }
11
+ import { isPlainObject } from '../../../utils/plain-object.js';
15
12
 
16
13
  export function resolvePerLayerInputsSession(manifestSession, runtimeSession) {
17
14
  if (!isPlainObject(runtimeSession)) {
@@ -161,6 +158,7 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
161
158
  step: state.decodeStepCount,
162
159
  phase: isDecodeMode ? 'decode' : 'prefill',
163
160
  operatorDiagnostics: state.operatorDiagnostics,
161
+ useFusedGateUpGelu: state.runtimeConfig?.inference?.session?.useFusedGateUpGelu === true,
164
162
  };
165
163
  }
166
164
 
@@ -191,7 +189,7 @@ export function releaseSharedAttentionState(sharedAttentionState, recorder = nul
191
189
  }
192
190
 
193
191
 
194
- export function getWeightBufferConfig(state) {
192
+ function getWeightBufferConfig(state) {
195
193
  return {
196
194
  rmsNormWeightOffset: state.modelConfig.rmsNormWeightOffset,
197
195
  };
@@ -228,5 +226,6 @@ export function getLogitsConfig(state) {
228
226
  activationDtype: effectiveActivationDtype,
229
227
  kernelPath: activeKernelPath,
230
228
  executionPolicies: state.executionV1State?.policies ?? null,
229
+ debugProbes: state.runtimeConfig.shared.debug.probes,
231
230
  };
232
231
  }