doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -31,6 +31,7 @@ import {
31
31
  applyAttentionValueNorm,
32
32
  } from './projections.js';
33
33
  import { prepareAttentionProjectionInput } from './output-projection.js';
34
+ import { runProbes } from '../probes.js';
34
35
 
35
36
  import { releaseOrTrack, shouldDebugLayer } from './types.js';
36
37
  import {
@@ -47,10 +48,12 @@ import {
47
48
  buildContiguousQuantAttentionOptions,
48
49
  } from './quant-options.js';
49
50
  import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
51
+ import { getRuntimeConfig } from '../../../../config/runtime.js';
50
52
  import {
51
53
  resolveAttentionPrecisionContract,
52
54
  isAttentionKvDtypeExplicit,
53
55
  } from './precision-contract.js';
56
+ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
54
57
 
55
58
  const ATTENTION_DTYPE_LOGGED = new Set();
56
59
 
@@ -135,6 +138,7 @@ export async function recordLayerAttentionGPU(
135
138
  let finalOutput = null;
136
139
  let oProjInputTemp = null;
137
140
  let retainSharedKvBuffers = false;
141
+ let valueAliasesKey = false;
138
142
  if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
139
143
  assertAttentionDtypeTransitionAllowed(
140
144
  state,
@@ -160,9 +164,37 @@ export async function recordLayerAttentionGPU(
160
164
  const kvSize = numTokens * numKVHeads * headDim;
161
165
 
162
166
  // 1. Input norm
167
+ // Opt-in fusion: when useFusedRmsnormWideTile is set, defer the standalone
168
+ // rmsnorm into each q/k/v_proj matmul (fused kernel runs norm internally).
169
+ let fusedNormWeightRec = null;
170
+ let fusedNormEpsRec = null;
171
+ let fusedNormOffsetRec = false;
172
+ let fusedNormOwnedRec = false;
173
+ // state.runtimeConfig is a stale snapshot whose inference.session is an empty
174
+ // object; the live module-level runtime config carries the merged
175
+ // profile/override session. Read session-level flags from getRuntimeConfig().
176
+ const rmsNormFusionFlagRec = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
177
+ const canSelectFusedRmsNormProjectionRec = canUseRmsNormWideTileProjectionFusion(
178
+ layerWeights,
179
+ sharedKVSourceLayerIdx != null
180
+ );
181
+ const canFuseInputNormProjRec = rmsNormFusionFlagRec
182
+ && canSelectFusedRmsNormProjectionRec
183
+ && !skipInputNorm
184
+ && layerWeights.inputNorm
185
+ && getNormWeightBuffer
186
+ && isPrefill
187
+ && numTokens > 1
188
+ && attentionInput.dtype === 'f32';
163
189
 
164
190
  try {
165
- if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
191
+ if (canFuseInputNormProjRec) {
192
+ fusedNormWeightRec = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
193
+ fusedNormEpsRec = rmsNormEps;
194
+ fusedNormOffsetRec = config.rmsNormWeightOffset === true;
195
+ fusedNormOwnedRec = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
196
+ // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
197
+ } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
166
198
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
167
199
  normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
168
200
  batchSize: numTokens,
@@ -170,6 +202,11 @@ export async function recordLayerAttentionGPU(
170
202
  rmsNormWeightOffset: config.rmsNormWeightOffset,
171
203
  });
172
204
  if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
205
+ await runProbes('post_input_norm', normed.buffer, {
206
+ layerIdx, numTokens, hiddenSize,
207
+ probes: state.debugProbes, recorder,
208
+ operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
209
+ });
173
210
  }
174
211
 
175
212
  const debugLayers = debugFlags.debugLayers;
@@ -217,7 +254,7 @@ export async function recordLayerAttentionGPU(
217
254
  }
218
255
  const reusesSharedKV = sharedKVEntry != null;
219
256
  retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
220
- ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
257
+ ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey } = await projectAttentionQKV({
221
258
  recorder,
222
259
  normed,
223
260
  layerWeights,
@@ -242,7 +279,33 @@ export async function recordLayerAttentionGPU(
242
279
  trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
243
280
  }
244
281
  : null,
282
+ fusedNormWeight: fusedNormWeightRec,
283
+ fusedNormEps: fusedNormEpsRec,
284
+ fusedNormOffset: fusedNormOffsetRec,
245
285
  }));
286
+ // Deferred release of the norm weight buffer for fused path.
287
+ if (fusedNormWeightRec && fusedNormOwnedRec) {
288
+ releaseOrTrack(recorder, fusedNormWeightRec);
289
+ }
290
+ await runProbes('q_proj', qTensor.buffer, {
291
+ layerIdx, numTokens, hiddenSize: numHeads * headDim,
292
+ probes: state.debugProbes, recorder,
293
+ operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
294
+ });
295
+ if (kTensor) {
296
+ await runProbes('k_proj', kTensor.buffer, {
297
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
298
+ probes: state.debugProbes, recorder,
299
+ operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
300
+ });
301
+ }
302
+ if (vTensor) {
303
+ await runProbes('v_proj', vTensor.buffer, {
304
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
305
+ probes: state.debugProbes, recorder,
306
+ operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
307
+ });
308
+ }
246
309
 
247
310
  // Optional per-head Q/K normalization.
248
311
  // Some models use RMSNorm with (1+weight) offset formula, controlled by rmsNormWeightOffset.
@@ -264,9 +327,23 @@ export async function recordLayerAttentionGPU(
264
327
  rmsNormWeightOffset: config.rmsNormWeightOffset,
265
328
  releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
266
329
  skipKNorm: reusesSharedKV,
330
+ retainKInput: valueAliasesKey,
267
331
  }));
332
+ await runProbes('q_norm', qTensor.buffer, {
333
+ layerIdx, numTokens, hiddenSize: numHeads * headDim,
334
+ probes: state.debugProbes, recorder,
335
+ operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
336
+ });
337
+ if (kTensor) {
338
+ await runProbes('k_norm', kTensor.buffer, {
339
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
340
+ probes: state.debugProbes, recorder,
341
+ operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
342
+ });
343
+ }
268
344
 
269
345
  if (config.valueNorm === true && !reusesSharedKV) {
346
+ const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
270
347
  vTensor = await applyAttentionValueNorm({
271
348
  recorder,
272
349
  vTensor,
@@ -274,7 +351,16 @@ export async function recordLayerAttentionGPU(
274
351
  numTokens,
275
352
  numKVHeads,
276
353
  headDim,
277
- releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
354
+ releaseTemporary: (buffer) => {
355
+ if (!valueNormInputAliasesKey) {
356
+ releaseOrTrack(recorder, buffer);
357
+ }
358
+ },
359
+ });
360
+ await runProbes('v_norm', vTensor.buffer, {
361
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
362
+ probes: state.debugProbes, recorder,
363
+ operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
278
364
  });
279
365
  }
280
366
 
@@ -287,6 +373,7 @@ export async function recordLayerAttentionGPU(
287
373
  numHeads,
288
374
  headDim,
289
375
  rotaryDim: config.ropeRotaryDim,
376
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
290
377
  interleaved: config.ropeInterleaved,
291
378
  startPos: currentSeqLen,
292
379
  executionPolicies: state.executionPolicies ?? null,
@@ -296,6 +383,7 @@ export async function recordLayerAttentionGPU(
296
383
  numHeads: numKVHeads,
297
384
  headDim,
298
385
  rotaryDim: config.ropeRotaryDim,
386
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
299
387
  interleaved: config.ropeInterleaved,
300
388
  startPos: currentSeqLen,
301
389
  executionPolicies: state.executionPolicies ?? null,
@@ -507,6 +595,15 @@ export async function recordLayerAttentionGPU(
507
595
  if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
508
596
  if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
509
597
  }
598
+ // Session precedence is runtime-over-manifest per config-style-guide
599
+ // §Category Rules. getRuntimeConfig() returns the merged session (manifest
600
+ // is the base layer, runtime fields win field-by-field via merge.js).
601
+ // Kernel enforces head_dim=256, f16 KV, contiguous layout; only applies
602
+ // when numTokens > 1 (prefill). Same flag semantics as the non-recorder
603
+ // path in ./run.js.
604
+ const mergedSessionRec = getRuntimeConfig()?.inference?.session;
605
+ const useFlashPrefillRec = mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
606
+ const useOrtFlashPrefillRec = mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
510
607
  return recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
511
608
  seqLen: numTokens,
512
609
  kvLen: kvState.kvLenForAttention,
@@ -524,6 +621,8 @@ export async function recordLayerAttentionGPU(
524
621
  kvPageTable: kvState.kvPageTable,
525
622
  kvPageSize: kvState.kvPageSize,
526
623
  kernelPath,
624
+ useFlashPrefill: useFlashPrefillRec,
625
+ useOrtFlashPrefill: useOrtFlashPrefillRec,
527
626
  });
528
627
  },
529
628
  };
@@ -533,13 +632,25 @@ export async function recordLayerAttentionGPU(
533
632
  }
534
633
 
535
634
  attnOutput = await runAttentionKernel();
635
+ await runProbes('attn_core_out', attnOutput.buffer, {
636
+ layerIdx,
637
+ numTokens,
638
+ hiddenSize: numHeads * headDim,
639
+ probes: state.debugProbes,
640
+ recorder,
641
+ operatorDiagnostics: state.operatorDiagnostics,
642
+ dtype: attnOutput.dtype,
643
+ });
536
644
 
537
645
  attnForProjection = attnOutput;
538
646
  if (qGateTensor) {
647
+ // Mirror run.js gate dispatch. Qwen3_5/Qwen 3.6 full attention applies
648
+ // sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
649
+ const gateActivation = 'sigmoid';
539
650
  attnForProjection = await recordSiLU(recorder, attnOutput, {
540
651
  size: numTokens * numHeads * headDim,
541
652
  gate: qGateTensor,
542
- gateActivation: 'sigmoid',
653
+ gateActivation,
543
654
  inputActivation: 'identity',
544
655
  swigluLimit: null,
545
656
  });
@@ -659,7 +770,9 @@ export async function recordLayerAttentionGPU(
659
770
  }
660
771
  if (!retainSharedKvBuffers) {
661
772
  recorder.trackTemporaryBuffer(kTensor.buffer);
662
- recorder.trackTemporaryBuffer(vTensor.buffer);
773
+ if (vTensor.buffer !== kTensor.buffer) {
774
+ recorder.trackTemporaryBuffer(vTensor.buffer);
775
+ }
663
776
  }
664
777
  for (const buffer of buffersToTrack) {
665
778
  recorder.trackTemporaryBuffer(buffer);
@@ -0,0 +1,8 @@
1
+ import type { LayerWeights } from '../types.js';
2
+
3
+ export declare function hasQ4KMaterialization(weight: unknown): boolean;
4
+
5
+ export declare function canUseRmsNormWideTileProjectionFusion(
6
+ layerWeights: LayerWeights | null | undefined,
7
+ reusesSharedKV: boolean
8
+ ): boolean;
@@ -0,0 +1,19 @@
1
+ import { isWeightBuffer } from '../../../../gpu/weight-buffer.js';
2
+
3
+ export function hasQ4KMaterialization(weight) {
4
+ return isWeightBuffer(weight) && Boolean(weight.materializations?.q4k?.buffer);
5
+ }
6
+
7
+ export function canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV) {
8
+ if (hasQ4KMaterialization(layerWeights?.qkvProj)) {
9
+ return true;
10
+ }
11
+ if (!hasQ4KMaterialization(layerWeights?.qProj)) {
12
+ return false;
13
+ }
14
+ if (reusesSharedKV) {
15
+ return true;
16
+ }
17
+ return hasQ4KMaterialization(layerWeights?.kProj)
18
+ && (layerWeights?.vProj == null || hasQ4KMaterialization(layerWeights.vProj));
19
+ }
@@ -2,6 +2,7 @@
2
2
 
3
3
  import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
4
  import { getDevice } from '../../../../gpu/device.js';
5
+ import { getRuntimeConfig } from '../../../../config/runtime.js';
5
6
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
6
7
  import {
7
8
  runMatmul,
@@ -58,6 +59,7 @@ import {
58
59
  resolveAttentionPrecisionContract,
59
60
  isAttentionKvDtypeExplicit,
60
61
  } from './precision-contract.js';
62
+ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
61
63
 
62
64
  const ATTENTION_DTYPE_LOGGED = new Set();
63
65
 
@@ -140,6 +142,7 @@ export async function runLayerAttentionGPU(
140
142
  let output = null;
141
143
  let finalOutput = null;
142
144
  let oProjInputTemp = null;
145
+ let valueAliasesKey = false;
143
146
  if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
144
147
  assertAttentionDtypeTransitionAllowed(
145
148
  state,
@@ -192,9 +195,36 @@ export async function runLayerAttentionGPU(
192
195
  const reusesSharedKV = sharedKVEntry != null;
193
196
 
194
197
  // 1. Input norm
195
-
198
+ // Opt-in fusion: when useFusedRmsnormWideTile is set AND preconditions match,
199
+ // defer the rmsnorm into each q/k/v_proj matmul via a dedicated WideTile
200
+ // variant. Saves one standalone rmsnorm dispatch per layer.
201
+ let fusedNormWeight = null;
202
+ let fusedNormEps = null;
203
+ let fusedNormOffset = false;
204
+ let fusedNormWeightOwned = false;
205
+ // state.runtimeConfig.inference.session is a stale snapshot ({}); the live
206
+ // module-level runtime config carries the merged profile/override session.
207
+ const rmsNormFusionFlag = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
208
+ const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
209
+ const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
210
+ const canFuseInputNormIntoProjections = rmsNormFusionFlag
211
+ && hasInputNormWeight
212
+ && canSelectFusedRmsNormProjection
213
+ && !skipInputNorm
214
+ && isPrefill
215
+ && numTokens > 1
216
+ && attentionInput.dtype === 'f32';
217
+
196
218
  try {
197
- if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
219
+ if (canFuseInputNormIntoProjections) {
220
+ const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
221
+ fusedNormWeight = normWeightBuf;
222
+ fusedNormEps = rmsNormEps;
223
+ fusedNormOffset = config.rmsNormWeightOffset === true;
224
+ fusedNormWeightOwned = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
225
+ // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs the
226
+ // norm internally via the fused kernel variant.
227
+ } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
198
228
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
199
229
  try {
200
230
  // Debug: norm weights for configured layers
@@ -275,7 +305,7 @@ export async function runLayerAttentionGPU(
275
305
  }),
276
306
  });
277
307
  let usedFusedQKV = false;
278
- ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
308
+ ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey } = await projectAttentionQKV({
279
309
  recorder: null,
280
310
  normed,
281
311
  layerWeights,
@@ -300,7 +330,14 @@ export async function runLayerAttentionGPU(
300
330
  trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
301
331
  }
302
332
  : null,
333
+ fusedNormWeight,
334
+ fusedNormEps,
335
+ fusedNormOffset,
303
336
  }));
337
+ // Release the in-flight norm weight buffer used by the fused projections.
338
+ if (fusedNormWeight && fusedNormWeightOwned) {
339
+ releaseBuffer(fusedNormWeight);
340
+ }
304
341
 
305
342
  // Trace Q/K/V projections
306
343
  if (kernelTrace.enabled) {
@@ -413,9 +450,11 @@ export async function runLayerAttentionGPU(
413
450
  });
414
451
  }
415
452
  : null,
453
+ retainKInput: valueAliasesKey,
416
454
  }));
417
455
 
418
456
  if (config.valueNorm === true && !reusesSharedKV) {
457
+ const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
419
458
  vTensor = await applyAttentionValueNorm({
420
459
  recorder: null,
421
460
  vTensor,
@@ -423,7 +462,19 @@ export async function runLayerAttentionGPU(
423
462
  numTokens,
424
463
  numKVHeads,
425
464
  headDim,
426
- releaseTemporary: (buffer) => releaseBuffer(buffer),
465
+ releaseTemporary: (buffer) => {
466
+ if (!valueNormInputAliasesKey) {
467
+ releaseBuffer(buffer);
468
+ }
469
+ },
470
+ });
471
+ await runProbes('v_norm', vTensor.buffer, {
472
+ layerIdx,
473
+ numTokens,
474
+ hiddenSize: numKVHeads * headDim,
475
+ probes: state.debugProbes,
476
+ operatorDiagnostics: state.operatorDiagnostics,
477
+ dtype: vTensor.dtype,
427
478
  });
428
479
  }
429
480
 
@@ -456,6 +507,7 @@ export async function runLayerAttentionGPU(
456
507
  numHeads,
457
508
  headDim,
458
509
  rotaryDim: config.ropeRotaryDim,
510
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
459
511
  interleaved: config.ropeInterleaved,
460
512
  startPos: currentSeqLen,
461
513
  executionPolicies: state.executionPolicies ?? null,
@@ -465,6 +517,7 @@ export async function runLayerAttentionGPU(
465
517
  numHeads: numKVHeads,
466
518
  headDim,
467
519
  rotaryDim: config.ropeRotaryDim,
520
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
468
521
  interleaved: config.ropeInterleaved,
469
522
  startPos: currentSeqLen,
470
523
  executionPolicies: state.executionPolicies ?? null,
@@ -763,6 +816,15 @@ export async function runLayerAttentionGPU(
763
816
  kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
764
817
  vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
765
818
  }
819
+ // Session precedence is runtime-over-manifest per config-style-guide
820
+ // §Category Rules: `resolvedSession = merge(manifest.session, runtime.session)`
821
+ // with runtime fields winning. getRuntimeConfig() returns that merged
822
+ // session (state.runtimeConfig.inference.session is a stale snapshot —
823
+ // see comment above at the rmsNorm fusion check). The kernel itself
824
+ // enforces remaining preconditions (head_dim=256, etc.).
825
+ const mergedSession = getRuntimeConfig()?.inference?.session;
826
+ const useFlashPrefill = mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
827
+ const useOrtFlashPrefill = mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
766
828
  const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
767
829
  seqLen: numTokens,
768
830
  kvLen: kvState.kvLenForAttention,
@@ -780,6 +842,8 @@ export async function runLayerAttentionGPU(
780
842
  kvPageTable: kvState.kvPageTable,
781
843
  kvPageSize: kvState.kvPageSize,
782
844
  kernelPath,
845
+ useFlashPrefill,
846
+ useOrtFlashPrefill,
783
847
  });
784
848
  if (prefillFallbackNeedsCast) {
785
849
  if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
@@ -795,6 +859,14 @@ export async function runLayerAttentionGPU(
795
859
  }
796
860
 
797
861
  attnOutput = await runAttentionKernel();
862
+ await runProbes('attn_core_out', attnOutput.buffer, {
863
+ layerIdx,
864
+ numTokens,
865
+ hiddenSize: numHeads * headDim,
866
+ probes: state.debugProbes,
867
+ operatorDiagnostics: state.operatorDiagnostics,
868
+ dtype: attnOutput.dtype,
869
+ });
798
870
 
799
871
  // Trace attention output
800
872
  if (kernelTrace.enabled) {
@@ -819,10 +891,27 @@ export async function runLayerAttentionGPU(
819
891
 
820
892
  attnForProjection = attnOutput;
821
893
  if (qGateTensor) {
894
+ // The Qwen3_5 reference (HF transformers) full-attention path
895
+ // applies sigmoid(gate) — NOT silu(gate) — to the attention
896
+ // output regardless of how the architecture's gate label is
897
+ // surfaced in the HF config. See Qwen3_5Attention.forward in
898
+ // transformers/models/qwen3_5/modeling_qwen3_5.py:
899
+ // attn_output = attn_output * torch.sigmoid(gate)
900
+ // The earlier `swish` -> `silu` mapping here was incorrect and
901
+ // produced sign-flipped output for any token whose gate value
902
+ // was negative (silu is sign-preserving via x * sigmoid(x);
903
+ // sigmoid is unconditionally in (0, 1)). At Qwen 3.6 27B scale
904
+ // this drove residual-stream divergence by L=3 (HF reference vs
905
+ // upstream rel_l2≈1.37, mean per-token correlation ≈ -0.96 — a
906
+ // sign-flip pattern). config.outputGateType is retained for
907
+ // future architectures whose gate truly is silu, but Qwen 3.6's
908
+ // `output_gate_type=swish` still routes here as sigmoid because
909
+ // that is what the reference forward computes.
910
+ const gateActivation = 'sigmoid';
822
911
  attnForProjection = await runSiLU(attnOutput, {
823
912
  size: numTokens * numHeads * headDim,
824
913
  gate: qGateTensor,
825
- gateActivation: 'sigmoid',
914
+ gateActivation,
826
915
  inputActivation: 'identity',
827
916
  swigluLimit: null,
828
917
  });
@@ -973,7 +1062,9 @@ export async function runLayerAttentionGPU(
973
1062
  }
974
1063
  if (!reusesSharedKV && !storeSharedKV) {
975
1064
  releaseBuffer(kTensor.buffer);
976
- releaseBuffer(vTensor.buffer);
1065
+ if (vTensor.buffer !== kTensor.buffer) {
1066
+ releaseBuffer(vTensor.buffer);
1067
+ }
977
1068
  }
978
1069
  for (const buffer of buffersToRelease) {
979
1070
  releaseBuffer(buffer);
@@ -56,6 +56,8 @@ export interface AttentionConfig {
56
56
  } | null;
57
57
  /** RoPE rotary dimension (may differ from headDim with partial rotary). */
58
58
  ropeRotaryDim?: number;
59
+ /** RoPE frequency base dimension; non-interleaved kernels use this as the rotate-half pair span. */
60
+ ropeFrequencyBaseDim?: number;
59
61
  /** Whether RoPE uses interleaved layout. */
60
62
  ropeInterleaved?: boolean;
61
63
  /** Token IDs for the current micro-batch (required by BDPA KV ingestion). */
@@ -36,18 +36,8 @@ export type ChatTemplateType =
36
36
  | 'translategemma'
37
37
  | null;
38
38
 
39
- export declare function formatGemmaChat(messages: ChatMessage[]): string;
40
-
41
39
  export interface ChatFormatOptions {
42
40
  thinking?: boolean;
43
41
  }
44
42
 
45
- export declare function formatGemma4Chat(messages: ChatMessage[], options?: ChatFormatOptions): string;
46
-
47
- export declare function formatLlama3Chat(messages: ChatMessage[]): string;
48
-
49
- export declare function formatGptOssChat(messages: ChatMessage[]): string;
50
-
51
- export declare function formatTranslateGemmaChat(messages: ChatMessage[]): string;
52
-
53
43
  export declare function formatChatMessages(messages: ChatMessage[], templateType?: ChatTemplateType, options?: ChatFormatOptions): string;
@@ -229,7 +229,7 @@ function formatGemma4(messages, options) {
229
229
  if (options?.thinking === true) {
230
230
  parts.push('<|turn>model\n<|think|>\n');
231
231
  } else {
232
- parts.push('<|turn>model\n');
232
+ parts.push('<|turn>model\n<|channel>thought\n<channel|>');
233
233
  }
234
234
  return parts.join('');
235
235
  }
@@ -440,10 +440,3 @@ export function formatChatMessages(messages, templateType, options) {
440
440
  }
441
441
  throw new Error(`Unrecognized chat template type: ${templateType}`);
442
442
  }
443
-
444
- export const formatGemmaChat = formatTurnBased;
445
- export const formatGemma4Chat = formatGemma4;
446
- export const formatLlama3Chat = formatHeaderBased;
447
- export const formatGptOssChat = formatChannelBased;
448
- export const formatQwenChat = formatQwen;
449
- export const formatTranslateGemmaChat = formatTranslateGemma;
@@ -46,6 +46,8 @@ export interface RawConfig {
46
46
  n_head?: number;
47
47
  attentionHeadCount?: number;
48
48
  num_key_value_heads?: number;
49
+ num_global_key_value_heads?: number;
50
+ num_global_kv_heads?: number;
49
51
  attentionHeadCountKV?: number;
50
52
  head_dim?: number;
51
53
  global_head_dim?: number;
@@ -109,12 +111,6 @@ export interface TensorInfo {
109
111
 
110
112
  export type Manifest = RuntimeModelContract;
111
113
 
112
- export interface AttentionParams {
113
- numHeads: number;
114
- numKVHeads: number;
115
- headDim: number;
116
- }
117
-
118
114
  export interface ParsedModelConfig {
119
115
  numLayers: number;
120
116
  hiddenSize: number;
@@ -123,6 +119,7 @@ export interface ParsedModelConfig {
123
119
  maxIntermediateSize: number;
124
120
  numHeads: number;
125
121
  numKVHeads: number;
122
+ numGlobalKVHeads: number | null;
126
123
  headDim: number;
127
124
  globalHeadDim: number | null;
128
125
  vocabSize: number;
@@ -250,12 +247,6 @@ export interface AudioEncoderConfig {
250
247
  export function getStopTokenIds(manifest: Manifest): number[];
251
248
  export function resolveLayerIntermediateSize(config: ParsedModelConfig, layerIdx: number): number;
252
249
 
253
- /**
254
- * Validate that a chatTemplate.type is a known formatter type.
255
- * Logs a warning for unknown types. Returns true if valid or null.
256
- */
257
- export function validateChatTemplateType(type: string | null | undefined, modelId: string): boolean;
258
-
259
250
  /**
260
251
  * Extended manifest with inference config for manifest-first parsing.
261
252
  */
@@ -270,24 +261,11 @@ export interface ManifestWithInference {
270
261
  eos_token_id: number | number[];
271
262
  }
272
263
 
273
- /**
274
- * Check if manifest has inference config for manifest-first parsing.
275
- */
276
- export function hasManifestInference(manifest: Manifest): manifest is Manifest & { inference: ManifestInferenceSchema };
277
-
278
264
  export function validateRequiredInferenceFields(
279
265
  inf: ManifestInferenceSchema,
280
266
  modelId: string
281
267
  ): void;
282
268
 
283
- /**
284
- * Convert MergedConfig to ParsedModelConfig.
285
- */
286
- export function toParsedConfigFromMerged(
287
- merged: MergedConfig,
288
- manifest: ManifestWithInference
289
- ): ParsedModelConfig;
290
-
291
269
  /**
292
270
  * Parse model config from manifest using manifest-first resolution.
293
271
  */