doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -19,7 +19,6 @@ import {
19
19
  import { getWeightBuffer, getNormWeightBuffer } from './weights.js';
20
20
  import { logLayer, logAttn, getBufferStats, isKernelDebugEnabled, dumpTokenVector, logKernelStep, shouldDebugLayerOutput } from './debug-utils/index.js';
21
21
  import { runProbes } from './probes.js';
22
- import { getLayerPlanSteps, filterLayerPlanStepsByPhase } from './layer-plan.js';
23
22
  import { selectRuleValue } from '../../../rules/rule-registry.js';
24
23
  import { recordCheckFiniteness } from '../../../gpu/kernels/check-finiteness.js';
25
24
  import { shouldRunFinitenessGuard } from './finiteness-policy.js';
@@ -27,6 +26,7 @@ import { runLinearAttentionLayer } from './linear-attention.js';
27
26
  import { validateAttnConfig } from './attention/attn-config.js';
28
27
  import { createPerLayerInputTensor, resolveDensePleProjectionWeight } from './per-layer-inputs.js';
29
28
  import { isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
29
+ import { processLayerPlanGPU } from './layer-plan-gpu.js';
30
30
 
31
31
  // ============================================================================
32
32
  // Architecture Detection
@@ -60,7 +60,7 @@ export function isMoELayer(layerIdx, config) {
60
60
  return true;
61
61
  }
62
62
 
63
- function resolveActivationDtype(dtype) {
63
+ export function resolveActivationDtype(dtype) {
64
64
  return selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype });
65
65
  }
66
66
 
@@ -88,12 +88,12 @@ function assertSupportedLayerRuntime(layerIdx, config) {
88
88
  }
89
89
  }
90
90
 
91
- function getConvLayerState(convLayerStates, layerIdx) {
91
+ export function getConvLayerState(convLayerStates, layerIdx) {
92
92
  if (!convLayerStates) return {};
93
93
  return convLayerStates.get(layerIdx) ?? {};
94
94
  }
95
95
 
96
- function isSlidingLayerType(layerType) {
96
+ export function isSlidingLayerType(layerType) {
97
97
  const normalized = normalizeLayerType(layerType);
98
98
  return normalized === 'sliding_attention'
99
99
  || normalized === 'local_attention'
@@ -125,21 +125,57 @@ function isLinearLayerType(layerType) {
125
125
  || normalized === 'gated_delta_net';
126
126
  }
127
127
 
128
- function resolveAttentionRotaryDim(config, layerType) {
128
+ export function resolveAttentionRotaryDim(config, layerType) {
129
129
  if (isSlidingLayerType(layerType)) {
130
130
  return config.ropeLocalRotaryDim ?? config.ropeRotaryDim;
131
131
  }
132
132
  return config.ropeRotaryDim;
133
133
  }
134
134
 
135
- function resolveAttentionHeadDim(config, layerType) {
135
+ export function resolveAttentionFrequencyBaseDim(config, layerType) {
136
+ if (isSlidingLayerType(layerType)) {
137
+ return config.ropeLocalFrequencyBaseDim ?? resolveAttentionRotaryDim(config, layerType);
138
+ }
139
+ return config.ropeFrequencyBaseDim ?? config.ropeRotaryDim;
140
+ }
141
+
142
+ export function resolveAttentionHeadDim(config, layerType) {
136
143
  if (isSlidingLayerType(layerType)) {
137
144
  return config.headDim;
138
145
  }
139
146
  return config.globalHeadDim ?? config.headDim;
140
147
  }
141
148
 
142
- function resolveAttentionKVSharing(config, layerIdx, layerType) {
149
+ function resolveProjectionOutputRows(layerWeight, hiddenSize) {
150
+ if (!layerWeight || !Array.isArray(layerWeight.shape) || layerWeight.shape.length < 2) {
151
+ return null;
152
+ }
153
+ const dim0 = Number(layerWeight.shape[0]);
154
+ const dim1 = Number(layerWeight.shape[1]);
155
+ if (!Number.isFinite(dim0) || !Number.isFinite(dim1)) {
156
+ return null;
157
+ }
158
+ if (dim1 === hiddenSize) {
159
+ return Math.trunc(dim0);
160
+ }
161
+ if (dim0 === hiddenSize) {
162
+ return Math.trunc(dim1);
163
+ }
164
+ return null;
165
+ }
166
+
167
+ export function resolveAttentionNumKVHeads(config, layerType, layerWeights, headDim) {
168
+ const kRows = resolveProjectionOutputRows(layerWeights?.kProj, config.hiddenSize);
169
+ if (kRows != null && Number.isFinite(headDim) && headDim > 0 && kRows % headDim === 0) {
170
+ return kRows / headDim;
171
+ }
172
+ if (!isSlidingLayerType(layerType) && Number.isFinite(config.numGlobalKVHeads) && config.numGlobalKVHeads > 0) {
173
+ return Math.trunc(config.numGlobalKVHeads);
174
+ }
175
+ return config.numKVHeads;
176
+ }
177
+
178
+ export function resolveAttentionKVSharing(config, layerIdx, layerType) {
143
179
  const layerTypes = Array.isArray(config?.layerTypes) ? config.layerTypes : null;
144
180
  const numKvSharedLayers = Number(config?.numKvSharedLayers ?? 0);
145
181
  if (!layerTypes || layerTypes.length === 0 || !Number.isFinite(numKvSharedLayers) || numKvSharedLayers <= 0) {
@@ -177,12 +213,12 @@ function resolveAttentionKVSharing(config, layerIdx, layerType) {
177
213
  };
178
214
  }
179
215
 
180
- function hasPerLayerInputBlock(config) {
216
+ export function hasPerLayerInputBlock(config) {
181
217
  const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
182
218
  return Number.isFinite(hiddenSizePerLayerInput) && hiddenSizePerLayerInput > 0;
183
219
  }
184
220
 
185
- function resolveLayerScalarValue(layerScalar) {
221
+ export function resolveLayerScalarValue(layerScalar) {
186
222
  if (layerScalar == null) {
187
223
  return 1;
188
224
  }
@@ -199,6 +235,16 @@ function resolveLayerScalarValue(layerScalar) {
199
235
  return value;
200
236
  }
201
237
 
238
+ export async function applyLayerScalar(layerIdx, tensor, size, context, layerWeights) {
239
+ const layerScalar = resolveLayerScalarValue(layerWeights?.layerScalar ?? null);
240
+ if (layerScalar === 1) {
241
+ return tensor;
242
+ }
243
+ return context.recorder
244
+ ? recordScale(context.recorder, tensor, layerScalar, { count: size })
245
+ : runScale(tensor, layerScalar, { count: size });
246
+ }
247
+
202
248
  async function debugLayerTensor(context, layerIdx, label, tensor, numTokens, hiddenSize) {
203
249
  if (!context.debugCheckBuffer) return;
204
250
  if (!shouldDebugLayerOutput(layerIdx, context.debugLayers)) return;
@@ -206,7 +252,7 @@ async function debugLayerTensor(context, layerIdx, label, tensor, numTokens, hid
206
252
  await context.debugCheckBuffer(tensor.buffer, `L${layerIdx} ${label} (GPU)`, numTokens, hiddenSize);
207
253
  }
208
254
 
209
- async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size, context, layerWeights) {
255
+ export async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size, context, layerWeights) {
210
256
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
211
257
  if (!hasPerLayerInputBlock(config)) {
212
258
  return hiddenTensor;
@@ -242,6 +288,16 @@ async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size,
242
288
  context,
243
289
  layerWeights
244
290
  );
291
+ // The gate weight may be stored as f32 (small projection, not always quantized),
292
+ // forcing the matmul into the f32 variant whose output dtype is fixed by the
293
+ // kernel registry, not the caller's requestedOutputDtype. doGeLU below dispatches
294
+ // gelu_f16 when input is f16 and reads `gate: array<f16>` — binding an f32 buffer
295
+ // there reinterprets bytes as 2x f16 per f32 element and produces NaN/garbage.
296
+ if (gateTensor.dtype !== hiddenTensor.dtype) {
297
+ const widened = gateTensor;
298
+ gateTensor = await doCast(widened, hiddenTensor.dtype, recorder);
299
+ releaseOrTrack(recorder, widened.buffer, decodeBuffers);
300
+ }
245
301
  await runProbes('per_layer_input_gate', gateTensor.buffer, {
246
302
  layerIdx,
247
303
  numTokens,
@@ -335,19 +391,6 @@ async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size,
335
391
  releaseOrTrack(recorder, normalizedTensor.buffer, decodeBuffers);
336
392
  normalizedTensor = null;
337
393
 
338
- const layerScalar = resolveLayerScalarValue(layerWeights.layerScalar);
339
- if (layerScalar !== 1) {
340
- const scaledOutputTensor = recorder
341
- ? await recordScale(recorder, outputTensor, layerScalar, {
342
- count: size,
343
- })
344
- : await runScale(outputTensor, layerScalar, {
345
- count: size,
346
- });
347
- releaseOrTrack(recorder, outputTensor.buffer, decodeBuffers);
348
- outputTensor = scaledOutputTensor;
349
- }
350
-
351
394
  await runProbes('post_per_layer_input', outputTensor.buffer, {
352
395
  layerIdx,
353
396
  numTokens,
@@ -569,8 +612,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
569
612
  });
570
613
  } else {
571
614
  let attentionNumHeads = numHeads;
572
- let attentionNumKVHeads = numKVHeads;
573
615
  let attentionHeadDim = resolveAttentionHeadDim(config, layerType);
616
+ let attentionNumKVHeads = resolveAttentionNumKVHeads(config, layerType, layerWeights, attentionHeadDim);
574
617
  let disableRoPE = false;
575
618
  let queryKeyNorm = config.queryKeyNorm;
576
619
  const { sharedKVSourceLayerIdx, storeSharedKV } = resolveAttentionKVSharing(config, layerIdx, layerType);
@@ -596,15 +639,18 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
596
639
  queryKeyNorm,
597
640
  valueNorm: config.valueNorm,
598
641
  attentionOutputGate: config.attentionOutputGate,
642
+ outputGateType: config.outputGateType ?? null,
599
643
  causalAttention: config.causalAttention,
600
644
  multimodalBidirectionalSpan: isSlidingLayerType(layerType)
601
645
  ? (context.multimodalBidirectionalSpan ?? null)
602
646
  : null,
603
647
  rmsNormWeightOffset: config.rmsNormWeightOffset,
604
648
  ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
649
+ ropeFrequencyBaseDim: resolveAttentionFrequencyBaseDim(config, layerType),
605
650
  ropeInterleaved: config.ropeInterleaved,
606
651
  tokenIds: context.currentTokenIds ?? null,
607
652
  kernelPath: context.kernelPath ?? null,
653
+ sessionSettings: config.sessionSettings ?? null,
608
654
  disableRoPE,
609
655
  sharedKVSourceLayerIdx,
610
656
  storeSharedKV,
@@ -779,6 +825,18 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
779
825
  outputTensor = await processFFNStandard(layerIdx, postAttn, numTokens, size, context, layerWeights, fusedResidualForFFN);
780
826
  }
781
827
 
828
+ // Keep activation dtype consistent across layers. Some FFN paths can emit f32
829
+ // tensors even when the execution plan is f16; leaving that unnormalized causes
830
+ // downstream kernels to decode the buffer with the wrong dtype contract. Apply
831
+ // the cast BEFORE the PLE block so per_layer_input_gate's matmul inherits a
832
+ // consistent outputDtype (gelu_f16 reads gate as array<f16>; binding an f32
833
+ // buffer there reinterprets bytes and produces NaN/garbage).
834
+ if (outputTensor.dtype !== activationDtype) {
835
+ const widened = outputTensor;
836
+ outputTensor = await doCast(widened, activationDtype, recorder);
837
+ releaseOrTrack(recorder, widened.buffer, context.decodeBuffers);
838
+ }
839
+
782
840
  if (hasPerLayerInputBlock(config)) {
783
841
  const outputWithPerLayerInput = await applyPerLayerInputBlock(
784
842
  layerIdx,
@@ -794,19 +852,31 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
794
852
  outputTensor = outputWithPerLayerInput;
795
853
  }
796
854
 
797
- // Keep activation dtype consistent across layers. Some FFN paths can emit f32
798
- // tensors even when the execution plan is f16; leaving that unnormalized causes
799
- // downstream kernels to decode the buffer with the wrong dtype contract.
800
- let finalOutput = outputTensor;
855
+ // Re-normalize after PLE: the residual add inside applyPerLayerInputBlock
856
+ // may emit f32 even when the layer's activation dtype is f16, which would
857
+ // misroute the next layer's f16 input bindings.
801
858
  if (outputTensor.dtype !== activationDtype) {
802
- finalOutput = await doCast(
803
- outputTensor,
804
- activationDtype,
805
- recorder
806
- );
807
- releaseOrTrack(recorder, outputTensor.buffer, context.decodeBuffers);
859
+ const widened = outputTensor;
860
+ outputTensor = await doCast(widened, activationDtype, recorder);
861
+ releaseOrTrack(recorder, widened.buffer, context.decodeBuffers);
862
+ }
863
+
864
+ let finalOutput = outputTensor;
865
+ const scaledOutput = await applyLayerScalar(layerIdx, finalOutput, size, context, layerWeights);
866
+ if (scaledOutput.buffer !== finalOutput.buffer) {
867
+ releaseOrTrack(recorder, finalOutput.buffer, context.decodeBuffers);
868
+ finalOutput = scaledOutput;
808
869
  }
809
870
  await debugLayerTensor(context, layerIdx, 'final layer output', finalOutput, numTokens, hiddenSize);
871
+ await runProbes('layer_out', finalOutput.buffer, {
872
+ layerIdx,
873
+ numTokens,
874
+ hiddenSize,
875
+ probes: context.debugProbes,
876
+ recorder,
877
+ operatorDiagnostics: context.operatorDiagnostics,
878
+ dtype: finalOutput.dtype,
879
+ });
810
880
 
811
881
  // Early-stop check for F16 NaN/Infinity bounds
812
882
  const computeConfig = context.runtimeComputeConfig ?? null;
@@ -840,514 +910,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
840
910
  }
841
911
  }
842
912
 
843
- // ============================================================================
844
- // Configurable Layer Pipeline (JSON-Driven)
845
- // ============================================================================
846
-
847
-
848
- function resolveNormWeightForPlan(weight, layerWeights) {
849
- if (!layerWeights) return null;
850
- switch (weight) {
851
- case 'input':
852
- return layerWeights.inputNorm;
853
- case 'post_attention':
854
- return layerWeights.postAttentionNorm ?? layerWeights.postAttnNorm ?? null;
855
- case 'post_attn':
856
- return layerWeights.postAttnNorm ?? layerWeights.postAttentionNorm ?? null;
857
- case 'pre_ffn':
858
- return layerWeights.preFeedforwardNorm ?? null;
859
- case 'post_ffn':
860
- return layerWeights.postFeedforwardNorm ?? null;
861
- default:
862
- return null;
863
- }
864
- }
865
-
866
-
867
- async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill, size, context, layerWeights, sandwichNorm) {
868
- const { config, weightConfig, debugFlags, kvCache, ropeFreqsCos, ropeFreqsSin, recorder } = context;
869
- const { hiddenSize, numHeads, numKVHeads, headDim, rmsNormEps } = config;
870
-
871
- if (!context.pipelinePlan) {
872
- throw new Error('Layer pipeline plan missing from context');
873
- }
874
-
875
- const planSteps = getLayerPlanSteps(context.pipelinePlan, layerIdx);
876
- const steps = filterLayerPlanStepsByPhase(planSteps, isPrefill);
877
- const device = getDevice();
878
- if (!device) throw new Error('No GPU device available');
879
-
880
- const layerType = config.layerTypes?.[layerIdx];
881
- const isLocalLayer = isSlidingLayerType(layerType);
882
- const activationDtype = resolveActivationDtype(context.activationDtype);
883
-
884
- const attnState = {
885
- ropeFreqsCos: (isLocalLayer && context.ropeLocalCos)
886
- ? (context.ropeLocalCos)
887
- : (ropeFreqsCos),
888
- ropeFreqsSin: (isLocalLayer && context.ropeLocalSin)
889
- ? (context.ropeLocalSin)
890
- : (ropeFreqsSin),
891
- sharedAttentionState: context.sharedAttentionState ?? null,
892
- kvCache: ((kvCache)),
893
- linearRuntime: context.linearAttentionRuntime ?? null,
894
- operatorDiagnostics: context.operatorDiagnostics,
895
- executionPolicies: context.executionPolicies ?? null,
896
- };
897
-
898
- const allowResidualFuse = numTokens === 1 && !(sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm);
899
-
900
-
901
- const slots = new Map();
902
- const slotDtypes = new Map();
903
-
904
- const refCounts = new Map();
905
- const protectedBuffers = new Set([inputBuffer]);
906
-
907
-
908
- const addRef = (buf) => {
909
- refCounts.set(buf, (refCounts.get(buf) ?? 0) + 1);
910
- };
911
-
912
- const releaseRef = (buf) => {
913
- const next = (refCounts.get(buf) ?? 0) - 1;
914
- if (next > 0) {
915
- refCounts.set(buf, next);
916
- return;
917
- }
918
- refCounts.delete(buf);
919
- if (protectedBuffers.has(buf)) return;
920
- if (recorder) {
921
- recorder.trackTemporaryBuffer(buf);
922
- } else {
923
- releaseBuffer(buf);
924
- }
925
- };
926
-
927
- const getSlot = (name) => {
928
- const key = name.trim() || 'state';
929
- const buf = slots.get(key);
930
- if (!buf) {
931
- throw new Error(`Layer pipeline missing slot "${key}" at L${layerIdx}`);
932
- }
933
- return buf;
934
- };
935
-
936
- const getSlotDtype = (name) => {
937
- const key = name.trim() || 'state';
938
- return slotDtypes.get(key) ?? null;
939
- };
940
-
941
- const resolveStepInputDtype = (step, slotName) => {
942
- const slotDtype = getSlotDtype(slotName) ?? resolveActivationDtype(context.activationDtype);
943
- if (!step.inputDtype) {
944
- return slotDtype;
945
- }
946
- const required = resolveActivationDtype(step.inputDtype);
947
- if (slotDtype !== required) {
948
- throw new Error(
949
- `Layer pipeline dtype mismatch at L${layerIdx} step "${step.op}": ` +
950
- `slot "${slotName}" is ${slotDtype} but step requires ${required}. ` +
951
- 'Insert an explicit cast step.'
952
- );
953
- }
954
- return required;
955
- };
956
-
957
- const resolveStepOutputDtype = (step, actualOutputDtype) => {
958
- if (!step.outputDtype) {
959
- return actualOutputDtype;
960
- }
961
- const required = resolveActivationDtype(step.outputDtype);
962
- if (actualOutputDtype !== required) {
963
- throw new Error(
964
- `Layer pipeline output dtype mismatch at L${layerIdx} step "${step.op}": ` +
965
- `kernel produced ${actualOutputDtype} but step declares ${required}.`
966
- );
967
- }
968
- return required;
969
- };
970
-
971
- const setSlot = (name, buf, dtype) => {
972
- const key = name.trim() || 'state';
973
- const prev = slots.get(key);
974
- if (prev && prev !== buf) {
975
- releaseRef(prev);
976
- }
977
- slots.set(key, buf);
978
- if (dtype) {
979
- slotDtypes.set(key, dtype);
980
- }
981
- addRef(buf);
982
- };
983
-
984
- const clearSlot = (name) => {
985
- const key = name.trim() || 'state';
986
- const prev = slots.get(key);
987
- if (!prev) return;
988
- slots.delete(key);
989
- slotDtypes.delete(key);
990
- releaseRef(prev);
991
- };
992
-
993
- setSlot('state', inputBuffer, activationDtype);
994
-
995
- const cleanupSlots = () => {
996
- for (const [name, buf] of slots) {
997
- if (name === 'state' || protectedBuffers.has(buf)) continue;
998
- const refs = refCounts.get(buf) ?? 0;
999
- if (refs > 0) {
1000
- refCounts.delete(buf);
1001
- if (recorder) {
1002
- recorder.trackTemporaryBuffer(buf);
1003
- } else {
1004
- releaseBuffer(buf);
1005
- }
1006
- }
1007
- }
1008
- };
1009
-
1010
- try {
1011
- for (const step of steps) {
1012
- switch (step.op) {
1013
- case 'save': {
1014
- const src = getSlot(step.src);
1015
- const srcDtype = getSlotDtype(step.src) ?? resolveActivationDtype(context.activationDtype);
1016
- setSlot((step.name), src, srcDtype);
1017
- break;
1018
- }
1019
- case 'load': {
1020
- const src = getSlot((step.name));
1021
- const srcDtype = getSlotDtype(step.name) ?? resolveActivationDtype(context.activationDtype);
1022
- setSlot(step.dst, src, srcDtype);
1023
- break;
1024
- }
1025
- case 'attention': {
1026
- const srcBuf = getSlot(step.src);
1027
- const residualBuf = step.residual ? getSlot(step.residual) : null;
1028
-
1029
-
1030
- const activationDtype = resolveStepInputDtype(step, step.src);
1031
- const srcTensor = createTensor(srcBuf, activationDtype, [numTokens, hiddenSize], 'plan_attn_src');
1032
- const residualTensor = (residualBuf && allowResidualFuse)
1033
- ? createTensor(residualBuf, activationDtype, [numTokens, hiddenSize], 'plan_attn_residual')
1034
- : null;
1035
-
1036
-
1037
- const attnConfig = {
1038
- layerIdx,
1039
- numTokens,
1040
- isPrefill,
1041
- numHeads,
1042
- numKVHeads,
1043
- headDim: resolveAttentionHeadDim(config, layerType),
1044
- hiddenSize,
1045
- rmsNormEps,
1046
- currentSeqLen: context.currentSeqLen,
1047
- slidingWindow: config.slidingWindow,
1048
- layerType,
1049
- residualTensor,
1050
- attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
1051
- queryPreAttnScalar: config.queryPreAttnScalar,
1052
- queryKeyNorm: config.queryKeyNorm,
1053
- valueNorm: config.valueNorm,
1054
- attentionOutputGate: config.attentionOutputGate,
1055
- causalAttention: config.causalAttention,
1056
- rmsNormWeightOffset: config.rmsNormWeightOffset,
1057
- ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
1058
- ropeInterleaved: config.ropeInterleaved,
1059
- tokenIds: context.currentTokenIds ?? null,
1060
- skipInputNorm: step.skipInputNorm === true,
1061
- activationDtype,
1062
- inputDtype: step.inputDtype ?? undefined,
1063
- outputDtype: step.outputDtype ?? undefined,
1064
- kvDtype: step.kvDtype ?? undefined,
1065
- kernelPath: context.kernelPath ?? null,
1066
- ...resolveAttentionKVSharing(config, layerIdx, layerType),
1067
- };
1068
-
1069
- const result = await doAttention(
1070
- srcTensor,
1071
- layerWeights ?? null,
1072
- attnConfig,
1073
- attnState,
1074
- context.debug,
1075
- { debugLayers: context.debugLayers },
1076
- (weight, label) => getWeightBuffer(weight, label),
1077
- (weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags),
1078
- context.debugCheckBuffer,
1079
- recorder,
1080
- context.lora
1081
- );
1082
-
1083
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(result.output.dtype));
1084
- setSlot(step.dst, result.output.buffer, outputDtype);
1085
- if (step.probeStage) {
1086
- await runProbes(step.probeStage, result.output.buffer, {
1087
- layerIdx,
1088
- numTokens,
1089
- hiddenSize,
1090
- probes: context.debugProbes,
1091
- recorder,
1092
- operatorDiagnostics: context.operatorDiagnostics,
1093
- dtype: outputDtype,
1094
- });
1095
- }
1096
- break;
1097
- }
1098
- case 'conv': {
1099
- const srcBuf = getSlot(step.src);
1100
- const inputDtype = resolveStepInputDtype(step, step.src);
1101
- const srcTensor = createTensor(srcBuf, inputDtype, [numTokens, hiddenSize], 'plan_conv_src');
1102
-
1103
- const convInProj = layerWeights?.convInProj ?? null;
1104
- const convOutProj = layerWeights?.convOutProj ?? null;
1105
- if (!convInProj || !convOutProj) {
1106
- throw new Error(
1107
- `Layer pipeline conv step missing conv weights at L${layerIdx}. ` +
1108
- 'Expected conv.in_proj.weight and conv.out_proj.weight.'
1109
- );
1110
- }
1111
- const convKernel = layerWeights?.convKernel ?? null;
1112
-
1113
- const outputTensor = await doConv(
1114
- srcTensor,
1115
- getWeightBuffer(convInProj, `L${layerIdx}.plan_conv_in_proj`),
1116
- convKernel ? getWeightBuffer(convKernel, `L${layerIdx}.plan_conv_kernel`) : null,
1117
- getWeightBuffer(convOutProj, `L${layerIdx}.plan_conv_out_proj`),
1118
- {
1119
- numTokens,
1120
- hiddenSize,
1121
- layerIdx,
1122
- label: `L${layerIdx}.plan_conv`,
1123
- swigluLimit: config.swigluLimit,
1124
- kernelPath: context.kernelPath ?? null,
1125
- executionPolicies: context.executionPolicies ?? null,
1126
- convState: getConvLayerState(context.convLayerStates, layerIdx),
1127
- },
1128
- recorder
1129
- );
1130
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
1131
- setSlot(step.dst, outputTensor.buffer, outputDtype);
1132
- if (step.probeStage) {
1133
- await runProbes(step.probeStage, outputTensor.buffer, {
1134
- layerIdx,
1135
- numTokens,
1136
- hiddenSize,
1137
- probes: context.debugProbes,
1138
- recorder,
1139
- operatorDiagnostics: context.operatorDiagnostics,
1140
- dtype: outputDtype,
1141
- });
1142
- }
1143
- break;
1144
- }
1145
- case 'rmsnorm': {
1146
- const srcBuf = getSlot(step.src);
1147
- const weight = resolveNormWeightForPlan((step.weight), layerWeights);
1148
- if (!weight) {
1149
- throw new Error(`Layer pipeline rmsnorm missing weights for "${step.weight}" at L${layerIdx}`);
1150
- }
1151
- const normWeightBuf = getNormWeightBuffer(weight, `rmsnorm_${step.weight}`, weightConfig, debugFlags);
1152
- const residualBuf = step.residual ? getSlot(step.residual) : null;
1153
-
1154
- const activationDtype = resolveStepInputDtype(step, step.src);
1155
- const srcTensor = createTensor(srcBuf, activationDtype, [numTokens, hiddenSize], 'plan_rmsnorm_src');
1156
- const residualTensor = residualBuf ? createTensor(residualBuf, activationDtype, [numTokens, hiddenSize], 'plan_rmsnorm_residual') : null;
1157
- const outputTensor = await doRMSNorm(srcTensor, normWeightBuf, rmsNormEps, {
1158
- batchSize: numTokens,
1159
- hiddenSize,
1160
- residual: residualTensor,
1161
- label: `L${layerIdx}.rmsnorm_${step.weight}`,
1162
- layerIdx,
1163
- rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
1164
- }, recorder);
1165
- if (!isGpuBufferInstance(weight) && !isWeightBuffer(weight)) releaseOrTrack(recorder, normWeightBuf);
1166
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
1167
- setSlot(step.dst, outputTensor.buffer, outputDtype);
1168
- if (step.probeStage) {
1169
- await runProbes(step.probeStage, outputTensor.buffer, {
1170
- layerIdx,
1171
- numTokens,
1172
- hiddenSize,
1173
- probes: context.debugProbes,
1174
- recorder,
1175
- operatorDiagnostics: context.operatorDiagnostics,
1176
- dtype: outputDtype,
1177
- });
1178
- }
1179
- break;
1180
- }
1181
- case 'ffn': {
1182
- const srcBuf = getSlot(step.src);
1183
-
1184
- const activationDtype = resolveStepInputDtype(step, step.src);
1185
- const srcTensor = createTensor(srcBuf, activationDtype, [numTokens, hiddenSize], 'plan_ffn_src');
1186
- const ffnContext = step.inputDtype || step.outputDtype
1187
- ? {
1188
- ...context,
1189
- ffnStepPrecision: {
1190
- inputDtype: step.inputDtype ?? null,
1191
- outputDtype: step.outputDtype ?? null,
1192
- },
1193
- }
1194
- : context;
1195
-
1196
- let outputTensor;
1197
- const { runMoEFFNGPU, runDenseFFNGPU } = await import('./ffn/index.js');
1198
-
1199
- const canAutoMoe = config.useMoE && isMoELayer(layerIdx, config);
1200
- const useMoe = selectRuleValue(
1201
- 'inference',
1202
- 'layer',
1203
- 'ffnMode',
1204
- { variant: step.variant, canAutoMoe }
1205
- );
1206
- if (useMoe) {
1207
- outputTensor = await runMoEFFNGPU(layerIdx, srcTensor, numTokens, ffnContext);
1208
- } else {
1209
- outputTensor = await runDenseFFNGPU(layerIdx, srcTensor, numTokens, ffnContext, layerWeights);
1210
- }
1211
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
1212
- setSlot(step.dst, outputTensor.buffer, outputDtype);
1213
- if (step.probeStage) {
1214
- await runProbes(step.probeStage, outputTensor.buffer, {
1215
- layerIdx,
1216
- numTokens,
1217
- hiddenSize,
1218
- probes: context.debugProbes,
1219
- recorder,
1220
- operatorDiagnostics: context.operatorDiagnostics,
1221
- dtype: outputDtype,
1222
- });
1223
- }
1224
- break;
1225
- }
1226
- case 'residual_add': {
1227
- const aBuf = getSlot(step.a ?? 'state');
1228
- const bBuf = getSlot(step.b ?? 'residual');
1229
-
1230
- const activationDtype = resolveStepInputDtype(step, step.a ?? 'state');
1231
- const aTensor = createTensor(aBuf, activationDtype, [numTokens, hiddenSize], 'plan_residual_a');
1232
- const bTensor = createTensor(bBuf, activationDtype, [numTokens, hiddenSize], 'plan_residual_b');
1233
- const outputTensor = await doResidualAdd(aTensor, bTensor, size, recorder, {
1234
- label: `L${layerIdx}.residual_add`,
1235
- layerIdx,
1236
- executionPolicies: context.executionPolicies ?? null,
1237
- });
1238
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
1239
- setSlot(step.dst, outputTensor.buffer, outputDtype);
1240
- if (step.probeStage) {
1241
- await runProbes(step.probeStage, outputTensor.buffer, {
1242
- layerIdx,
1243
- numTokens,
1244
- hiddenSize,
1245
- probes: context.debugProbes,
1246
- recorder,
1247
- operatorDiagnostics: context.operatorDiagnostics,
1248
- dtype: outputDtype,
1249
- });
1250
- }
1251
- break;
1252
- }
1253
- case 'cast': {
1254
- const srcBuf = getSlot(step.src);
1255
- const inputDtype = resolveStepInputDtype(step, step.src);
1256
- const srcTensor = createTensor(srcBuf, inputDtype, [numTokens, hiddenSize], 'plan_cast_src');
1257
- if (step.fromDtype) {
1258
- const expected = resolveActivationDtype(step.fromDtype);
1259
- if (inputDtype !== expected) {
1260
- throw new Error(
1261
- `Layer pipeline cast mismatch at L${layerIdx}: fromDtype=${expected}, actual=${inputDtype}`
1262
- );
1263
- }
1264
- }
1265
- const toDtype = resolveActivationDtype(step.toDtype);
1266
- const outputTensor = await doCast(srcTensor, toDtype, recorder);
1267
- setSlot(step.dst, outputTensor.buffer, toDtype);
1268
- if (step.probeStage) {
1269
- await runProbes(step.probeStage, outputTensor.buffer, {
1270
- layerIdx,
1271
- numTokens,
1272
- hiddenSize,
1273
- probes: context.debugProbes,
1274
- recorder,
1275
- operatorDiagnostics: context.operatorDiagnostics,
1276
- dtype: toDtype,
1277
- });
1278
- }
1279
- break;
1280
- }
1281
- case 'noop':
1282
- break;
1283
- default:
1284
- throw new Error(`Unknown layer pipeline op "${step.op}" at L${layerIdx}`);
1285
- }
1286
- }
1287
-
1288
- // Normal cleanup
1289
- for (const name of Array.from(slots.keys())) {
1290
- if (name !== 'state') {
1291
- clearSlot(name);
1292
- }
1293
- }
1294
-
1295
- if (hasPerLayerInputBlock(config)) {
1296
- const stateBuffer = getSlot('state');
1297
- const stateDtype = getSlotDtype('state') ?? activationDtype;
1298
- const stateTensor = createTensor(stateBuffer, stateDtype, [numTokens, hiddenSize], 'plan_state');
1299
- const outputTensor = await applyPerLayerInputBlock(
1300
- layerIdx,
1301
- stateTensor,
1302
- numTokens,
1303
- size,
1304
- context,
1305
- layerWeights
1306
- );
1307
- const outputDtype = resolveActivationDtype(outputTensor.dtype);
1308
- setSlot('state', outputTensor.buffer, outputDtype);
1309
- }
1310
- } catch (err) {
1311
- cleanupSlots();
1312
- throw err;
1313
- }
1314
-
1315
- const output = getSlot('state');
1316
- await runProbes('layer_out', output, {
1317
- layerIdx,
1318
- numTokens,
1319
- hiddenSize,
1320
- probes: context.debugProbes,
1321
- recorder,
1322
- operatorDiagnostics: context.operatorDiagnostics,
1323
- dtype: getSlotDtype('state') ?? activationDtype,
1324
- });
1325
-
1326
- const computeConfig = context.runtimeComputeConfig ?? null;
1327
- const shouldCheckFiniteness = context.finitenessGuardEnabled !== undefined
1328
- ? context.finitenessGuardEnabled
1329
- : shouldRunFinitenessGuard(context.activationDtype, computeConfig);
1330
- if (context.finitenessBuffer && context.activationDtype === 'f16' && shouldCheckFiniteness) {
1331
- recordCheckFiniteness(
1332
- recorder,
1333
- output,
1334
- size,
1335
- context.finitenessBuffer,
1336
- layerIdx,
1337
- context.step,
1338
- context.finitenessAbsThreshold
1339
- );
1340
- }
1341
-
1342
- return output;
1343
- }
1344
-
1345
913
  // ============================================================================
1346
914
  // CPU Fallback
1347
915
  // ============================================================================
1348
916
 
1349
917
 
1350
- export async function processLayerCPU(layerIdx, hiddenStates, numTokens, isPrefill, context) {
918
+ async function processLayerCPU(layerIdx, hiddenStates, numTokens, isPrefill, context) {
1351
919
  const { config } = context;
1352
920
  assertSupportedLayerRuntime(layerIdx, config);
1353
921
  const { hiddenSize } = config;