doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,13 +1,14 @@
1
1
 
2
2
 
3
3
  import { log, trace } from '../../../debug/index.js';
4
+ import { getRuntimeConfig } from '../../../config/runtime.js';
4
5
  import { getDevice } from '../../../gpu/device.js';
5
6
  import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
6
7
  import { allowReadback } from '../../../gpu/perf-guards.js';
7
8
  import { createTensor } from '../../../gpu/tensor.js';
8
9
  import { recordScale, runScale } from '../../../gpu/kernel-selector.js';
9
10
  import {
10
- doAttention, doRMSNorm, doResidualAdd, doMatmul, doGeLU,
11
+ doAttention, doRMSNorm, doSandwichRMSNormPair, doResidualAdd, doMatmul, doGeLU,
11
12
  doConv,
12
13
  doCast,
13
14
  releaseOrTrack
@@ -19,14 +20,15 @@ import {
19
20
  import { getWeightBuffer, getNormWeightBuffer } from './weights.js';
20
21
  import { logLayer, logAttn, getBufferStats, isKernelDebugEnabled, dumpTokenVector, logKernelStep, shouldDebugLayerOutput } from './debug-utils/index.js';
21
22
  import { runProbes } from './probes.js';
22
- import { getLayerPlanSteps, filterLayerPlanStepsByPhase } from './layer-plan.js';
23
23
  import { selectRuleValue } from '../../../rules/rule-registry.js';
24
24
  import { recordCheckFiniteness } from '../../../gpu/kernels/check-finiteness.js';
25
+ import { RMSNORM_PAIR_CACHE_LIMIT } from '../../../gpu/kernel-selector.js';
25
26
  import { shouldRunFinitenessGuard } from './finiteness-policy.js';
26
27
  import { runLinearAttentionLayer } from './linear-attention.js';
27
28
  import { validateAttnConfig } from './attention/attn-config.js';
28
29
  import { createPerLayerInputTensor, resolveDensePleProjectionWeight } from './per-layer-inputs.js';
29
30
  import { isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
31
+ import { processLayerPlanGPU } from './layer-plan-gpu.js';
30
32
 
31
33
  // ============================================================================
32
34
  // Architecture Detection
@@ -46,6 +48,122 @@ export function detectSandwichNorm(config) {
46
48
  };
47
49
  }
48
50
 
51
+ function shouldUseSandwichRMSNormPairFusion({
52
+ context,
53
+ sandwichNorm,
54
+ layerWeights,
55
+ numTokens,
56
+ hiddenSize,
57
+ attnOutput,
58
+ inputTensor,
59
+ }) {
60
+ const mergedSession = getRuntimeConfig()?.inference?.session;
61
+ if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
62
+ return false;
63
+ }
64
+ if (numTokens !== 1) {
65
+ return false;
66
+ }
67
+ if (!sandwichNorm.useSandwichNorm || !sandwichNorm.hasPostAttentionNorm || !sandwichNorm.hasPreFeedforwardNorm) {
68
+ return false;
69
+ }
70
+ if (!layerWeights?.postAttentionNorm || !layerWeights?.preFeedforwardNorm) {
71
+ return false;
72
+ }
73
+ if (attnOutput?.dtype !== 'f32' || (inputTensor && inputTensor.dtype !== 'f32')) {
74
+ throw new Error(
75
+ 'useSandwichRMSNormPairFusion requires f32 attention and residual tensors ' +
76
+ `(attn=${String(attnOutput?.dtype)}, residual=${String(inputTensor?.dtype)}).`
77
+ );
78
+ }
79
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0 || hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
80
+ throw new Error(
81
+ `useSandwichRMSNormPairFusion requires hiddenSize in 1..${RMSNORM_PAIR_CACHE_LIMIT}; got ${String(hiddenSize)}.`
82
+ );
83
+ }
84
+ return true;
85
+ }
86
+
87
+ function releasePrecomputedInputNorm(context, recorder) {
88
+ const precomputed = context.__precomputedInputNorm ?? null;
89
+ context.__precomputedInputNorm = null;
90
+ const buffer = precomputed?.tensor?.buffer ?? null;
91
+ if (buffer) {
92
+ releaseOrTrack(recorder, buffer, context.decodeBuffers);
93
+ }
94
+ }
95
+
96
+ function takePrecomputedInputNorm(context, layerIdx, recorder) {
97
+ const precomputed = context.__precomputedInputNorm ?? null;
98
+ if (!precomputed) {
99
+ return null;
100
+ }
101
+ context.__precomputedInputNorm = null;
102
+ if (precomputed.layerIdx !== layerIdx) {
103
+ const buffer = precomputed?.tensor?.buffer ?? null;
104
+ if (buffer) {
105
+ releaseOrTrack(recorder, buffer, context.decodeBuffers);
106
+ }
107
+ throw new Error(
108
+ `Layer ${layerIdx} received stale precomputed input norm for layer ${String(precomputed.layerIdx)}.`
109
+ );
110
+ }
111
+ return precomputed.tensor;
112
+ }
113
+
114
+ function shouldUsePostFfnNextInputRMSNormPairFusion({
115
+ context,
116
+ config,
117
+ sandwichNorm,
118
+ layerIdx,
119
+ layerWeights,
120
+ nextLayerWeights,
121
+ numTokens,
122
+ hiddenSize,
123
+ activationDtype,
124
+ layerScalar,
125
+ }) {
126
+ const mergedSession = getRuntimeConfig()?.inference?.session;
127
+ if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
128
+ return false;
129
+ }
130
+ if (numTokens !== 1 || context.phase !== 'decode' || context.diffusionGemmaDecoder === true) {
131
+ return false;
132
+ }
133
+ if (context.debug === true || context.debugProbes?.length || context.operatorDiagnostics?.enabled === true) {
134
+ return false;
135
+ }
136
+ if (!context.decodeBuffers || context.pipelinePlan || hasPerLayerInputBlock(config)) {
137
+ return false;
138
+ }
139
+ if (layerScalar !== 1 || activationDtype !== 'f32') {
140
+ return false;
141
+ }
142
+ const nextLayerIdx = layerIdx + 1;
143
+ if (nextLayerIdx >= config.numLayers) {
144
+ return false;
145
+ }
146
+ const nextLayerType = config.layerTypes?.[nextLayerIdx];
147
+ if (isConvLayerType(nextLayerType) || isLinearLayerType(nextLayerType)) {
148
+ return false;
149
+ }
150
+ if (!sandwichNorm.useSandwichNorm || !sandwichNorm.hasPostFeedforwardNorm) {
151
+ return false;
152
+ }
153
+ if (!layerWeights?.postFeedforwardNorm || !nextLayerWeights?.inputNorm) {
154
+ return false;
155
+ }
156
+ if (config.useMoE && isMoELayer(layerIdx, config)) {
157
+ return false;
158
+ }
159
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0 || hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
160
+ throw new Error(
161
+ `usePostFfnNextInputRMSNormPairFusion requires hiddenSize in 1..${RMSNORM_PAIR_CACHE_LIMIT}; got ${String(hiddenSize)}.`
162
+ );
163
+ }
164
+ return true;
165
+ }
166
+
49
167
 
50
168
  export function isMoELayer(layerIdx, config) {
51
169
  if (!config.useMoE) return false;
@@ -60,7 +178,7 @@ export function isMoELayer(layerIdx, config) {
60
178
  return true;
61
179
  }
62
180
 
63
- function resolveActivationDtype(dtype) {
181
+ export function resolveActivationDtype(dtype) {
64
182
  return selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype });
65
183
  }
66
184
 
@@ -88,12 +206,12 @@ function assertSupportedLayerRuntime(layerIdx, config) {
88
206
  }
89
207
  }
90
208
 
91
- function getConvLayerState(convLayerStates, layerIdx) {
209
+ export function getConvLayerState(convLayerStates, layerIdx) {
92
210
  if (!convLayerStates) return {};
93
211
  return convLayerStates.get(layerIdx) ?? {};
94
212
  }
95
213
 
96
- function isSlidingLayerType(layerType) {
214
+ export function isSlidingLayerType(layerType) {
97
215
  const normalized = normalizeLayerType(layerType);
98
216
  return normalized === 'sliding_attention'
99
217
  || normalized === 'local_attention'
@@ -125,21 +243,57 @@ function isLinearLayerType(layerType) {
125
243
  || normalized === 'gated_delta_net';
126
244
  }
127
245
 
128
- function resolveAttentionRotaryDim(config, layerType) {
246
+ export function resolveAttentionRotaryDim(config, layerType) {
129
247
  if (isSlidingLayerType(layerType)) {
130
248
  return config.ropeLocalRotaryDim ?? config.ropeRotaryDim;
131
249
  }
132
250
  return config.ropeRotaryDim;
133
251
  }
134
252
 
135
- function resolveAttentionHeadDim(config, layerType) {
253
+ export function resolveAttentionFrequencyBaseDim(config, layerType) {
254
+ if (isSlidingLayerType(layerType)) {
255
+ return config.ropeLocalFrequencyBaseDim ?? resolveAttentionRotaryDim(config, layerType);
256
+ }
257
+ return config.ropeFrequencyBaseDim ?? config.ropeRotaryDim;
258
+ }
259
+
260
+ export function resolveAttentionHeadDim(config, layerType) {
136
261
  if (isSlidingLayerType(layerType)) {
137
262
  return config.headDim;
138
263
  }
139
264
  return config.globalHeadDim ?? config.headDim;
140
265
  }
141
266
 
142
- function resolveAttentionKVSharing(config, layerIdx, layerType) {
267
+ function resolveProjectionOutputRows(layerWeight, hiddenSize) {
268
+ if (!layerWeight || !Array.isArray(layerWeight.shape) || layerWeight.shape.length < 2) {
269
+ return null;
270
+ }
271
+ const dim0 = Number(layerWeight.shape[0]);
272
+ const dim1 = Number(layerWeight.shape[1]);
273
+ if (!Number.isFinite(dim0) || !Number.isFinite(dim1)) {
274
+ return null;
275
+ }
276
+ if (dim1 === hiddenSize) {
277
+ return Math.trunc(dim0);
278
+ }
279
+ if (dim0 === hiddenSize) {
280
+ return Math.trunc(dim1);
281
+ }
282
+ return null;
283
+ }
284
+
285
+ export function resolveAttentionNumKVHeads(config, layerType, layerWeights, headDim) {
286
+ const kRows = resolveProjectionOutputRows(layerWeights?.kProj, config.hiddenSize);
287
+ if (kRows != null && Number.isFinite(headDim) && headDim > 0 && kRows % headDim === 0) {
288
+ return kRows / headDim;
289
+ }
290
+ if (!isSlidingLayerType(layerType) && Number.isFinite(config.numGlobalKVHeads) && config.numGlobalKVHeads > 0) {
291
+ return Math.trunc(config.numGlobalKVHeads);
292
+ }
293
+ return config.numKVHeads;
294
+ }
295
+
296
+ export function resolveAttentionKVSharing(config, layerIdx, layerType) {
143
297
  const layerTypes = Array.isArray(config?.layerTypes) ? config.layerTypes : null;
144
298
  const numKvSharedLayers = Number(config?.numKvSharedLayers ?? 0);
145
299
  if (!layerTypes || layerTypes.length === 0 || !Number.isFinite(numKvSharedLayers) || numKvSharedLayers <= 0) {
@@ -177,12 +331,12 @@ function resolveAttentionKVSharing(config, layerIdx, layerType) {
177
331
  };
178
332
  }
179
333
 
180
- function hasPerLayerInputBlock(config) {
334
+ export function hasPerLayerInputBlock(config) {
181
335
  const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
182
336
  return Number.isFinite(hiddenSizePerLayerInput) && hiddenSizePerLayerInput > 0;
183
337
  }
184
338
 
185
- function resolveLayerScalarValue(layerScalar) {
339
+ export function resolveLayerScalarValue(layerScalar) {
186
340
  if (layerScalar == null) {
187
341
  return 1;
188
342
  }
@@ -199,6 +353,16 @@ function resolveLayerScalarValue(layerScalar) {
199
353
  return value;
200
354
  }
201
355
 
356
+ export async function applyLayerScalar(layerIdx, tensor, size, context, layerWeights) {
357
+ const layerScalar = resolveLayerScalarValue(layerWeights?.layerScalar ?? null);
358
+ if (layerScalar === 1) {
359
+ return tensor;
360
+ }
361
+ return context.recorder
362
+ ? recordScale(context.recorder, tensor, layerScalar, { count: size })
363
+ : runScale(tensor, layerScalar, { count: size });
364
+ }
365
+
202
366
  async function debugLayerTensor(context, layerIdx, label, tensor, numTokens, hiddenSize) {
203
367
  if (!context.debugCheckBuffer) return;
204
368
  if (!shouldDebugLayerOutput(layerIdx, context.debugLayers)) return;
@@ -206,7 +370,7 @@ async function debugLayerTensor(context, layerIdx, label, tensor, numTokens, hid
206
370
  await context.debugCheckBuffer(tensor.buffer, `L${layerIdx} ${label} (GPU)`, numTokens, hiddenSize);
207
371
  }
208
372
 
209
- async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size, context, layerWeights) {
373
+ export async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size, context, layerWeights) {
210
374
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
211
375
  if (!hasPerLayerInputBlock(config)) {
212
376
  return hiddenTensor;
@@ -242,6 +406,16 @@ async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size,
242
406
  context,
243
407
  layerWeights
244
408
  );
409
+ // The gate weight may be stored as f32 (small projection, not always quantized),
410
+ // forcing the matmul into the f32 variant whose output dtype is fixed by the
411
+ // kernel registry, not the caller's requestedOutputDtype. doGeLU below dispatches
412
+ // gelu_f16 when input is f16 and reads `gate: array<f16>` — binding an f32 buffer
413
+ // there reinterprets bytes as 2x f16 per f32 element and produces NaN/garbage.
414
+ if (gateTensor.dtype !== hiddenTensor.dtype) {
415
+ const widened = gateTensor;
416
+ gateTensor = await doCast(widened, hiddenTensor.dtype, recorder);
417
+ releaseOrTrack(recorder, widened.buffer, decodeBuffers);
418
+ }
245
419
  await runProbes('per_layer_input_gate', gateTensor.buffer, {
246
420
  layerIdx,
247
421
  numTokens,
@@ -335,19 +509,6 @@ async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens, size,
335
509
  releaseOrTrack(recorder, normalizedTensor.buffer, decodeBuffers);
336
510
  normalizedTensor = null;
337
511
 
338
- const layerScalar = resolveLayerScalarValue(layerWeights.layerScalar);
339
- if (layerScalar !== 1) {
340
- const scaledOutputTensor = recorder
341
- ? await recordScale(recorder, outputTensor, layerScalar, {
342
- count: size,
343
- })
344
- : await runScale(outputTensor, layerScalar, {
345
- count: size,
346
- });
347
- releaseOrTrack(recorder, outputTensor.buffer, decodeBuffers);
348
- outputTensor = scaledOutputTensor;
349
- }
350
-
351
512
  await runProbes('post_per_layer_input', outputTensor.buffer, {
352
513
  layerIdx,
353
514
  numTokens,
@@ -461,10 +622,10 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
461
622
  // Debug entry (uses debug-utils)
462
623
  logLayer(layerIdx, 'enter', isPrefill, { numTokens });
463
624
 
464
- const device = getDevice();
625
+ const { config, weights, weightConfig, debugFlags, kvCache, ropeFreqsCos, ropeFreqsSin, recorder } = context;
626
+ const device = recorder?.device ?? getDevice();
465
627
  if (!device) throw new Error('No GPU device available');
466
628
 
467
- const { config, weights, weightConfig, debugFlags, kvCache, ropeFreqsCos, ropeFreqsSin, recorder } = context;
468
629
  assertSupportedLayerRuntime(layerIdx, config);
469
630
  const { hiddenSize, numHeads, numKVHeads, headDim, rmsNormEps } = config;
470
631
 
@@ -479,6 +640,16 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
479
640
  const sandwichNorm = detectSandwichNorm(config);
480
641
  const lastTokenIdx = Math.max(0, numTokens - 1);
481
642
 
643
+ await runProbes('layer_in', inputBuffer, {
644
+ layerIdx,
645
+ numTokens,
646
+ hiddenSize,
647
+ probes: context.debugProbes,
648
+ recorder,
649
+ operatorDiagnostics: context.operatorDiagnostics,
650
+ dtype: inputTensor.dtype,
651
+ });
652
+
482
653
  if (context.pipelinePlan) {
483
654
  return processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill, size, context, layerWeights, sandwichNorm);
484
655
  }
@@ -569,10 +740,14 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
569
740
  });
570
741
  } else {
571
742
  let attentionNumHeads = numHeads;
572
- let attentionNumKVHeads = numKVHeads;
573
743
  let attentionHeadDim = resolveAttentionHeadDim(config, layerType);
744
+ let attentionNumKVHeads = resolveAttentionNumKVHeads(config, layerType, layerWeights, attentionHeadDim);
574
745
  let disableRoPE = false;
575
- let queryKeyNorm = config.queryKeyNorm;
746
+ let queryKeyNorm = config.queryKeyNorm === true;
747
+ const diffusionGemmaDecoder = context.diffusionGemmaDecoder === true;
748
+ if (queryKeyNorm && Array.isArray(config.queryKeyNormLayers)) {
749
+ queryKeyNorm = config.queryKeyNormLayers.includes(layerIdx);
750
+ }
576
751
  const { sharedKVSourceLayerIdx, storeSharedKV } = resolveAttentionKVSharing(config, layerIdx, layerType);
577
752
 
578
753
  const attnConfig = {
@@ -594,23 +769,29 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
594
769
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
595
770
  queryPreAttnScalar: config.queryPreAttnScalar,
596
771
  queryKeyNorm,
772
+ queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
597
773
  valueNorm: config.valueNorm,
598
774
  attentionOutputGate: config.attentionOutputGate,
599
- causalAttention: config.causalAttention,
600
- multimodalBidirectionalSpan: isSlidingLayerType(layerType)
775
+ outputGateType: config.outputGateType ?? null,
776
+ causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
777
+ multimodalBidirectionalSpan: !diffusionGemmaDecoder && isSlidingLayerType(layerType)
601
778
  ? (context.multimodalBidirectionalSpan ?? null)
602
779
  : null,
603
780
  rmsNormWeightOffset: config.rmsNormWeightOffset,
604
781
  ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
782
+ ropeFrequencyBaseDim: resolveAttentionFrequencyBaseDim(config, layerType),
605
783
  ropeInterleaved: config.ropeInterleaved,
606
784
  tokenIds: context.currentTokenIds ?? null,
607
785
  kernelPath: context.kernelPath ?? null,
786
+ sessionSettings: config.sessionSettings ?? null,
608
787
  disableRoPE,
609
788
  sharedKVSourceLayerIdx,
610
789
  storeSharedKV,
790
+ diffusionGemmaDecoder,
611
791
  };
612
792
 
613
793
  validateAttnConfig(attnConfig, `L${layerIdx}`);
794
+ attnConfig.precomputedInputNorm = takePrecomputedInputNorm(context, layerIdx, recorder);
614
795
 
615
796
  const attnState = {
616
797
  ropeFreqsCos: (isLocalLayer && context.ropeLocalCos)
@@ -692,9 +873,37 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
692
873
 
693
874
  // 2. Handle residual connection based on architecture
694
875
 
876
+ let precomputedFfnInput = null;
695
877
  if (residualFused) {
696
878
  postAttn = attnOutput;
697
- if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
879
+ if (shouldUseSandwichRMSNormPairFusion({
880
+ context,
881
+ sandwichNorm,
882
+ layerWeights,
883
+ numTokens,
884
+ hiddenSize,
885
+ attnOutput,
886
+ inputTensor: null,
887
+ })) {
888
+ const postNormWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
889
+ const preNormWeightBuf = getNormWeightBuffer(layerWeights.preFeedforwardNorm, 'pre_feedforward_norm', weightConfig, debugFlags);
890
+ const pair = await doSandwichRMSNormPair(attnOutput, null, postNormWeightBuf, preNormWeightBuf, rmsNormEps, {
891
+ batchSize: numTokens,
892
+ hiddenSize,
893
+ label: `L${layerIdx}.post_attn_pre_ffn_norm`,
894
+ layerIdx,
895
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
896
+ }, recorder);
897
+ postAttn = pair.postAttn;
898
+ precomputedFfnInput = pair.ffnInput;
899
+ if (!isGpuBufferInstance(layerWeights.postAttentionNorm) && !isWeightBuffer(layerWeights.postAttentionNorm)) releaseOrTrack(recorder, postNormWeightBuf);
900
+ if (!isGpuBufferInstance(layerWeights.preFeedforwardNorm) && !isWeightBuffer(layerWeights.preFeedforwardNorm)) releaseOrTrack(recorder, preNormWeightBuf);
901
+ if (recorder) {
902
+ recorder.trackTemporaryBuffer(attnOutput.buffer);
903
+ } else {
904
+ releaseBuffer(attnOutput.buffer);
905
+ }
906
+ } else if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
698
907
  const normWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
699
908
  postAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
700
909
  batchSize: numTokens,
@@ -712,21 +921,52 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
712
921
  }
713
922
  } else if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
714
923
  const normWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
715
- const normalizedAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
716
- batchSize: numTokens,
924
+ if (shouldUseSandwichRMSNormPairFusion({
925
+ context,
926
+ sandwichNorm,
927
+ layerWeights,
928
+ numTokens,
717
929
  hiddenSize,
718
- label: `L${layerIdx}.post_attn_norm`,
719
- layerIdx,
720
- rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
721
- }, recorder);
722
- postAttn = await doResidualAdd(normalizedAttn, inputTensor, size, recorder, {
723
- label: `L${layerIdx}.post_attn_residual`,
724
- layerIdx,
725
- executionPolicies: context.executionPolicies ?? null,
726
- });
930
+ attnOutput,
931
+ inputTensor,
932
+ })) {
933
+ const preNormWeightBuf = getNormWeightBuffer(layerWeights.preFeedforwardNorm, 'pre_feedforward_norm', weightConfig, debugFlags);
934
+ const pair = await doSandwichRMSNormPair(attnOutput, inputTensor, normWeightBuf, preNormWeightBuf, rmsNormEps, {
935
+ batchSize: numTokens,
936
+ hiddenSize,
937
+ label: `L${layerIdx}.post_attn_pre_ffn_norm`,
938
+ layerIdx,
939
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
940
+ }, recorder);
941
+ postAttn = pair.postAttn;
942
+ precomputedFfnInput = pair.ffnInput;
943
+ if (!isGpuBufferInstance(layerWeights.preFeedforwardNorm) && !isWeightBuffer(layerWeights.preFeedforwardNorm)) releaseOrTrack(recorder, preNormWeightBuf);
944
+ } else if (attnOutput.dtype === inputTensor.dtype) {
945
+ postAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
946
+ batchSize: numTokens,
947
+ hiddenSize,
948
+ residual: inputTensor,
949
+ label: `L${layerIdx}.post_attn_norm`,
950
+ layerIdx,
951
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
952
+ }, recorder);
953
+ } else {
954
+ const normalizedAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
955
+ batchSize: numTokens,
956
+ hiddenSize,
957
+ label: `L${layerIdx}.post_attn_norm`,
958
+ layerIdx,
959
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
960
+ }, recorder);
961
+ postAttn = await doResidualAdd(normalizedAttn, inputTensor, size, recorder, {
962
+ label: `L${layerIdx}.post_attn_residual`,
963
+ layerIdx,
964
+ executionPolicies: context.executionPolicies ?? null,
965
+ });
966
+ releaseOrTrack(recorder, normalizedAttn.buffer, context.decodeBuffers);
967
+ }
727
968
 
728
969
  if (!isGpuBufferInstance(layerWeights.postAttentionNorm) && !isWeightBuffer(layerWeights.postAttentionNorm)) releaseOrTrack(recorder, normWeightBuf);
729
- releaseOrTrack(recorder, normalizedAttn.buffer, context.decodeBuffers);
730
970
  if (recorder) {
731
971
  recorder.trackTemporaryBuffer(attnOutput.buffer);
732
972
  } else {
@@ -773,10 +1013,72 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
773
1013
  // 3. Feed-forward network
774
1014
 
775
1015
  let outputTensor;
1016
+ const layerScalar = resolveLayerScalarValue(layerWeights?.layerScalar ?? null);
1017
+ const requestFfnLayerScalarFusion = layerScalar !== 1
1018
+ && !hasPerLayerInputBlock(config);
1019
+ const nextLayerIdx = layerIdx + 1;
1020
+ const nextLayerWeights = nextLayerIdx < config.numLayers
1021
+ ? weights.get(`layer_${nextLayerIdx}`)
1022
+ : null;
1023
+ const usePostFfnNextInputNormPair = shouldUsePostFfnNextInputRMSNormPairFusion({
1024
+ context,
1025
+ config,
1026
+ sandwichNorm,
1027
+ layerIdx,
1028
+ layerWeights,
1029
+ nextLayerWeights,
1030
+ numTokens,
1031
+ hiddenSize,
1032
+ activationDtype,
1033
+ layerScalar,
1034
+ });
1035
+ let layerScalarFused = false;
776
1036
  if (sandwichNorm.useSandwichNorm) {
777
- outputTensor = await processFFNWithSandwichNorm(layerIdx, postAttn, numTokens, size, context, layerWeights, sandwichNorm);
1037
+ context.__postFfnNextInputNorm = usePostFfnNextInputNormPair
1038
+ ? { layerIdx: nextLayerIdx, weight: nextLayerWeights.inputNorm }
1039
+ : null;
1040
+ try {
1041
+ outputTensor = await processFFNWithSandwichNorm(
1042
+ layerIdx,
1043
+ postAttn,
1044
+ numTokens,
1045
+ size,
1046
+ context,
1047
+ layerWeights,
1048
+ sandwichNorm,
1049
+ requestFfnLayerScalarFusion ? layerScalar : 1,
1050
+ precomputedFfnInput
1051
+ );
1052
+ } finally {
1053
+ context.__postFfnNextInputNorm = null;
1054
+ }
1055
+ layerScalarFused = context.__layerScalarFusedFired === true;
1056
+ context.__layerScalarFusedFired = false;
778
1057
  } else {
779
- outputTensor = await processFFNStandard(layerIdx, postAttn, numTokens, size, context, layerWeights, fusedResidualForFFN);
1058
+ outputTensor = await processFFNStandard(
1059
+ layerIdx,
1060
+ postAttn,
1061
+ numTokens,
1062
+ size,
1063
+ context,
1064
+ layerWeights,
1065
+ fusedResidualForFFN,
1066
+ requestFfnLayerScalarFusion ? layerScalar : 1
1067
+ );
1068
+ layerScalarFused = context.__layerScalarFusedFired === true;
1069
+ context.__layerScalarFusedFired = false;
1070
+ }
1071
+
1072
+ // Keep activation dtype consistent across layers. Some FFN paths can emit f32
1073
+ // tensors even when the execution plan is f16; leaving that unnormalized causes
1074
+ // downstream kernels to decode the buffer with the wrong dtype contract. Apply
1075
+ // the cast BEFORE the PLE block so per_layer_input_gate's matmul inherits a
1076
+ // consistent outputDtype (gelu_f16 reads gate as array<f16>; binding an f32
1077
+ // buffer there reinterprets bytes and produces NaN/garbage).
1078
+ if (outputTensor.dtype !== activationDtype) {
1079
+ const widened = outputTensor;
1080
+ outputTensor = await doCast(widened, activationDtype, recorder);
1081
+ releaseOrTrack(recorder, widened.buffer, context.decodeBuffers);
780
1082
  }
781
1083
 
782
1084
  if (hasPerLayerInputBlock(config)) {
@@ -794,19 +1096,33 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
794
1096
  outputTensor = outputWithPerLayerInput;
795
1097
  }
796
1098
 
797
- // Keep activation dtype consistent across layers. Some FFN paths can emit f32
798
- // tensors even when the execution plan is f16; leaving that unnormalized causes
799
- // downstream kernels to decode the buffer with the wrong dtype contract.
800
- let finalOutput = outputTensor;
1099
+ // Re-normalize after PLE: the residual add inside applyPerLayerInputBlock
1100
+ // may emit f32 even when the layer's activation dtype is f16, which would
1101
+ // misroute the next layer's f16 input bindings.
801
1102
  if (outputTensor.dtype !== activationDtype) {
802
- finalOutput = await doCast(
803
- outputTensor,
804
- activationDtype,
805
- recorder
806
- );
807
- releaseOrTrack(recorder, outputTensor.buffer, context.decodeBuffers);
1103
+ const widened = outputTensor;
1104
+ outputTensor = await doCast(widened, activationDtype, recorder);
1105
+ releaseOrTrack(recorder, widened.buffer, context.decodeBuffers);
1106
+ }
1107
+
1108
+ let finalOutput = outputTensor;
1109
+ if (!layerScalarFused) {
1110
+ const scaledOutput = await applyLayerScalar(layerIdx, finalOutput, size, context, layerWeights);
1111
+ if (scaledOutput.buffer !== finalOutput.buffer) {
1112
+ releaseOrTrack(recorder, finalOutput.buffer, context.decodeBuffers);
1113
+ finalOutput = scaledOutput;
1114
+ }
808
1115
  }
809
1116
  await debugLayerTensor(context, layerIdx, 'final layer output', finalOutput, numTokens, hiddenSize);
1117
+ await runProbes('layer_out', finalOutput.buffer, {
1118
+ layerIdx,
1119
+ numTokens,
1120
+ hiddenSize,
1121
+ probes: context.debugProbes,
1122
+ recorder,
1123
+ operatorDiagnostics: context.operatorDiagnostics,
1124
+ dtype: finalOutput.dtype,
1125
+ });
810
1126
 
811
1127
  // Early-stop check for F16 NaN/Infinity bounds
812
1128
  const computeConfig = context.runtimeComputeConfig ?? null;
@@ -834,520 +1150,19 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
834
1150
  released.add(buf);
835
1151
  releaseOrTrack(recorder, buf);
836
1152
  };
1153
+ releasePrecomputedInputNorm(context, recorder);
837
1154
  if (postAttn?.buffer) releaseOnce(postAttn.buffer);
838
1155
  if (attnOutput?.buffer && attnOutput.buffer !== postAttn?.buffer) releaseOnce(attnOutput.buffer);
839
1156
  throw error;
840
1157
  }
841
1158
  }
842
1159
 
843
- // ============================================================================
844
- // Configurable Layer Pipeline (JSON-Driven)
845
- // ============================================================================
846
-
847
-
848
- function resolveNormWeightForPlan(weight, layerWeights) {
849
- if (!layerWeights) return null;
850
- switch (weight) {
851
- case 'input':
852
- return layerWeights.inputNorm;
853
- case 'post_attention':
854
- return layerWeights.postAttentionNorm ?? layerWeights.postAttnNorm ?? null;
855
- case 'post_attn':
856
- return layerWeights.postAttnNorm ?? layerWeights.postAttentionNorm ?? null;
857
- case 'pre_ffn':
858
- return layerWeights.preFeedforwardNorm ?? null;
859
- case 'post_ffn':
860
- return layerWeights.postFeedforwardNorm ?? null;
861
- default:
862
- return null;
863
- }
864
- }
865
-
866
-
867
- async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill, size, context, layerWeights, sandwichNorm) {
868
- const { config, weightConfig, debugFlags, kvCache, ropeFreqsCos, ropeFreqsSin, recorder } = context;
869
- const { hiddenSize, numHeads, numKVHeads, headDim, rmsNormEps } = config;
870
-
871
- if (!context.pipelinePlan) {
872
- throw new Error('Layer pipeline plan missing from context');
873
- }
874
-
875
- const planSteps = getLayerPlanSteps(context.pipelinePlan, layerIdx);
876
- const steps = filterLayerPlanStepsByPhase(planSteps, isPrefill);
877
- const device = getDevice();
878
- if (!device) throw new Error('No GPU device available');
879
-
880
- const layerType = config.layerTypes?.[layerIdx];
881
- const isLocalLayer = isSlidingLayerType(layerType);
882
- const activationDtype = resolveActivationDtype(context.activationDtype);
883
-
884
- const attnState = {
885
- ropeFreqsCos: (isLocalLayer && context.ropeLocalCos)
886
- ? (context.ropeLocalCos)
887
- : (ropeFreqsCos),
888
- ropeFreqsSin: (isLocalLayer && context.ropeLocalSin)
889
- ? (context.ropeLocalSin)
890
- : (ropeFreqsSin),
891
- sharedAttentionState: context.sharedAttentionState ?? null,
892
- kvCache: ((kvCache)),
893
- linearRuntime: context.linearAttentionRuntime ?? null,
894
- operatorDiagnostics: context.operatorDiagnostics,
895
- executionPolicies: context.executionPolicies ?? null,
896
- };
897
-
898
- const allowResidualFuse = numTokens === 1 && !(sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm);
899
-
900
-
901
- const slots = new Map();
902
- const slotDtypes = new Map();
903
-
904
- const refCounts = new Map();
905
- const protectedBuffers = new Set([inputBuffer]);
906
-
907
-
908
- const addRef = (buf) => {
909
- refCounts.set(buf, (refCounts.get(buf) ?? 0) + 1);
910
- };
911
-
912
- const releaseRef = (buf) => {
913
- const next = (refCounts.get(buf) ?? 0) - 1;
914
- if (next > 0) {
915
- refCounts.set(buf, next);
916
- return;
917
- }
918
- refCounts.delete(buf);
919
- if (protectedBuffers.has(buf)) return;
920
- if (recorder) {
921
- recorder.trackTemporaryBuffer(buf);
922
- } else {
923
- releaseBuffer(buf);
924
- }
925
- };
926
-
927
- const getSlot = (name) => {
928
- const key = name.trim() || 'state';
929
- const buf = slots.get(key);
930
- if (!buf) {
931
- throw new Error(`Layer pipeline missing slot "${key}" at L${layerIdx}`);
932
- }
933
- return buf;
934
- };
935
-
936
- const getSlotDtype = (name) => {
937
- const key = name.trim() || 'state';
938
- return slotDtypes.get(key) ?? null;
939
- };
940
-
941
- const resolveStepInputDtype = (step, slotName) => {
942
- const slotDtype = getSlotDtype(slotName) ?? resolveActivationDtype(context.activationDtype);
943
- if (!step.inputDtype) {
944
- return slotDtype;
945
- }
946
- const required = resolveActivationDtype(step.inputDtype);
947
- if (slotDtype !== required) {
948
- throw new Error(
949
- `Layer pipeline dtype mismatch at L${layerIdx} step "${step.op}": ` +
950
- `slot "${slotName}" is ${slotDtype} but step requires ${required}. ` +
951
- 'Insert an explicit cast step.'
952
- );
953
- }
954
- return required;
955
- };
956
-
957
- const resolveStepOutputDtype = (step, actualOutputDtype) => {
958
- if (!step.outputDtype) {
959
- return actualOutputDtype;
960
- }
961
- const required = resolveActivationDtype(step.outputDtype);
962
- if (actualOutputDtype !== required) {
963
- throw new Error(
964
- `Layer pipeline output dtype mismatch at L${layerIdx} step "${step.op}": ` +
965
- `kernel produced ${actualOutputDtype} but step declares ${required}.`
966
- );
967
- }
968
- return required;
969
- };
970
-
971
- const setSlot = (name, buf, dtype) => {
972
- const key = name.trim() || 'state';
973
- const prev = slots.get(key);
974
- if (prev && prev !== buf) {
975
- releaseRef(prev);
976
- }
977
- slots.set(key, buf);
978
- if (dtype) {
979
- slotDtypes.set(key, dtype);
980
- }
981
- addRef(buf);
982
- };
983
-
984
- const clearSlot = (name) => {
985
- const key = name.trim() || 'state';
986
- const prev = slots.get(key);
987
- if (!prev) return;
988
- slots.delete(key);
989
- slotDtypes.delete(key);
990
- releaseRef(prev);
991
- };
992
-
993
- setSlot('state', inputBuffer, activationDtype);
994
-
995
- const cleanupSlots = () => {
996
- for (const [name, buf] of slots) {
997
- if (name === 'state' || protectedBuffers.has(buf)) continue;
998
- const refs = refCounts.get(buf) ?? 0;
999
- if (refs > 0) {
1000
- refCounts.delete(buf);
1001
- if (recorder) {
1002
- recorder.trackTemporaryBuffer(buf);
1003
- } else {
1004
- releaseBuffer(buf);
1005
- }
1006
- }
1007
- }
1008
- };
1009
-
1010
- try {
1011
- for (const step of steps) {
1012
- switch (step.op) {
1013
- case 'save': {
1014
- const src = getSlot(step.src);
1015
- const srcDtype = getSlotDtype(step.src) ?? resolveActivationDtype(context.activationDtype);
1016
- setSlot((step.name), src, srcDtype);
1017
- break;
1018
- }
1019
- case 'load': {
1020
- const src = getSlot((step.name));
1021
- const srcDtype = getSlotDtype(step.name) ?? resolveActivationDtype(context.activationDtype);
1022
- setSlot(step.dst, src, srcDtype);
1023
- break;
1024
- }
1025
- case 'attention': {
1026
- const srcBuf = getSlot(step.src);
1027
- const residualBuf = step.residual ? getSlot(step.residual) : null;
1028
-
1029
-
1030
- const activationDtype = resolveStepInputDtype(step, step.src);
1031
- const srcTensor = createTensor(srcBuf, activationDtype, [numTokens, hiddenSize], 'plan_attn_src');
1032
- const residualTensor = (residualBuf && allowResidualFuse)
1033
- ? createTensor(residualBuf, activationDtype, [numTokens, hiddenSize], 'plan_attn_residual')
1034
- : null;
1035
-
1036
-
1037
- const attnConfig = {
1038
- layerIdx,
1039
- numTokens,
1040
- isPrefill,
1041
- numHeads,
1042
- numKVHeads,
1043
- headDim: resolveAttentionHeadDim(config, layerType),
1044
- hiddenSize,
1045
- rmsNormEps,
1046
- currentSeqLen: context.currentSeqLen,
1047
- slidingWindow: config.slidingWindow,
1048
- layerType,
1049
- residualTensor,
1050
- attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
1051
- queryPreAttnScalar: config.queryPreAttnScalar,
1052
- queryKeyNorm: config.queryKeyNorm,
1053
- valueNorm: config.valueNorm,
1054
- attentionOutputGate: config.attentionOutputGate,
1055
- causalAttention: config.causalAttention,
1056
- rmsNormWeightOffset: config.rmsNormWeightOffset,
1057
- ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
1058
- ropeInterleaved: config.ropeInterleaved,
1059
- tokenIds: context.currentTokenIds ?? null,
1060
- skipInputNorm: step.skipInputNorm === true,
1061
- activationDtype,
1062
- inputDtype: step.inputDtype ?? undefined,
1063
- outputDtype: step.outputDtype ?? undefined,
1064
- kvDtype: step.kvDtype ?? undefined,
1065
- kernelPath: context.kernelPath ?? null,
1066
- ...resolveAttentionKVSharing(config, layerIdx, layerType),
1067
- };
1068
-
1069
- const result = await doAttention(
1070
- srcTensor,
1071
- layerWeights ?? null,
1072
- attnConfig,
1073
- attnState,
1074
- context.debug,
1075
- { debugLayers: context.debugLayers },
1076
- (weight, label) => getWeightBuffer(weight, label),
1077
- (weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags),
1078
- context.debugCheckBuffer,
1079
- recorder,
1080
- context.lora
1081
- );
1082
-
1083
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(result.output.dtype));
1084
- setSlot(step.dst, result.output.buffer, outputDtype);
1085
- if (step.probeStage) {
1086
- await runProbes(step.probeStage, result.output.buffer, {
1087
- layerIdx,
1088
- numTokens,
1089
- hiddenSize,
1090
- probes: context.debugProbes,
1091
- recorder,
1092
- operatorDiagnostics: context.operatorDiagnostics,
1093
- dtype: outputDtype,
1094
- });
1095
- }
1096
- break;
1097
- }
1098
- case 'conv': {
1099
- const srcBuf = getSlot(step.src);
1100
- const inputDtype = resolveStepInputDtype(step, step.src);
1101
- const srcTensor = createTensor(srcBuf, inputDtype, [numTokens, hiddenSize], 'plan_conv_src');
1102
-
1103
- const convInProj = layerWeights?.convInProj ?? null;
1104
- const convOutProj = layerWeights?.convOutProj ?? null;
1105
- if (!convInProj || !convOutProj) {
1106
- throw new Error(
1107
- `Layer pipeline conv step missing conv weights at L${layerIdx}. ` +
1108
- 'Expected conv.in_proj.weight and conv.out_proj.weight.'
1109
- );
1110
- }
1111
- const convKernel = layerWeights?.convKernel ?? null;
1112
-
1113
- const outputTensor = await doConv(
1114
- srcTensor,
1115
- getWeightBuffer(convInProj, `L${layerIdx}.plan_conv_in_proj`),
1116
- convKernel ? getWeightBuffer(convKernel, `L${layerIdx}.plan_conv_kernel`) : null,
1117
- getWeightBuffer(convOutProj, `L${layerIdx}.plan_conv_out_proj`),
1118
- {
1119
- numTokens,
1120
- hiddenSize,
1121
- layerIdx,
1122
- label: `L${layerIdx}.plan_conv`,
1123
- swigluLimit: config.swigluLimit,
1124
- kernelPath: context.kernelPath ?? null,
1125
- executionPolicies: context.executionPolicies ?? null,
1126
- convState: getConvLayerState(context.convLayerStates, layerIdx),
1127
- },
1128
- recorder
1129
- );
1130
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
1131
- setSlot(step.dst, outputTensor.buffer, outputDtype);
1132
- if (step.probeStage) {
1133
- await runProbes(step.probeStage, outputTensor.buffer, {
1134
- layerIdx,
1135
- numTokens,
1136
- hiddenSize,
1137
- probes: context.debugProbes,
1138
- recorder,
1139
- operatorDiagnostics: context.operatorDiagnostics,
1140
- dtype: outputDtype,
1141
- });
1142
- }
1143
- break;
1144
- }
1145
- case 'rmsnorm': {
1146
- const srcBuf = getSlot(step.src);
1147
- const weight = resolveNormWeightForPlan((step.weight), layerWeights);
1148
- if (!weight) {
1149
- throw new Error(`Layer pipeline rmsnorm missing weights for "${step.weight}" at L${layerIdx}`);
1150
- }
1151
- const normWeightBuf = getNormWeightBuffer(weight, `rmsnorm_${step.weight}`, weightConfig, debugFlags);
1152
- const residualBuf = step.residual ? getSlot(step.residual) : null;
1153
-
1154
- const activationDtype = resolveStepInputDtype(step, step.src);
1155
- const srcTensor = createTensor(srcBuf, activationDtype, [numTokens, hiddenSize], 'plan_rmsnorm_src');
1156
- const residualTensor = residualBuf ? createTensor(residualBuf, activationDtype, [numTokens, hiddenSize], 'plan_rmsnorm_residual') : null;
1157
- const outputTensor = await doRMSNorm(srcTensor, normWeightBuf, rmsNormEps, {
1158
- batchSize: numTokens,
1159
- hiddenSize,
1160
- residual: residualTensor,
1161
- label: `L${layerIdx}.rmsnorm_${step.weight}`,
1162
- layerIdx,
1163
- rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
1164
- }, recorder);
1165
- if (!isGpuBufferInstance(weight) && !isWeightBuffer(weight)) releaseOrTrack(recorder, normWeightBuf);
1166
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
1167
- setSlot(step.dst, outputTensor.buffer, outputDtype);
1168
- if (step.probeStage) {
1169
- await runProbes(step.probeStage, outputTensor.buffer, {
1170
- layerIdx,
1171
- numTokens,
1172
- hiddenSize,
1173
- probes: context.debugProbes,
1174
- recorder,
1175
- operatorDiagnostics: context.operatorDiagnostics,
1176
- dtype: outputDtype,
1177
- });
1178
- }
1179
- break;
1180
- }
1181
- case 'ffn': {
1182
- const srcBuf = getSlot(step.src);
1183
-
1184
- const activationDtype = resolveStepInputDtype(step, step.src);
1185
- const srcTensor = createTensor(srcBuf, activationDtype, [numTokens, hiddenSize], 'plan_ffn_src');
1186
- const ffnContext = step.inputDtype || step.outputDtype
1187
- ? {
1188
- ...context,
1189
- ffnStepPrecision: {
1190
- inputDtype: step.inputDtype ?? null,
1191
- outputDtype: step.outputDtype ?? null,
1192
- },
1193
- }
1194
- : context;
1195
-
1196
- let outputTensor;
1197
- const { runMoEFFNGPU, runDenseFFNGPU } = await import('./ffn/index.js');
1198
-
1199
- const canAutoMoe = config.useMoE && isMoELayer(layerIdx, config);
1200
- const useMoe = selectRuleValue(
1201
- 'inference',
1202
- 'layer',
1203
- 'ffnMode',
1204
- { variant: step.variant, canAutoMoe }
1205
- );
1206
- if (useMoe) {
1207
- outputTensor = await runMoEFFNGPU(layerIdx, srcTensor, numTokens, ffnContext);
1208
- } else {
1209
- outputTensor = await runDenseFFNGPU(layerIdx, srcTensor, numTokens, ffnContext, layerWeights);
1210
- }
1211
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
1212
- setSlot(step.dst, outputTensor.buffer, outputDtype);
1213
- if (step.probeStage) {
1214
- await runProbes(step.probeStage, outputTensor.buffer, {
1215
- layerIdx,
1216
- numTokens,
1217
- hiddenSize,
1218
- probes: context.debugProbes,
1219
- recorder,
1220
- operatorDiagnostics: context.operatorDiagnostics,
1221
- dtype: outputDtype,
1222
- });
1223
- }
1224
- break;
1225
- }
1226
- case 'residual_add': {
1227
- const aBuf = getSlot(step.a ?? 'state');
1228
- const bBuf = getSlot(step.b ?? 'residual');
1229
-
1230
- const activationDtype = resolveStepInputDtype(step, step.a ?? 'state');
1231
- const aTensor = createTensor(aBuf, activationDtype, [numTokens, hiddenSize], 'plan_residual_a');
1232
- const bTensor = createTensor(bBuf, activationDtype, [numTokens, hiddenSize], 'plan_residual_b');
1233
- const outputTensor = await doResidualAdd(aTensor, bTensor, size, recorder, {
1234
- label: `L${layerIdx}.residual_add`,
1235
- layerIdx,
1236
- executionPolicies: context.executionPolicies ?? null,
1237
- });
1238
- const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
1239
- setSlot(step.dst, outputTensor.buffer, outputDtype);
1240
- if (step.probeStage) {
1241
- await runProbes(step.probeStage, outputTensor.buffer, {
1242
- layerIdx,
1243
- numTokens,
1244
- hiddenSize,
1245
- probes: context.debugProbes,
1246
- recorder,
1247
- operatorDiagnostics: context.operatorDiagnostics,
1248
- dtype: outputDtype,
1249
- });
1250
- }
1251
- break;
1252
- }
1253
- case 'cast': {
1254
- const srcBuf = getSlot(step.src);
1255
- const inputDtype = resolveStepInputDtype(step, step.src);
1256
- const srcTensor = createTensor(srcBuf, inputDtype, [numTokens, hiddenSize], 'plan_cast_src');
1257
- if (step.fromDtype) {
1258
- const expected = resolveActivationDtype(step.fromDtype);
1259
- if (inputDtype !== expected) {
1260
- throw new Error(
1261
- `Layer pipeline cast mismatch at L${layerIdx}: fromDtype=${expected}, actual=${inputDtype}`
1262
- );
1263
- }
1264
- }
1265
- const toDtype = resolveActivationDtype(step.toDtype);
1266
- const outputTensor = await doCast(srcTensor, toDtype, recorder);
1267
- setSlot(step.dst, outputTensor.buffer, toDtype);
1268
- if (step.probeStage) {
1269
- await runProbes(step.probeStage, outputTensor.buffer, {
1270
- layerIdx,
1271
- numTokens,
1272
- hiddenSize,
1273
- probes: context.debugProbes,
1274
- recorder,
1275
- operatorDiagnostics: context.operatorDiagnostics,
1276
- dtype: toDtype,
1277
- });
1278
- }
1279
- break;
1280
- }
1281
- case 'noop':
1282
- break;
1283
- default:
1284
- throw new Error(`Unknown layer pipeline op "${step.op}" at L${layerIdx}`);
1285
- }
1286
- }
1287
-
1288
- // Normal cleanup
1289
- for (const name of Array.from(slots.keys())) {
1290
- if (name !== 'state') {
1291
- clearSlot(name);
1292
- }
1293
- }
1294
-
1295
- if (hasPerLayerInputBlock(config)) {
1296
- const stateBuffer = getSlot('state');
1297
- const stateDtype = getSlotDtype('state') ?? activationDtype;
1298
- const stateTensor = createTensor(stateBuffer, stateDtype, [numTokens, hiddenSize], 'plan_state');
1299
- const outputTensor = await applyPerLayerInputBlock(
1300
- layerIdx,
1301
- stateTensor,
1302
- numTokens,
1303
- size,
1304
- context,
1305
- layerWeights
1306
- );
1307
- const outputDtype = resolveActivationDtype(outputTensor.dtype);
1308
- setSlot('state', outputTensor.buffer, outputDtype);
1309
- }
1310
- } catch (err) {
1311
- cleanupSlots();
1312
- throw err;
1313
- }
1314
-
1315
- const output = getSlot('state');
1316
- await runProbes('layer_out', output, {
1317
- layerIdx,
1318
- numTokens,
1319
- hiddenSize,
1320
- probes: context.debugProbes,
1321
- recorder,
1322
- operatorDiagnostics: context.operatorDiagnostics,
1323
- dtype: getSlotDtype('state') ?? activationDtype,
1324
- });
1325
-
1326
- const computeConfig = context.runtimeComputeConfig ?? null;
1327
- const shouldCheckFiniteness = context.finitenessGuardEnabled !== undefined
1328
- ? context.finitenessGuardEnabled
1329
- : shouldRunFinitenessGuard(context.activationDtype, computeConfig);
1330
- if (context.finitenessBuffer && context.activationDtype === 'f16' && shouldCheckFiniteness) {
1331
- recordCheckFiniteness(
1332
- recorder,
1333
- output,
1334
- size,
1335
- context.finitenessBuffer,
1336
- layerIdx,
1337
- context.step,
1338
- context.finitenessAbsThreshold
1339
- );
1340
- }
1341
-
1342
- return output;
1343
- }
1344
-
1345
1160
  // ============================================================================
1346
1161
  // CPU Fallback
1347
1162
  // ============================================================================
1348
1163
 
1349
1164
 
1350
- export async function processLayerCPU(layerIdx, hiddenStates, numTokens, isPrefill, context) {
1165
+ async function processLayerCPU(layerIdx, hiddenStates, numTokens, isPrefill, context) {
1351
1166
  const { config } = context;
1352
1167
  assertSupportedLayerRuntime(layerIdx, config);
1353
1168
  const { hiddenSize } = config;