doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -1,16 +1,18 @@
1
1
 
2
2
 
3
- import { getDevice, getKernelCapabilities } from '../device.js';
3
+ import { getDevice, getDeviceEpoch, getKernelCapabilities } from '../device.js';
4
4
  import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
5
5
  import { createTensor } from '../tensor.js';
6
6
  import { KernelBase } from './kernel-base.js';
7
7
  import { FFN_DISPATCH } from './constants.js';
8
8
  import { createUniformBufferWithView, getPipelineBindGroupLayout } from './utils.js';
9
+ import { recordDispatch } from './dispatch.js';
9
10
  import { trace, isTraceEnabled } from '../../debug/index.js';
10
11
  import { getBuffer, getWeightDtype } from '../weight-buffer.js';
11
12
  import { isFusedQ4KDisabled } from './matmul.js';
12
13
  import { getKernelThresholds, QK_K } from '../../config/schema/index.js';
13
14
  import { selectRuleValue } from './rule-registry.js';
15
+ import { resolveNormWeightDtype } from './rmsnorm.js';
14
16
 
15
17
  class FusedFFNKernel extends KernelBase {
16
18
 
@@ -32,6 +34,7 @@ class FusedFFNKernel extends KernelBase {
32
34
  const SHARED_INPUT_SIZE_VARIANTS = new Set([
33
35
  'default',
34
36
  'batched',
37
+ 'f16_batched',
35
38
  'f16',
36
39
  'multi',
37
40
  'f16_native',
@@ -50,7 +53,15 @@ const F16_OUTPUT_VARIANTS = new Set([
50
53
  'f16_native_batched',
51
54
  ]);
52
55
 
53
- function selectFFNVariant(batchSize, weightDtype, intermediateSize, hiddenSize, inputDtype) {
56
+ const Q4K_VARIANTS = new Set([
57
+ 'q4k',
58
+ 'q4k_metal_simd16',
59
+ 'q4k_batched',
60
+ 'q4k_f16a',
61
+ 'q4k_batched_f16a',
62
+ ]);
63
+
64
+ function selectFFNVariant(batchSize, weightDtype, intermediateSize, hiddenSize, inputDtype, variantOverride = null) {
54
65
  const { multiOutputThreshold } = getKernelThresholds().ffn;
55
66
  const capabilities = getKernelCapabilities();
56
67
  const isQ4K = weightDtype === 'q4k';
@@ -60,7 +71,7 @@ function selectFFNVariant(batchSize, weightDtype, intermediateSize, hiddenSize,
60
71
  const hasF16 = capabilities.hasF16;
61
72
  const useF16Input = inputDtype === 'f16';
62
73
 
63
- return selectRuleValue(
74
+ const selected = selectRuleValue(
64
75
  'fusedFfn',
65
76
  'variant',
66
77
  {
@@ -74,6 +85,19 @@ function selectFFNVariant(batchSize, weightDtype, intermediateSize, hiddenSize,
74
85
  useF16Input,
75
86
  }
76
87
  );
88
+ if (variantOverride == null) {
89
+ return selected;
90
+ }
91
+ if (variantOverride !== 'q4k_metal_simd16') {
92
+ throw new Error(`Fused FFN does not support explicit variant "${String(variantOverride)}".`);
93
+ }
94
+ if (selected !== 'q4k') {
95
+ throw new Error(
96
+ `Fused FFN variant "${variantOverride}" requires single-token Q4_K decode with f32 activations; ` +
97
+ `the resolved base variant is "${selected}".`
98
+ );
99
+ }
100
+ return variantOverride;
77
101
  }
78
102
 
79
103
 
@@ -99,6 +123,277 @@ function createFFNUniformBuffer(device, recorder, params) {
99
123
  );
100
124
  }
101
125
 
126
+ function createNormedFFNUniformBuffer(device, recorder, params) {
127
+ const swigluLimit = resolveSwigluLimit(params.swigluLimit, 'FusedNormedFFN uniforms');
128
+ return createUniformBufferWithView(
129
+ 'fused_normed_ffn_uniforms',
130
+ 48,
131
+ (view) => {
132
+ view.setUint32(0, params.M, true);
133
+ view.setUint32(4, params.hiddenSize, true);
134
+ view.setUint32(8, params.intermediateSize, true);
135
+ view.setFloat32(12, params.alpha, true);
136
+ view.setUint32(16, params.activation === 'silu' ? 0 : 1, true);
137
+ view.setUint32(20, Math.ceil(params.hiddenSize / QK_K), true);
138
+ view.setFloat32(24, swigluLimit, true);
139
+ view.setUint32(28, params.rmsNormWeightOffset ? 1 : 0, true);
140
+ view.setUint32(32, params.normWeightDtype === 'f16' ? 1 : 0, true);
141
+ },
142
+ recorder,
143
+ device
144
+ );
145
+ }
146
+
147
+ function buildFusedNormedFFNQ4KShader() {
148
+ return /* wgsl */ `const QK_K: u32 = 256u;
149
+ const BLOCK_SIZE: u32 = 144u;
150
+ const SUBBLOCK_SIZE: u32 = 32u;
151
+ const MAX_WORKGROUP_SIZE: u32 = 256u;
152
+ override WORKGROUP_SIZE: u32 = 256u;
153
+ override COLS_PER_WG: u32 = 32u;
154
+ override THREADS_PER_COL: u32 = 8u;
155
+ override USE_FULL_BLOCK_FAST_PATH: bool = false;
156
+
157
+ struct Uniforms {
158
+ M: u32,
159
+ hidden_size: u32,
160
+ intermediate_size: u32,
161
+ alpha: f32,
162
+ activation: u32,
163
+ num_blocks_per_row: u32,
164
+ clamp_max: f32,
165
+ rms_norm_weight_offset: u32,
166
+ norm_weight_is_f16: u32,
167
+ _pad0: u32,
168
+ _pad1: u32,
169
+ _pad2: u32,
170
+ }
171
+
172
+ struct Q4KBlock {
173
+ d_dmin: u32,
174
+ scales: array<u32, 3>,
175
+ qs: array<u32, 32>,
176
+ }
177
+
178
+ @group(0) @binding(0) var<uniform> u: Uniforms;
179
+ @group(0) @binding(1) var<storage, read> prenorm_input: array<f32>;
180
+ @group(0) @binding(2) var<storage, read> norm_weight: array<u32>;
181
+ @group(0) @binding(3) var<storage, read> inv_rms: array<f32>;
182
+ @group(0) @binding(4) var<storage, read> W_gate_q4k: array<Q4KBlock>;
183
+ @group(0) @binding(5) var<storage, read> W_up_q4k: array<Q4KBlock>;
184
+ @group(0) @binding(6) var<storage, read_write> output: array<f32>;
185
+
186
+ var<workgroup> multicol_gate: array<f32, MAX_WORKGROUP_SIZE>;
187
+ var<workgroup> multicol_up: array<f32, MAX_WORKGROUP_SIZE>;
188
+
189
+ fn unpack_f16_lo(packed: u32) -> f32 {
190
+ return unpack2x16float(packed).x;
191
+ }
192
+
193
+ fn unpack_f16_hi(packed: u32) -> f32 {
194
+ return unpack2x16float(packed).y;
195
+ }
196
+
197
+ fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
198
+ let word_idx = byte_idx / 4u;
199
+ let byte_in_word = byte_idx % 4u;
200
+ return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
201
+ }
202
+
203
+ fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
204
+ var sc: u32;
205
+ var mn: u32;
206
+ if (j < 4u) {
207
+ sc = get_scale_byte(scales, j) & 63u;
208
+ mn = get_scale_byte(scales, j + 4u) & 63u;
209
+ } else {
210
+ let q_j = get_scale_byte(scales, j + 4u);
211
+ let q_lo = get_scale_byte(scales, j - 4u);
212
+ let q_hi = get_scale_byte(scales, j);
213
+ sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
214
+ mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
215
+ }
216
+ return vec2<u32>(sc, mn);
217
+ }
218
+
219
+ fn unpack_q4_word(word: u32, nibble_shift: u32) -> vec4<u32> {
220
+ return vec4<u32>(
221
+ (word >> nibble_shift) & 0xFu,
222
+ (word >> (8u + nibble_shift)) & 0xFu,
223
+ (word >> (16u + nibble_shift)) & 0xFu,
224
+ (word >> (24u + nibble_shift)) & 0xFu
225
+ );
226
+ }
227
+
228
+ fn load_norm_weight(idx: u32) -> f32 {
229
+ var w: f32;
230
+ if (u.norm_weight_is_f16 != 0u) {
231
+ let packed = norm_weight[idx >> 1u];
232
+ let pair = unpack2x16float(packed);
233
+ w = select(pair.x, pair.y, (idx & 1u) == 1u);
234
+ } else {
235
+ w = bitcast<f32>(norm_weight[idx]);
236
+ }
237
+ if (u.rms_norm_weight_offset != 0u) {
238
+ return 1.0 + w;
239
+ }
240
+ return w;
241
+ }
242
+
243
+ fn load_normed_input(row: u32, idx: u32) -> f32 {
244
+ let flat_idx = row * u.hidden_size + idx;
245
+ return prenorm_input[flat_idx] * inv_rms[row] * load_norm_weight(idx);
246
+ }
247
+
248
+ fn clamp_swiglu(x: f32) -> f32 {
249
+ if (u.clamp_max <= 0.0 || u.activation != 0u) {
250
+ return x;
251
+ }
252
+ return clamp(x, -u.clamp_max, u.clamp_max);
253
+ }
254
+
255
+ fn silu(x: f32) -> f32 {
256
+ if (x >= 0.0) {
257
+ let z = exp(-x);
258
+ return x / (1.0 + z);
259
+ }
260
+ let z = exp(x);
261
+ return x * z / (1.0 + z);
262
+ }
263
+
264
+ fn gelu(x: f32) -> f32 {
265
+ let c = 0.7978845608;
266
+ let inner = c * (x + 0.044715 * x * x * x);
267
+ return 0.5 * x * (1.0 + tanh(clamp(inner, -15.0, 15.0)));
268
+ }
269
+
270
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
271
+ fn main(@builtin(local_invocation_id) lid: vec3<u32>,
272
+ @builtin(workgroup_id) wg_id: vec3<u32>) {
273
+ if (WORKGROUP_SIZE > MAX_WORKGROUP_SIZE || COLS_PER_WG * THREADS_PER_COL != WORKGROUP_SIZE) {
274
+ return;
275
+ }
276
+ let local_id = lid.x;
277
+ let col_in_wg = local_id / THREADS_PER_COL;
278
+ let tid_in_col = local_id % THREADS_PER_COL;
279
+ let col = wg_id.x * COLS_PER_WG + col_in_wg;
280
+ let row = wg_id.y;
281
+ let is_valid = col < u.intermediate_size && row < u.M;
282
+ var partial_gate = 0.0;
283
+ var partial_up = 0.0;
284
+
285
+ if (is_valid) {
286
+ let num_blocks = u.num_blocks_per_row;
287
+ for (var b = tid_in_col; b < num_blocks; b = b + THREADS_PER_COL) {
288
+ let gate_block = W_gate_q4k[col * num_blocks + b];
289
+ let up_block = W_up_q4k[col * num_blocks + b];
290
+ let gate_d = unpack_f16_lo(gate_block.d_dmin);
291
+ let gate_dmin = unpack_f16_hi(gate_block.d_dmin);
292
+ let up_d = unpack_f16_lo(up_block.d_dmin);
293
+ let up_dmin = unpack_f16_hi(up_block.d_dmin);
294
+ let k_base = b * QK_K;
295
+ for (var sb = 0u; sb < 8u; sb = sb + 1u) {
296
+ let sb_base = sb * SUBBLOCK_SIZE;
297
+ if (!USE_FULL_BLOCK_FAST_PATH && k_base + sb_base >= u.hidden_size) { break; }
298
+ let gate_sm = get_scale_min_k4(gate_block.scales, sb);
299
+ let gate_scale = gate_d * f32(gate_sm.x);
300
+ let gate_min = gate_dmin * f32(gate_sm.y);
301
+ let up_sm = get_scale_min_k4(up_block.scales, sb);
302
+ let up_scale = up_d * f32(up_sm.x);
303
+ let up_min = up_dmin * f32(up_sm.y);
304
+ let chunk = sb >> 1u;
305
+ let nibble_shift = (sb & 1u) * 4u;
306
+ let word_base = chunk * 8u;
307
+ for (var i = 0u; i < SUBBLOCK_SIZE; i = i + 4u) {
308
+ let k0 = k_base + sb_base + i;
309
+ let k1 = k0 + 1u;
310
+ let k2 = k0 + 2u;
311
+ let k3 = k0 + 3u;
312
+ let a0 = load_normed_input(row, k0);
313
+ let a1 = load_normed_input(row, k1);
314
+ let a2 = load_normed_input(row, k2);
315
+ let a3 = load_normed_input(row, k3);
316
+ let word_idx = word_base + (i >> 2u);
317
+ let gate_q = unpack_q4_word(gate_block.qs[word_idx], nibble_shift);
318
+ let up_q = unpack_q4_word(up_block.qs[word_idx], nibble_shift);
319
+ let gw0 = gate_scale * f32(gate_q.x) - gate_min;
320
+ let gw1 = gate_scale * f32(gate_q.y) - gate_min;
321
+ let gw2 = gate_scale * f32(gate_q.z) - gate_min;
322
+ let gw3 = gate_scale * f32(gate_q.w) - gate_min;
323
+ let uw0 = up_scale * f32(up_q.x) - up_min;
324
+ let uw1 = up_scale * f32(up_q.y) - up_min;
325
+ let uw2 = up_scale * f32(up_q.z) - up_min;
326
+ let uw3 = up_scale * f32(up_q.w) - up_min;
327
+ partial_gate = partial_gate + a0 * gw0 + a1 * gw1 + a2 * gw2 + a3 * gw3;
328
+ partial_up = partial_up + a0 * uw0 + a1 * uw1 + a2 * uw2 + a3 * uw3;
329
+ }
330
+ }
331
+ }
332
+ }
333
+
334
+ multicol_gate[local_id] = partial_gate;
335
+ multicol_up[local_id] = partial_up;
336
+ workgroupBarrier();
337
+
338
+ if (tid_in_col == 0u && is_valid) {
339
+ var final_gate = 0.0;
340
+ var final_up = 0.0;
341
+ let base = col_in_wg * THREADS_PER_COL;
342
+ for (var i = 0u; i < THREADS_PER_COL; i = i + 1u) {
343
+ final_gate = final_gate + multicol_gate[base + i];
344
+ final_up = final_up + multicol_up[base + i];
345
+ }
346
+ var activated: f32;
347
+ if (u.activation == 0u) {
348
+ activated = silu(final_gate);
349
+ } else {
350
+ activated = gelu(final_gate);
351
+ }
352
+ output[row * u.intermediate_size + col] = clamp_swiglu(activated * final_up * u.alpha);
353
+ }
354
+ }
355
+ `;
356
+ }
357
+
358
+ let fusedNormedFFNEpoch = -1;
359
+ let fusedNormedFFNPipelines = new Map();
360
+
361
+ function pipelineConstantsKey(constants) {
362
+ if (!constants) return 'default';
363
+ return Object.keys(constants)
364
+ .sort()
365
+ .map((key) => `${key}=${String(constants[key])}`)
366
+ .join(';');
367
+ }
368
+
369
+ function getFusedNormedFFNPipeline(device, constants = null) {
370
+ const epoch = getDeviceEpoch();
371
+ if (fusedNormedFFNEpoch !== epoch) {
372
+ fusedNormedFFNEpoch = epoch;
373
+ fusedNormedFFNPipelines = new Map();
374
+ }
375
+ const pipelineKey = pipelineConstantsKey(constants);
376
+ const cached = fusedNormedFFNPipelines.get(pipelineKey);
377
+ if (cached) return cached;
378
+ const module = device.createShaderModule({
379
+ label: 'fused_normed_ffn_q4k',
380
+ code: buildFusedNormedFFNQ4KShader(),
381
+ });
382
+ const pipeline = device.createComputePipeline({
383
+ label: pipelineKey === 'default'
384
+ ? 'fused_normed_ffn_q4k_pipeline'
385
+ : `fused_normed_ffn_q4k_pipeline_${pipelineKey}`,
386
+ layout: 'auto',
387
+ compute: {
388
+ module,
389
+ entryPoint: 'main',
390
+ ...(constants ? { constants } : {}),
391
+ },
392
+ });
393
+ fusedNormedFFNPipelines.set(pipelineKey, pipeline);
394
+ return pipeline;
395
+ }
396
+
102
397
  function resolveSwigluLimit(value, context) {
103
398
  if (value === undefined) {
104
399
  throw new Error(`${context} requires an explicit swigluLimit (null or number).`);
@@ -107,21 +402,67 @@ function resolveSwigluLimit(value, context) {
107
402
  return value;
108
403
  }
109
404
 
110
- function calculateFFNDispatch(variant, batchSize, intermediateSize) {
405
+ function assertPositiveIntegerConstant(value, name, context) {
406
+ if (!Number.isInteger(value) || value <= 0) {
407
+ throw new Error(`${context} requires ${name} to be a positive integer; got ${String(value)}.`);
408
+ }
409
+ }
410
+
411
+ function normalizeQ4KPipelineConstants(constants, context) {
412
+ if (constants == null) return null;
413
+ if (typeof constants !== 'object') {
414
+ throw new Error(`${context} pipelineConstants must be an object or null.`);
415
+ }
416
+ const colsPerWorkgroup = constants.COLS_PER_WG;
417
+ const threadsPerCol = constants.THREADS_PER_COL ?? constants.THREADS_PER_COL_GEMV;
418
+ const workgroupSize = constants.WORKGROUP_SIZE ?? 256;
419
+ const useFullBlockFastPath = constants.USE_FULL_BLOCK_FAST_PATH;
420
+ const hasWorkgroupConstants = colsPerWorkgroup !== undefined ||
421
+ threadsPerCol !== undefined ||
422
+ constants.WORKGROUP_SIZE !== undefined;
423
+ if (!hasWorkgroupConstants && useFullBlockFastPath === undefined) {
424
+ return null;
425
+ }
426
+ if (useFullBlockFastPath !== undefined && typeof useFullBlockFastPath !== 'boolean') {
427
+ throw new Error(`${context} requires USE_FULL_BLOCK_FAST_PATH to be a boolean; got ${String(useFullBlockFastPath)}.`);
428
+ }
429
+ const normalized = {};
430
+ if (hasWorkgroupConstants) {
431
+ assertPositiveIntegerConstant(colsPerWorkgroup, 'COLS_PER_WG', context);
432
+ assertPositiveIntegerConstant(threadsPerCol, 'THREADS_PER_COL', context);
433
+ assertPositiveIntegerConstant(workgroupSize, 'WORKGROUP_SIZE', context);
434
+ if (colsPerWorkgroup * threadsPerCol !== workgroupSize) {
435
+ throw new Error(
436
+ `${context} requires COLS_PER_WG * THREADS_PER_COL to equal WORKGROUP_SIZE; ` +
437
+ `got ${colsPerWorkgroup} * ${threadsPerCol} != ${workgroupSize}.`
438
+ );
439
+ }
440
+ if (constants.WORKGROUP_SIZE !== undefined) {
441
+ normalized.WORKGROUP_SIZE = workgroupSize;
442
+ }
443
+ normalized.COLS_PER_WG = colsPerWorkgroup;
444
+ normalized.THREADS_PER_COL = threadsPerCol;
445
+ }
446
+ if (useFullBlockFastPath !== undefined) {
447
+ normalized.USE_FULL_BLOCK_FAST_PATH = useFullBlockFastPath ? 1 : 0;
448
+ }
449
+ return normalized;
450
+ }
451
+
452
+ function q4kColsPerWorkgroup(constants) {
453
+ return constants?.COLS_PER_WG ?? FFN_DISPATCH.Q4K_COLS_PER_WG;
454
+ }
455
+
456
+ function calculateFFNDispatch(variant, batchSize, intermediateSize, constants = null) {
111
457
  let workgroupsX;
112
458
  let workgroupsY = 1;
113
459
 
114
460
  if (variant === 'multi') {
115
461
  workgroupsX = Math.ceil(intermediateSize / FFN_DISPATCH.MULTI_OUTPUTS_PER_WG);
116
- } else if (
117
- variant === 'q4k'
118
- || variant === 'q4k_batched'
119
- || variant === 'q4k_f16a'
120
- || variant === 'q4k_batched_f16a'
121
- ) {
122
- workgroupsX = Math.ceil(intermediateSize / FFN_DISPATCH.Q4K_COLS_PER_WG);
462
+ } else if (Q4K_VARIANTS.has(variant)) {
463
+ workgroupsX = Math.ceil(intermediateSize / q4kColsPerWorkgroup(constants));
123
464
  workgroupsY = (variant === 'q4k_batched' || variant === 'q4k_batched_f16a') ? batchSize : 1;
124
- } else if (variant === 'batched' || variant === 'f16_native_batched') {
465
+ } else if (variant === 'batched' || variant === 'f16_batched' || variant === 'f16_native_batched') {
125
466
  workgroupsX = intermediateSize;
126
467
  workgroupsY = batchSize;
127
468
  } else {
@@ -131,7 +472,10 @@ function calculateFFNDispatch(variant, batchSize, intermediateSize) {
131
472
  return { workgroupsX, workgroupsY };
132
473
  }
133
474
 
134
- function resolveFusedFFNPipelineConstants(variant, hiddenSize) {
475
+ function resolveFusedFFNPipelineConstants(variant, hiddenSize, pipelineConstants = null) {
476
+ if (Q4K_VARIANTS.has(variant)) {
477
+ return normalizeQ4KPipelineConstants(pipelineConstants, 'FusedFFN Q4K');
478
+ }
135
479
  if (!SHARED_INPUT_SIZE_VARIANTS.has(variant)) {
136
480
  return null;
137
481
  }
@@ -168,6 +512,8 @@ export async function runFusedFFN(
168
512
  activation = 'silu',
169
513
  alpha = 1.0,
170
514
  outputBuffer = null,
515
+ pipelineConstants = null,
516
+ variant: variantOverride = null,
171
517
  swigluLimit,
172
518
  } = options;
173
519
  resolveSwigluLimit(swigluLimit, 'FusedFFN');
@@ -185,7 +531,14 @@ export async function runFusedFFN(
185
531
  }
186
532
 
187
533
  const isQ4K = gateDtype === 'q4k';
188
- const variant = selectFFNVariant(batchSize, gateDtype, intermediateSize, hiddenSize, input.dtype);
534
+ const variant = selectFFNVariant(
535
+ batchSize,
536
+ gateDtype,
537
+ intermediateSize,
538
+ hiddenSize,
539
+ input.dtype,
540
+ variantOverride
541
+ );
189
542
  const requiresF16Input = F16_INPUT_VARIANTS.has(variant);
190
543
  const usesF16Output = F16_OUTPUT_VARIANTS.has(variant);
191
544
 
@@ -200,7 +553,7 @@ export async function runFusedFFN(
200
553
  trace.kernels(`FusedFFN: variant=${variant}, batch=${batchSize}, hidden=${hiddenSize}, intermediate=${intermediateSize}, activation=${activation}, isQ4K=${isQ4K}`);
201
554
 
202
555
  const kernel = new FusedFFNKernel(device);
203
- const constants = resolveFusedFFNPipelineConstants(variant, hiddenSize);
556
+ const constants = resolveFusedFFNPipelineConstants(variant, hiddenSize, pipelineConstants);
204
557
  const pipeline = await kernel.getPipeline(variant, constants);
205
558
 
206
559
  // Native f16 weight kernels narrow back to f16 output. Q4K f16-activation
@@ -236,7 +589,7 @@ export async function runFusedFFN(
236
589
  ],
237
590
  });
238
591
 
239
- const { workgroupsX, workgroupsY } = calculateFFNDispatch(variant, batchSize, intermediateSize);
592
+ const { workgroupsX, workgroupsY } = calculateFFNDispatch(variant, batchSize, intermediateSize, constants);
240
593
  kernel.dispatch(pipeline, bindGroup, workgroupsX, workgroupsY);
241
594
  } catch (error) {
242
595
  releaseRunResources(uniformBuffer, [ownedOutput]);
@@ -264,6 +617,8 @@ export async function recordFusedFFN(
264
617
  activation = 'silu',
265
618
  alpha = 1.0,
266
619
  outputBuffer = null,
620
+ pipelineConstants = null,
621
+ variant: variantOverride = null,
267
622
  swigluLimit,
268
623
  } = options;
269
624
  resolveSwigluLimit(swigluLimit, 'FusedFFN');
@@ -281,7 +636,14 @@ export async function recordFusedFFN(
281
636
  }
282
637
 
283
638
  const isQ4K = gateDtype === 'q4k';
284
- const variant = selectFFNVariant(batchSize, gateDtype, intermediateSize, hiddenSize, input.dtype);
639
+ const variant = selectFFNVariant(
640
+ batchSize,
641
+ gateDtype,
642
+ intermediateSize,
643
+ hiddenSize,
644
+ input.dtype,
645
+ variantOverride
646
+ );
285
647
  const requiresF16Input = F16_INPUT_VARIANTS.has(variant);
286
648
  const usesF16Output = F16_OUTPUT_VARIANTS.has(variant);
287
649
 
@@ -296,7 +658,7 @@ export async function recordFusedFFN(
296
658
  trace.kernels(`FusedFFN record: variant=${variant}, batch=${batchSize}, hidden=${hiddenSize}, intermediate=${intermediateSize}, activation=${activation}, isQ4K=${isQ4K}`);
297
659
 
298
660
  const kernel = new FusedFFNKernel(device);
299
- const constants = resolveFusedFFNPipelineConstants(variant, hiddenSize);
661
+ const constants = resolveFusedFFNPipelineConstants(variant, hiddenSize, pipelineConstants);
300
662
  const pipeline = await kernel.getPipeline(variant, constants);
301
663
 
302
664
  const outputBytesPerElement = usesF16Output ? 2 : 4;
@@ -328,7 +690,7 @@ export async function recordFusedFFN(
328
690
  ],
329
691
  });
330
692
 
331
- const { workgroupsX, workgroupsY } = calculateFFNDispatch(variant, batchSize, intermediateSize);
693
+ const { workgroupsX, workgroupsY } = calculateFFNDispatch(variant, batchSize, intermediateSize, constants);
332
694
  kernel.record(recorder, pipeline, bindGroup, workgroupsX, workgroupsY);
333
695
  } catch (error) {
334
696
  if (ownedOutput) {
@@ -340,6 +702,165 @@ export async function recordFusedFFN(
340
702
  return createTensor(output, outputDtype, [batchSize, intermediateSize], 'fused_ffn_output');
341
703
  }
342
704
 
705
+ function validateFusedNormedFFNInputs(input, invRmsBuffer, normWeight, W_gate, W_up, hiddenSize, intermediateSize, options) {
706
+ const batchSize = options.batchSize ?? 1;
707
+ if (!Number.isInteger(batchSize) || batchSize <= 0) {
708
+ throw new Error(`Fused normed FFN batchSize must be a positive integer; got ${String(batchSize)}.`);
709
+ }
710
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0) {
711
+ throw new Error(`Fused normed FFN hiddenSize must be a positive integer; got ${String(hiddenSize)}.`);
712
+ }
713
+ if (!Number.isInteger(intermediateSize) || intermediateSize <= 0) {
714
+ throw new Error(`Fused normed FFN intermediateSize must be a positive integer; got ${String(intermediateSize)}.`);
715
+ }
716
+ if (input?.dtype !== 'f32') {
717
+ throw new Error(`Fused normed FFN requires f32 prenorm input; got ${String(input?.dtype)}.`);
718
+ }
719
+ if (!invRmsBuffer) {
720
+ throw new Error('Fused normed FFN requires an inverse-RMS buffer.');
721
+ }
722
+ if (!normWeight) {
723
+ throw new Error('Fused normed FFN requires a norm weight buffer.');
724
+ }
725
+ const gateDtype = getWeightDtype(W_gate);
726
+ const upDtype = getWeightDtype(W_up);
727
+ if (gateDtype !== 'q4k' || upDtype !== 'q4k') {
728
+ throw new Error(`Fused normed FFN requires Q4_K gate/up weights; got gate=${gateDtype}, up=${upDtype}.`);
729
+ }
730
+ if (hiddenSize % 32 !== 0) {
731
+ throw new Error(`Fused normed FFN requires hiddenSize aligned to 32; got ${hiddenSize}.`);
732
+ }
733
+ if (FFN_DISPATCH.Q4K_COLS_PER_WG <= 0 || 256 % FFN_DISPATCH.Q4K_COLS_PER_WG !== 0) {
734
+ throw new Error(`Fused normed FFN requires Q4K_COLS_PER_WG to divide 256; got ${FFN_DISPATCH.Q4K_COLS_PER_WG}.`);
735
+ }
736
+ const activation = options.activation ?? 'silu';
737
+ if (activation !== 'silu' && activation !== 'gelu') {
738
+ throw new Error(`Fused normed FFN activation must be silu|gelu; got ${String(activation)}.`);
739
+ }
740
+ return {
741
+ batchSize,
742
+ activation,
743
+ normWeightDtype: resolveNormWeightDtype(normWeight, hiddenSize),
744
+ };
745
+ }
746
+
747
+ function createFusedNormedBindGroup(device, pipeline, uniformBuffer, input, normWeight, invRmsBuffer, W_gate, W_up, output) {
748
+ return device.createBindGroup({
749
+ label: 'fused_normed_ffn_q4k_bind_group',
750
+ layout: getPipelineBindGroupLayout(pipeline, 0),
751
+ entries: [
752
+ { binding: 0, resource: { buffer: uniformBuffer } },
753
+ { binding: 1, resource: { buffer: input.buffer } },
754
+ { binding: 2, resource: { buffer: getBuffer(normWeight) } },
755
+ { binding: 3, resource: { buffer: invRmsBuffer } },
756
+ { binding: 4, resource: { buffer: getBuffer(W_gate) } },
757
+ { binding: 5, resource: { buffer: getBuffer(W_up) } },
758
+ { binding: 6, resource: { buffer: output } },
759
+ ],
760
+ });
761
+ }
762
+
763
+ function calculateFusedNormedDispatch(batchSize, intermediateSize, constants = null) {
764
+ return {
765
+ workgroupsX: Math.ceil(intermediateSize / q4kColsPerWorkgroup(constants)),
766
+ workgroupsY: batchSize,
767
+ };
768
+ }
769
+
770
+ export async function runFusedFFNFromRMSNormStats(
771
+ input,
772
+ invRmsBuffer,
773
+ normWeight,
774
+ W_gate,
775
+ W_up,
776
+ hiddenSize,
777
+ intermediateSize,
778
+ options = {}
779
+ ) {
780
+ const device = getDevice();
781
+ const {
782
+ batchSize,
783
+ activation,
784
+ normWeightDtype,
785
+ } = validateFusedNormedFFNInputs(input, invRmsBuffer, normWeight, W_gate, W_up, hiddenSize, intermediateSize, options);
786
+ resolveSwigluLimit(options.swigluLimit, 'FusedNormedFFN');
787
+ const constants = normalizeQ4KPipelineConstants(options.pipelineConstants, 'FusedNormedFFN Q4K');
788
+ const outputSize = batchSize * intermediateSize * 4;
789
+ const ownedOutput = options.outputBuffer ? null : acquireBuffer(outputSize, undefined, 'fused_normed_ffn_output');
790
+ const output = options.outputBuffer || ownedOutput;
791
+ const uniformBuffer = createNormedFFNUniformBuffer(device, null, {
792
+ M: batchSize,
793
+ hiddenSize,
794
+ intermediateSize,
795
+ alpha: options.alpha ?? 1.0,
796
+ activation,
797
+ swigluLimit: activation === 'silu' ? options.swigluLimit : null,
798
+ rmsNormWeightOffset: options.rmsNormWeightOffset === true,
799
+ normWeightDtype,
800
+ });
801
+ try {
802
+ const pipeline = getFusedNormedFFNPipeline(device, constants);
803
+ const bindGroup = createFusedNormedBindGroup(device, pipeline, uniformBuffer, input, normWeight, invRmsBuffer, W_gate, W_up, output);
804
+ const { workgroupsX, workgroupsY } = calculateFusedNormedDispatch(batchSize, intermediateSize, constants);
805
+ const encoder = device.createCommandEncoder({ label: 'fused_normed_ffn_q4k_encoder' });
806
+ const pass = encoder.beginComputePass({ label: 'fused_normed_ffn_q4k' });
807
+ pass.setPipeline(pipeline);
808
+ pass.setBindGroup(0, bindGroup);
809
+ pass.dispatchWorkgroups(workgroupsX, workgroupsY, 1);
810
+ pass.end();
811
+ device.queue.submit([encoder.finish()]);
812
+ } catch (error) {
813
+ if (ownedOutput) releaseBuffer(ownedOutput);
814
+ uniformBuffer.destroy();
815
+ throw error;
816
+ }
817
+ uniformBuffer.destroy();
818
+ return createTensor(output, 'f32', [batchSize, intermediateSize], 'fused_normed_ffn_output');
819
+ }
820
+
821
+ export async function recordFusedFFNFromRMSNormStats(
822
+ recorder,
823
+ input,
824
+ invRmsBuffer,
825
+ normWeight,
826
+ W_gate,
827
+ W_up,
828
+ hiddenSize,
829
+ intermediateSize,
830
+ options = {}
831
+ ) {
832
+ const {
833
+ batchSize,
834
+ activation,
835
+ normWeightDtype,
836
+ } = validateFusedNormedFFNInputs(input, invRmsBuffer, normWeight, W_gate, W_up, hiddenSize, intermediateSize, options);
837
+ resolveSwigluLimit(options.swigluLimit, 'FusedNormedFFN');
838
+ const constants = normalizeQ4KPipelineConstants(options.pipelineConstants, 'FusedNormedFFN Q4K');
839
+ const outputSize = batchSize * intermediateSize * 4;
840
+ const ownedOutput = options.outputBuffer ? null : acquireBuffer(outputSize, undefined, 'fused_normed_ffn_output');
841
+ const output = options.outputBuffer || ownedOutput;
842
+ const uniformBuffer = createNormedFFNUniformBuffer(recorder.device, recorder, {
843
+ M: batchSize,
844
+ hiddenSize,
845
+ intermediateSize,
846
+ alpha: options.alpha ?? 1.0,
847
+ activation,
848
+ swigluLimit: activation === 'silu' ? options.swigluLimit : null,
849
+ rmsNormWeightOffset: options.rmsNormWeightOffset === true,
850
+ normWeightDtype,
851
+ });
852
+ try {
853
+ const pipeline = getFusedNormedFFNPipeline(recorder.device, constants);
854
+ const bindGroup = createFusedNormedBindGroup(recorder.device, pipeline, uniformBuffer, input, normWeight, invRmsBuffer, W_gate, W_up, output);
855
+ const { workgroupsX, workgroupsY } = calculateFusedNormedDispatch(batchSize, intermediateSize, constants);
856
+ recordDispatch(recorder, pipeline, bindGroup, [workgroupsX, workgroupsY, 1], 'fused_normed_ffn_q4k');
857
+ } catch (error) {
858
+ if (ownedOutput) releaseBuffer(ownedOutput);
859
+ throw error;
860
+ }
861
+ return createTensor(output, 'f32', [batchSize, intermediateSize], 'fused_normed_ffn_output');
862
+ }
863
+
343
864
 
344
865
  export function calculateFusedFFNSavings(
345
866
  batchSize,
@@ -65,7 +65,8 @@ fn silu(x: f32) -> f32 {
65
65
  // GELU activation (approximate)
66
66
  fn gelu(x: f32) -> f32 {
67
67
  let c = 0.7978845608; // sqrt(2/pi)
68
- return 0.5 * x * (1.0 + tanh(c * (x + 0.044715 * x * x * x)));
68
+ let inner = c * (x + 0.044715 * x * x * x);
69
+ return 0.5 * x * (1.0 + tanh(clamp(inner, -15.0, 15.0)));
69
70
  }
70
71
 
71
72
  fn clamp_swiglu(x: f32) -> f32 {