doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -1,477 +0,0 @@
1
- --- src/gpu/kernels/fused_matmul_rmsnorm.wgsl
2
- +++ src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl
3
- @@ -1,16 +1,21 @@
4
- -// Fused GEMV + RMSNorm Kernel
5
- +// Fused GEMV + RMSNorm Kernel (F16 variant)
6
- //
7
- // For decode (M=1), combines the down projection matmul with RMSNorm in a single kernel:
8
- // 1. Compute GEMV: C[1, N] = A[1, K] × B[K, N] (down projection)
9
- // 2. Compute RMSNorm on C: output = C / sqrt(mean(C^2) + eps) * weight
10
- // 3. Optional residual: output = output + residual
11
- //
12
- +// F16 variant: All buffers are f16, accumulation in f32 for stability.
13
- +//
14
- // Benefits:
15
- // - Single GPU dispatch instead of 2
16
- // - No intermediate buffer for matmul output
17
- // - Better cache locality
18
- +// - 2x memory bandwidth vs F32 variant
19
- //
20
- -// Expected speedup: 1.2-1.5x for post-FFN normalization path
21
- +// Expected speedup: 1.3-1.8x for post-FFN normalization path
22
- +
23
- +enable f16;
24
-
25
- override WORKGROUP_SIZE: u32 = 256u;
26
- const MAX_WORKGROUP_SIZE: u32 = 256u;
27
- @@ -25,19 +30,19 @@
28
- eps: f32, // RMSNorm epsilon
29
- has_residual: u32, // 1 if residual provided, 0 otherwise
30
- transpose_b: u32, // 1 if weight is [N,K] (row-major), 0 if [K,N] (column-major)
31
- - _pad0: u32, // Padding to 24 bytes for alignment
32
- + _pad0: u32,
33
- _pad1: u32,
34
- _pad2: u32,
35
- }
36
-
37
- @group(0) @binding(0) var<uniform> u: Uniforms;
38
- -@group(0) @binding(1) var<storage, read> input: array<f32>; // [1, K] - activation from FFN
39
- -@group(0) @binding(2) var<storage, read> weight: array<f32>; // [K, N] - down projection weight
40
- +@group(0) @binding(1) var<storage, read> input: array<f16>; // [1, K] - activation from FFN
41
- +@group(0) @binding(2) var<storage, read> weight: array<f16>; // [K, N] - down projection weight
42
- @group(0) @binding(3) var<storage, read> norm_weight: array<u32>; // [N] - RMSNorm weight (F16 or F32)
43
- -@group(0) @binding(4) var<storage, read_write> output: array<f32>;// [1, N] - final output
44
- -@group(0) @binding(5) var<storage, read> residual: array<f32>; // [1, N] - optional residual
45
- -
46
- -// Shared memory for reduction
47
- +@group(0) @binding(4) var<storage, read_write> output: array<f16>;// [1, N] - final output
48
- +@group(0) @binding(5) var<storage, read> residual: array<f16>; // [1, N] - optional residual
49
- +
50
- +// Shared memory for reduction (f32 for numerical stability)
51
- var<workgroup> shared_partial: array<f32, MAX_WORKGROUP_SIZE>; // Partial dot products
52
- var<workgroup> shared_output: array<f32, MAX_WORKGROUP_SIZE>; // Output values for RMSNorm
53
- var<workgroup> shared_sum_sq: array<f32, MAX_WORKGROUP_SIZE>; // Sum of squares for reduction
54
- @@ -58,18 +63,166 @@
55
- return bitcast<f32>(norm_weight[idx]);
56
- }
57
-
58
- -// Fused GEMV + RMSNorm: Main entry point
59
- -// Each workgroup computes COLS_PER_WG output columns, then all workgroups
60
- -// cooperate to compute the RMSNorm across the full output row.
61
- -//
62
- -// Phase 1: GEMV - compute dot products for each output column
63
- -// Phase 2: Store outputs and compute partial sum of squares
64
- -// Phase 3: Global reduction of sum of squares (via atomic or multi-pass)
65
- -// Phase 4: Normalize outputs and optionally add residual
66
- -//
67
- -// Note: This kernel requires N <= WG_SIZE * COLS_PER_WG for single-pass RMSNorm.
68
- -// For larger N, use the multi-workgroup variant.
69
- -
70
- +// Optimized single-workgroup variant for small N (hidden_size <= WORKGROUP_SIZE)
71
- +@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
72
- +fn gemv_rmsnorm_small(
73
- + @builtin(local_invocation_id) local_id: vec3<u32>
74
- +) {
75
- + let tid = local_id.x;
76
- + let N = u.N;
77
- + let K = u.K;
78
- +
79
- + // Each thread computes one output column (for N <= WORKGROUP_SIZE)
80
- + var dot_sum: f32 = 0.0;
81
- + if (tid < N) {
82
- + // Unroll by 4 for better ILP
83
- + var k: u32 = 0u;
84
- + let k_aligned = (K / 4u) * 4u;
85
- +
86
- + for (; k < k_aligned; k = k + 4u) {
87
- + let a0 = f32(input[k]);
88
- + let a1 = f32(input[k + 1u]);
89
- + let a2 = f32(input[k + 2u]);
90
- + let a3 = f32(input[k + 3u]);
91
- +
92
- + if (u.transpose_b == 1u) {
93
- + // Row-major: weight[tid, k]
94
- + let base = tid * K + k;
95
- + dot_sum = dot_sum + a0 * f32(weight[base])
96
- + + a1 * f32(weight[base + 1u])
97
- + + a2 * f32(weight[base + 2u])
98
- + + a3 * f32(weight[base + 3u]);
99
- + } else {
100
- + // Column-major: weight[k, tid]
101
- + dot_sum = dot_sum + a0 * f32(weight[k * N + tid])
102
- + + a1 * f32(weight[(k + 1u) * N + tid])
103
- + + a2 * f32(weight[(k + 2u) * N + tid])
104
- + + a3 * f32(weight[(k + 3u) * N + tid]);
105
- + }
106
- + }
107
- +
108
- + // Handle remainder
109
- + for (; k < K; k = k + 1u) {
110
- + let w_idx = select(k * N + tid, tid * K + k, u.transpose_b == 1u);
111
- + dot_sum = dot_sum + f32(input[k]) * f32(weight[w_idx]);
112
- + }
113
- + }
114
- +
115
- + // Store output value and compute sum of squares
116
- + shared_output[tid] = dot_sum;
117
- + shared_sum_sq[tid] = select(0.0, dot_sum * dot_sum, tid < N);
118
- + workgroupBarrier();
119
- +
120
- + // Parallel reduction for sum of squares
121
- + for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
122
- + if (tid < stride) {
123
- + shared_sum_sq[tid] = shared_sum_sq[tid] + shared_sum_sq[tid + stride];
124
- + }
125
- + workgroupBarrier();
126
- + }
127
- +
128
- + // Compute inverse RMS
129
- + let mean_sq = shared_sum_sq[0] / f32(N);
130
- + let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
131
- +
132
- + // Write normalized output
133
- + if (tid < N) {
134
- + var result = shared_output[tid] * inv_rms * apply_weight(load_norm_weight(tid));
135
- + if (u.has_residual == 1u) {
136
- + result = result + f32(residual[tid]);
137
- + }
138
- + output[tid] = f16(result);
139
- + }
140
- +}
141
- +
142
- +// Medium variant for N up to ~4096 (single workgroup, multiple elements per thread)
143
- +// Handles Gemma's hidden_size=2304 and similar models
144
- +@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
145
- +fn gemv_rmsnorm_medium(
146
- + @builtin(local_invocation_id) local_id: vec3<u32>
147
- +) {
148
- + let tid = local_id.x;
149
- + let N = u.N;
150
- + let K = u.K;
151
- +
152
- + // Each thread handles multiple output columns
153
- + let cols_per_thread = (N + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
154
- +
155
- + // Phase 1: Compute dot products for all columns assigned to this thread
156
- + var local_sum_sq: f32 = 0.0;
157
- +
158
- + for (var c: u32 = 0u; c < cols_per_thread; c = c + 1u) {
159
- + let col = tid + c * WORKGROUP_SIZE;
160
- + if (col < N) {
161
- + // Compute dot product for this column
162
- + var dot_sum: f32 = 0.0;
163
- +
164
- + // Unroll by 4 for better ILP
165
- + var k: u32 = 0u;
166
- + let k_aligned = (K / 4u) * 4u;
167
- +
168
- + for (; k < k_aligned; k = k + 4u) {
169
- + let a0 = f32(input[k]);
170
- + let a1 = f32(input[k + 1u]);
171
- + let a2 = f32(input[k + 2u]);
172
- + let a3 = f32(input[k + 3u]);
173
- +
174
- + if (u.transpose_b == 1u) {
175
- + let base = col * K + k;
176
- + dot_sum = dot_sum + a0 * f32(weight[base])
177
- + + a1 * f32(weight[base + 1u])
178
- + + a2 * f32(weight[base + 2u])
179
- + + a3 * f32(weight[base + 3u]);
180
- + } else {
181
- + dot_sum = dot_sum + a0 * f32(weight[k * N + col])
182
- + + a1 * f32(weight[(k + 1u) * N + col])
183
- + + a2 * f32(weight[(k + 2u) * N + col])
184
- + + a3 * f32(weight[(k + 3u) * N + col]);
185
- + }
186
- + }
187
- +
188
- + // Handle remainder
189
- + for (; k < K; k = k + 1u) {
190
- + let w_idx = select(k * N + col, col * K + k, u.transpose_b == 1u);
191
- + dot_sum = dot_sum + f32(input[k]) * f32(weight[w_idx]);
192
- + }
193
- +
194
- + // Store to output buffer (will normalize in-place later)
195
- + output[col] = f16(dot_sum);
196
- + local_sum_sq = local_sum_sq + dot_sum * dot_sum;
197
- + }
198
- + }
199
- +
200
- + // Phase 2: Parallel reduction for sum of squares
201
- + shared_sum_sq[tid] = local_sum_sq;
202
- + workgroupBarrier();
203
- +
204
- + for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
205
- + if (tid < stride) {
206
- + shared_sum_sq[tid] = shared_sum_sq[tid] + shared_sum_sq[tid + stride];
207
- + }
208
- + workgroupBarrier();
209
- + }
210
- +
211
- + // Compute inverse RMS (all threads read the same value)
212
- + let mean_sq = shared_sum_sq[0] / f32(N);
213
- + let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
214
- +
215
- + // Phase 3: Normalize outputs in-place
216
- + for (var c: u32 = 0u; c < cols_per_thread; c = c + 1u) {
217
- + let col = tid + c * WORKGROUP_SIZE;
218
- + if (col < N) {
219
- + var result = f32(output[col]) * inv_rms * apply_weight(load_norm_weight(col));
220
- + if (u.has_residual == 1u) {
221
- + result = result + f32(residual[col]);
222
- + }
223
- + output[col] = f16(result);
224
- + }
225
- + }
226
- +}
227
- +
228
- +// Main entry point - dispatches based on N size
229
- +// Uses multiple workgroups for large N
230
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
231
- fn main(
232
- @builtin(global_invocation_id) global_id: vec3<u32>,
233
- @@ -82,23 +235,20 @@
234
- let K = u.K;
235
-
236
- // Each workgroup handles COLS_PER_WG columns of output
237
- - let col_in_wg = tid / THREADS_PER_COL; // Which column within this workgroup (0-3)
238
- - let thread_in_col = tid % THREADS_PER_COL; // Thread index within column (0-63)
239
- - let global_col = wg_id.x * COLS_PER_WG + col_in_wg; // Global column index
240
- + let col_in_wg = tid / THREADS_PER_COL;
241
- + let thread_in_col = tid % THREADS_PER_COL;
242
- + let global_col = wg_id.x * COLS_PER_WG + col_in_wg;
243
-
244
- // Phase 1: Compute dot product for this column
245
- var dot_sum: f32 = 0.0;
246
- if (global_col < N) {
247
- - // Each thread in a column processes a stripe of K
248
- let k_per_thread = (K + THREADS_PER_COL - 1u) / THREADS_PER_COL;
249
- let k_start = thread_in_col * k_per_thread;
250
- let k_end = min(k_start + k_per_thread, K);
251
-
252
- for (var k: u32 = k_start; k < k_end; k = k + 1u) {
253
- - // transpose_b=1: weight is [N,K], access weight[global_col * K + k]
254
- - // transpose_b=0: weight is [K,N], access weight[k * N + global_col]
255
- let w_idx = select(k * N + global_col, global_col * K + k, u.transpose_b == 1u);
256
- - dot_sum = dot_sum + input[k] * weight[w_idx];
257
- + dot_sum = dot_sum + f32(input[k]) * f32(weight[w_idx]);
258
- }
259
- }
260
-
261
- @@ -106,8 +256,7 @@
262
- shared_partial[tid] = dot_sum;
263
- workgroupBarrier();
264
-
265
- - // Reduce within each column (THREADS_PER_COL threads -> 1 value)
266
- - // Tree reduction: 64 -> 32 -> 16 -> 8 -> 4 -> 2 -> 1
267
- + // Reduce within each column
268
- for (var stride: u32 = THREADS_PER_COL / 2u; stride > 0u; stride = stride >> 1u) {
269
- if (thread_in_col < stride) {
270
- let idx = col_in_wg * THREADS_PER_COL + thread_in_col;
271
- @@ -125,14 +274,7 @@
272
- }
273
- workgroupBarrier();
274
-
275
- - // Phase 2: For single-workgroup case (N <= 1024), we can do RMSNorm directly
276
- - // For multi-workgroup, we'd need a second pass or atomic accumulation
277
- -
278
- - // This version handles N <= WORKGROUP_SIZE * COLS_PER_WG
279
- - // For larger N, spawn multiple workgroups and do global reduction
280
- -
281
- // Reduce sum of squares across columns in this workgroup
282
- - // All threads must participate in barriers, so move barrier outside condition
283
- for (var stride: u32 = COLS_PER_WG / 2u; stride > 0u; stride = stride >> 1u) {
284
- if (tid < stride && tid + stride < COLS_PER_WG) {
285
- shared_sum_sq[tid] = shared_sum_sq[tid] + shared_sum_sq[tid + stride];
286
- @@ -140,185 +282,20 @@
287
- workgroupBarrier();
288
- }
289
-
290
- - // For single-workgroup: compute RMS and normalize
291
- - // Thread 0 has partial sum; need global sum across all workgroups
292
- - // For now, assume single workgroup (N <= COLS_PER_WG = 4)
293
- - // Single-workgroup kernel; selection code enforces N within limits.
294
- -
295
- let local_sum_sq = shared_sum_sq[0];
296
-
297
- // Phase 3: Write normalized output
298
- if (thread_in_col == 0u && global_col < N) {
299
- let val = shared_output[col_in_wg];
300
- -
301
- - // For multi-WG: would need global sum here
302
- - // For now: just use local (works for small N)
303
- let mean_sq = local_sum_sq / f32(min(COLS_PER_WG, N));
304
- let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
305
-
306
- var result = val * inv_rms * apply_weight(load_norm_weight(global_col));
307
-
308
- if (u.has_residual == 1u) {
309
- - result = result + residual[global_col];
310
- - }
311
- -
312
- - output[global_col] = result;
313
- - }
314
- -}
315
- -
316
- -// Optimized single-workgroup variant for small N (hidden_size <= WORKGROUP_SIZE)
317
- -// All output columns computed by one workgroup, RMSNorm in single pass
318
- -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
319
- -fn gemv_rmsnorm_small(
320
- - @builtin(local_invocation_id) local_id: vec3<u32>
321
- -) {
322
- - let tid = local_id.x;
323
- - let N = u.N;
324
- - let K = u.K;
325
- -
326
- - // Each thread computes one output column (for N <= WORKGROUP_SIZE)
327
- - var dot_sum: f32 = 0.0;
328
- - if (tid < N) {
329
- - for (var k: u32 = 0u; k < K; k = k + 1u) {
330
- - let w_idx = select(k * N + tid, tid * K + k, u.transpose_b == 1u);
331
- - dot_sum = dot_sum + input[k] * weight[w_idx];
332
- - }
333
- - }
334
- -
335
- - // Store output value and compute sum of squares
336
- - shared_output[tid] = dot_sum;
337
- - shared_sum_sq[tid] = select(0.0, dot_sum * dot_sum, tid < N);
338
- - workgroupBarrier();
339
- -
340
- - // Parallel reduction for sum of squares
341
- - for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
342
- - if (tid < stride) {
343
- - shared_sum_sq[tid] = shared_sum_sq[tid] + shared_sum_sq[tid + stride];
344
- - }
345
- - workgroupBarrier();
346
- - }
347
- -
348
- - // Compute inverse RMS
349
- - let mean_sq = shared_sum_sq[0] / f32(N);
350
- - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
351
- -
352
- - // Write normalized output
353
- - if (tid < N) {
354
- - var result = shared_output[tid] * inv_rms * apply_weight(load_norm_weight(tid));
355
- - if (u.has_residual == 1u) {
356
- - result = result + residual[tid];
357
- - }
358
- - output[tid] = result;
359
- - }
360
- -}
361
- -
362
- -// Medium variant for N up to ~4096 (single workgroup, multiple elements per thread)
363
- -// Handles Gemma 3's hidden_size=1152 and similar models
364
- -// Each thread computes ceil(N/WORKGROUP_SIZE) output columns sequentially
365
- -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
366
- -fn gemv_rmsnorm_medium(
367
- - @builtin(local_invocation_id) local_id: vec3<u32>
368
- -) {
369
- - let tid = local_id.x;
370
- - let N = u.N;
371
- - let K = u.K;
372
- -
373
- - // Each thread handles multiple output columns
374
- - let cols_per_thread = (N + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
375
- -
376
- - // Phase 1: Compute dot products for all columns assigned to this thread
377
- - // Store partial sum of squares as we go
378
- - var local_sum_sq: f32 = 0.0;
379
- -
380
- - for (var c: u32 = 0u; c < cols_per_thread; c = c + 1u) {
381
- - let col = tid + c * WORKGROUP_SIZE;
382
- - if (col < N) {
383
- - // Compute dot product for this column
384
- - var dot_sum: f32 = 0.0;
385
- - for (var k: u32 = 0u; k < K; k = k + 1u) {
386
- - let w_idx = select(k * N + col, col * K + k, u.transpose_b == 1u);
387
- - dot_sum = dot_sum + input[k] * weight[w_idx];
388
- - }
389
- -
390
- - // Store to output buffer (will normalize in-place later)
391
- - output[col] = dot_sum;
392
- - local_sum_sq = local_sum_sq + dot_sum * dot_sum;
393
- - }
394
- - }
395
- -
396
- - // Phase 2: Parallel reduction for sum of squares
397
- - shared_sum_sq[tid] = local_sum_sq;
398
- - workgroupBarrier();
399
- -
400
- - for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
401
- - if (tid < stride) {
402
- - shared_sum_sq[tid] = shared_sum_sq[tid] + shared_sum_sq[tid + stride];
403
- - }
404
- - workgroupBarrier();
405
- - }
406
- -
407
- - // Compute inverse RMS (all threads read the same value)
408
- - let mean_sq = shared_sum_sq[0] / f32(N);
409
- - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
410
- -
411
- - // Phase 3: Normalize outputs in-place
412
- - for (var c: u32 = 0u; c < cols_per_thread; c = c + 1u) {
413
- - let col = tid + c * WORKGROUP_SIZE;
414
- - if (col < N) {
415
- - var result = output[col] * inv_rms * apply_weight(load_norm_weight(col));
416
- - if (u.has_residual == 1u) {
417
- - result = result + residual[col];
418
- - }
419
- - output[col] = result;
420
- - }
421
- - }
422
- -}
423
- -
424
- -// Multi-workgroup variant using two-phase approach
425
- -// Phase 1: Each workgroup computes output columns and partial sum of squares
426
- -// Phase 2: Separate kernel reduces partials and normalizes
427
- -// This entry point is Phase 1 only
428
- -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
429
- -fn gemv_rmsnorm_phase1(
430
- - @builtin(global_invocation_id) global_id: vec3<u32>,
431
- - @builtin(local_invocation_id) local_id: vec3<u32>,
432
- - @builtin(workgroup_id) wg_id: vec3<u32>
433
- -) {
434
- - let tid = local_id.x;
435
- - let N = u.N;
436
- - let K = u.K;
437
- -
438
- - let col_in_wg = tid / THREADS_PER_COL;
439
- - let thread_in_col = tid % THREADS_PER_COL;
440
- - let global_col = wg_id.x * COLS_PER_WG + col_in_wg;
441
- -
442
- - // Compute dot product
443
- - var dot_sum: f32 = 0.0;
444
- - if (global_col < N) {
445
- - let k_per_thread = (K + THREADS_PER_COL - 1u) / THREADS_PER_COL;
446
- - let k_start = thread_in_col * k_per_thread;
447
- - let k_end = min(k_start + k_per_thread, K);
448
- -
449
- - for (var k: u32 = k_start; k < k_end; k = k + 1u) {
450
- - let w_idx = select(k * N + global_col, global_col * K + k, u.transpose_b == 1u);
451
- - dot_sum = dot_sum + input[k] * weight[w_idx];
452
- - }
453
- - }
454
- -
455
- - shared_partial[tid] = dot_sum;
456
- - workgroupBarrier();
457
- -
458
- - // Reduce within column
459
- - for (var stride: u32 = THREADS_PER_COL / 2u; stride > 0u; stride = stride >> 1u) {
460
- - if (thread_in_col < stride) {
461
- - let idx = col_in_wg * THREADS_PER_COL + thread_in_col;
462
- - shared_partial[idx] = shared_partial[idx] + shared_partial[idx + stride];
463
- - }
464
- - workgroupBarrier();
465
- - }
466
- -
467
- - // Write matmul output (RMSNorm will be done in phase 2)
468
- - if (thread_in_col == 0u && global_col < N) {
469
- - output[global_col] = shared_partial[col_in_wg * THREADS_PER_COL];
470
- - }
471
- -}
472
- + result = result + f32(residual[global_col]);
473
- + }
474
- +
475
- + output[global_col] = f16(result);
476
- + }
477
- +}
@@ -1,60 +0,0 @@
1
- --- src/gpu/kernels/fused_swiglu.wgsl
2
- +++ src/gpu/kernels/fused_swiglu_f16.wgsl
3
- @@ -1,30 +1,24 @@
4
- -// fused_swiglu.wgsl
5
- +// fused_swiglu_f16.wgsl
6
-
7
- /**
8
- - * SwiGLU Activation Kernel (Row-Split Fused Gate/Up)
9
- - *
10
- - * Computes: output = SiLU(gate + gate_bias) * (up + up_bias)
11
- - *
12
- - * input layout: [num_tokens, 2 * dim] flattened, row-major
13
- - * row = [gate[0..dim), up[0..dim)]
14
- - * bias layout: [2 * dim]
15
- - * bias = [gate_bias[0..dim), up_bias[0..dim)]
16
- - * output layout: [num_tokens, dim]
17
- + * SwiGLU Activation Kernel (Row-Split Fused Gate/Up, f16 IO)
18
- */
19
- +
20
- +enable f16;
21
-
22
- override WORKGROUP_SIZE: u32 = 256u;
23
-
24
- struct Uniforms {
25
- num_tokens: u32,
26
- dim: u32,
27
- - bias_offset: u32, // byte offset into bias buffer (divide by 4 for F32 index)
28
- - clamp_max: f32, // SwiGLU clamp (0 = disabled)
29
- + bias_offset: u32,
30
- + clamp_max: f32,
31
- }
32
-
33
- @group(0) @binding(0) var<uniform> u: Uniforms;
34
- -@group(0) @binding(1) var<storage, read> input: array<f32>;
35
- -@group(0) @binding(2) var<storage, read> bias: array<f32>;
36
- -@group(0) @binding(3) var<storage, read_write> output: array<f32>;
37
- +@group(0) @binding(1) var<storage, read> input: array<f16>;
38
- +@group(0) @binding(2) var<storage, read> bias: array<f16>;
39
- +@group(0) @binding(3) var<storage, read_write> output: array<f16>;
40
-
41
- fn sigmoid(x: f32) -> f32 {
42
- return 1.0 / (1.0 + exp(-x));
43
- @@ -51,13 +45,11 @@
44
-
45
- let token_idx = idx / u.dim;
46
- let dim_idx = idx % u.dim;
47
- -
48
- - // bias_offset is in bytes, convert to F32 index
49
- - let bias_base = u.bias_offset / 4u;
50
- + let bias_base = u.bias_offset / 2u;
51
-
52
- let row_base = token_idx * u.dim * 2u;
53
- - let gate = input[row_base + dim_idx] + bias[bias_base + dim_idx];
54
- - let up = input[row_base + u.dim + dim_idx] + bias[bias_base + u.dim + dim_idx];
55
- + let gate = f32(input[row_base + dim_idx]) + f32(bias[bias_base + dim_idx]);
56
- + let up = f32(input[row_base + u.dim + dim_idx]) + f32(bias[bias_base + u.dim + dim_idx]);
57
-
58
- - output[idx] = clamp_swiglu(silu(gate) * up);
59
- + output[idx] = f16(clamp_swiglu(silu(gate) * up));
60
- }
@@ -1,41 +0,0 @@
1
- --- src/gpu/kernels/gelu.wgsl
2
- +++ src/gpu/kernels/gelu_f16.wgsl
3
- @@ -1,9 +1,9 @@
4
- -// GeLU Activation Kernel
5
- +// GeLU Activation Kernel with F16 Input/Output
6
- //
7
- -// GeLU(x) ≈ 0.5 * x * (1 + tanh(sqrt(2/π) * (x + 0.044715 * x^3)))
8
- -// Variants:
9
- -// - geglu: GeLU(gate) * up (separate gate buffer)
10
- -// - geglu_rowsplit: input = [numTokens, 2*dim] row-split
11
- +// F16 variant for reduced memory bandwidth when using F16 activations.
12
- +
13
- +enable f16;
14
-
15
- override WORKGROUP_SIZE: u32 = 256u;
16
- override HAS_GATE: bool = false;
17
- @@ -17,9 +17,9 @@
18
- }
19
-
20
- @group(0) @binding(0) var<uniform> u: Uniforms;
21
- -@group(0) @binding(1) var<storage, read> input: array<f32>;
22
- -@group(0) @binding(2) var<storage, read_write> output: array<f32>;
23
- -@group(0) @binding(3) var<storage, read> gate: array<f32>;
24
- +@group(0) @binding(1) var<storage, read> input: array<f16>;
25
- +@group(0) @binding(2) var<storage, read_write> output: array<f16>;
26
- +@group(0) @binding(3) var<storage, read> gate: array<f16>;
27
-
28
- -fn gelu(x: f32) -> f32 {
29
- - let sqrt_2_over_pi: f32 = 0.7978845608;
30
- - let c: f32 = 0.044715;
31
- +fn gelu(x: f16) -> f16 {
32
- + let sqrt_2_over_pi = f16(0.7978845608);
33
- + let c = f16(0.044715);
34
- let inner = sqrt_2_over_pi * (x + c * x * x * x);
35
- - let inner_clamped = clamp(inner, -15.0, 15.0);
36
- - return 0.5 * x * (1.0 + tanh(inner_clamped));
37
- + let inner_clamped = clamp(inner, f16(-15.0), f16(15.0));
38
- + return f16(0.5) * x * (f16(1.0) + tanh(inner_clamped));
39
- }
40
-
41
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
@@ -1,82 +0,0 @@
1
- --- src/gpu/kernels/layernorm.wgsl
2
- +++ src/gpu/kernels/layernorm_f16.wgsl
3
- @@ -1,20 +1,12 @@
4
- -// LayerNorm Kernel
5
- +// LayerNorm Kernel (F16 Input/Output)
6
- //
7
- // LayerNorm(x) = (x - mean) / sqrt(var + eps) * weight + bias
8
- -//
9
- -// Uses workgroup reduction for mean and variance per token.
10
-
11
- -// =============================================================================
12
- -// Override Constants
13
- -// =============================================================================
14
- +enable f16;
15
-
16
- override WORKGROUP_SIZE: u32 = 256u;
17
- const MAX_WORKGROUP_SIZE: u32 = 256u;
18
- override PARAMS_IS_F16: bool = false; // Weight/bias packed as f16 pairs
19
- -
20
- -// =============================================================================
21
- -// Uniforms
22
- -// =============================================================================
23
-
24
- struct Uniforms {
25
- size: u32,
26
- @@ -24,21 +16,13 @@
27
- }
28
-
29
- @group(0) @binding(0) var<uniform> u: Uniforms;
30
- -@group(0) @binding(1) var<storage, read> input: array<f32>;
31
- +@group(0) @binding(1) var<storage, read> input: array<f16>;
32
- @group(0) @binding(2) var<storage, read> weight: array<u32>;
33
- @group(0) @binding(3) var<storage, read> bias: array<u32>;
34
- -@group(0) @binding(4) var<storage, read_write> output: array<f32>;
35
- -
36
- -// =============================================================================
37
- -// Shared Memory
38
- -// =============================================================================
39
- +@group(0) @binding(4) var<storage, read_write> output: array<f16>;
40
-
41
- var<workgroup> shared_sum: array<f32, MAX_WORKGROUP_SIZE>;
42
- var<workgroup> shared_sum_sq: array<f32, MAX_WORKGROUP_SIZE>;
43
- -
44
- -// =============================================================================
45
- -// Helpers
46
- -// =============================================================================
47
-
48
- fn load_weight(idx: u32) -> f32 {
49
- if (PARAMS_IS_F16) {
50
- @@ -57,10 +41,6 @@
51
- }
52
- return bitcast<f32>(bias[idx]);
53
- }
54
- -
55
- -// =============================================================================
56
- -// Entry Point
57
- -// =============================================================================
58
-
59
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
60
- fn main(
61
- @@ -84,7 +64,7 @@
62
- for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
63
- let idx = thread_idx * elements_per_thread + i;
64
- if (idx < size) {
65
- - let x = input[base_offset + idx];
66
- + let x = f32(input[base_offset + idx]);
67
- local_sum = local_sum + x;
68
- local_sum_sq = local_sum_sq + x * x;
69
- }
70
- @@ -112,10 +92,10 @@
71
- for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
72
- let idx = thread_idx * elements_per_thread + i;
73
- if (idx < size) {
74
- - let x = input[base_offset + idx];
75
- + let x = f32(input[base_offset + idx]);
76
- let norm = (x - mean) * inv_std;
77
- let y = norm * load_weight(idx) + load_bias(idx);
78
- - output[base_offset + idx] = y;
79
- + output[base_offset + idx] = f16(y);
80
- }
81
- }
82
- }