doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -0,0 +1,2629 @@
1
+
2
+
3
+ // Import from main doppler repo (relative path from tests/kernels/browser/)
4
+ // When served from doppler/, paths are relative to that root
5
+ import { initDevice, getKernelCapabilities, getDeviceLimits, destroyDevice } from '../../../src/gpu/device.js';
6
+
7
+ // Import tensor abstraction for Tensor-based kernels
8
+ import { createTensor } from '../../../src/gpu/tensor.js';
9
+
10
+ // Ensure platform/registry lookups resolve to the main config paths when bundled
11
+ import { setPlatformsBaseUrl } from '../../../src/config/platforms/loader.js';
12
+ import { setRegistryUrl } from '../../../src/config/kernels/registry.js';
13
+ import { createPipeline, createUniformBufferWithView } from '../../../src/gpu/kernels/utils.js';
14
+ import { dispatch } from '../../../src/gpu/kernels/dispatch.js';
15
+
16
+ // Kernel path can be injected via URL param for targeted tests
17
+ import { resolveKernelPath, setActiveKernelPath } from '../../../src/config/kernel-path-loader.js';
18
+
19
+ import {
20
+ runBackwardKernel,
21
+ recordBackwardKernel,
22
+ runMatmulTransposeA,
23
+ } from '../../../src/gpu/kernels/backward/utils.js';
24
+ import * as kernelSelector from '../../../src/gpu/kernel-selector.js';
25
+
26
+ // Destructure available functions with defaults
27
+ const {
28
+ runMatmul = null,
29
+ runSoftmax = null,
30
+ runTopK = null,
31
+ runSoftmaxTopK = null,
32
+ runScatterAdd = null,
33
+ runMoEGather = null,
34
+ runRMSNorm = null,
35
+ runRoPE = null,
36
+ runSiLU = null,
37
+ runSwiGLURowsplitBias = null,
38
+ runScale = null,
39
+ runGather = null,
40
+ runResidualAdd = null,
41
+ runBiasAdd = null,
42
+ runAttention = null,
43
+ runAttentionTiered = null,
44
+ runAttentionTieredQuant = null,
45
+ dequantize = null,
46
+ dequantizeQ6K = null,
47
+ runBF16ToF32 = null,
48
+ runBF16ToF16 = null,
49
+ castF32ToF16 = null,
50
+ castF16ToF32 = null,
51
+ runGeLU = null,
52
+ runSplitQKV = null,
53
+ runConv2D = null,
54
+ runGroupNorm = null,
55
+ runLayerNorm = null,
56
+ runLayerNormBackward = null,
57
+ runModulate = null,
58
+ runPixelShuffle = null,
59
+ runUpsample2D = null,
60
+ runTranspose = null,
61
+ runKVQuantize = null,
62
+ runCrossEntropyLoss = null,
63
+ runCrossEntropyBackward = null,
64
+ runBiasAddBackward = null,
65
+ runUpsample2DBackward = null,
66
+ runPixelShuffleBackward = null,
67
+ runGroupNormBackward = null,
68
+ runConv2DBackward = null,
69
+ runEmbedBackward = null,
70
+ runGeluBackward = null,
71
+ runRmsNormBackward = null,
72
+ runRoPEBackward = null,
73
+ runScaleBackward = null,
74
+ runSiluBackward = null,
75
+ runSoftmaxBackward = null,
76
+ runAttentionBackward = null,
77
+ runAdam = null,
78
+ } = kernelSelector;
79
+
80
+ // Import dequant variant selector for test diagnostics
81
+ import { selectDequantKernel } from '../../../src/gpu/kernels/dequant.js';
82
+
83
+ // Import sample kernel
84
+ import * as sampleKernel from '../../../src/gpu/kernels/sample.js';
85
+
86
+ // Import check-stop kernel
87
+ import { checkStop } from '../../../src/gpu/kernels/check-stop.js';
88
+
89
+ // Import fused kernels
90
+ import { runMatmulResidualFused } from '../../../src/gpu/kernels/fused_matmul_residual.js';
91
+ import { runMatmulRMSNormFused } from '../../../src/gpu/kernels/fused_matmul_rmsnorm.js';
92
+ import { runFusedFFN } from '../../../src/gpu/kernels/fused_ffn.js';
93
+
94
+ // Optional buffer pool
95
+ let bufferPool = null;
96
+ try {
97
+ bufferPool = await import('../../../src/memory/buffer-pool.js');
98
+ } catch (e) {
99
+ console.warn('Buffer pool not available:', e.message);
100
+ }
101
+
102
+ // Import reference implementations
103
+ import * as references from '../reference/index.js';
104
+ import { compareArrays, generateTestData, KERNEL_TOLERANCES } from '../harness/tolerance.js';
105
+ import { createBuffer, readGPUBuffer, readAsFloat32, readAsUint32 } from '../harness/buffer-utils.js';
106
+ import { KernelBenchmark, computeMetrics } from '../harness/benchmark.js';
107
+ import { createWeightBuffer } from '../../../src/gpu/weight-buffer.js';
108
+
109
+ // Global state
110
+ let device = null;
111
+ let initialized = false;
112
+
113
+
114
+ function f16ToF32(h) {
115
+ const sign = (h & 0x8000) >> 15;
116
+ const exponent = (h & 0x7C00) >> 10;
117
+ const mantissa = h & 0x03FF;
118
+
119
+ if (exponent === 0) {
120
+ // Denormalized or zero
121
+ if (mantissa === 0) return sign ? -0 : 0;
122
+ return (sign ? -1 : 1) * Math.pow(2, -14) * (mantissa / 1024);
123
+ } else if (exponent === 31) {
124
+ // Infinity or NaN
125
+ return mantissa === 0 ? (sign ? -Infinity : Infinity) : NaN;
126
+ }
127
+
128
+ // Normalized
129
+ return (sign ? -1 : 1) * Math.pow(2, exponent - 15) * (1 + mantissa / 1024);
130
+ }
131
+
132
+ function f32ToF16Bits(value) {
133
+ return references.float32ToFloat16(value);
134
+ }
135
+
136
+ function toF16Array(values) {
137
+ const out = new Uint16Array(values.length);
138
+ for (let i = 0; i < values.length; i++) {
139
+ out[i] = f32ToF16Bits(values[i]);
140
+ }
141
+ return out;
142
+ }
143
+
144
+ function toF16RoundedFloat32(values) {
145
+ const out = new Float32Array(values.length);
146
+ for (let i = 0; i < values.length; i++) {
147
+ out[i] = f16ToF32(f32ToF16Bits(values[i]));
148
+ }
149
+ return out;
150
+ }
151
+
152
+ async function readTensorToFloat32(tensor, count) {
153
+ if (tensor.dtype === 'f16') {
154
+ const rawData = await readBufferData(tensor.buffer, count * 2);
155
+ const u16View = new Uint16Array(rawData);
156
+ const out = new Float32Array(u16View.length);
157
+ for (let i = 0; i < u16View.length; i++) {
158
+ out[i] = f16ToF32(u16View[i]);
159
+ }
160
+ return out;
161
+ }
162
+ return new Float32Array(await readBufferData(tensor.buffer, count * 4));
163
+ }
164
+
165
+ function buildAttentionKernelPath(id, kernelFile) {
166
+ return {
167
+ id,
168
+ name: id,
169
+ activationDtype: 'f16',
170
+ decode: {
171
+ steps: [
172
+ {
173
+ op: 'attention',
174
+ kernel: kernelFile,
175
+ entry: 'main',
176
+ constants: { SOFTCAP: 50.0 },
177
+ },
178
+ ],
179
+ },
180
+ };
181
+ }
182
+
183
+ function fillDeterministic(values, scale = 0.01) {
184
+ for (let i = 0; i < values.length; i++) {
185
+ values[i] = Math.sin(i * 0.13) * scale;
186
+ }
187
+ return values;
188
+ }
189
+
190
+
191
+ async function initGPU() {
192
+ if (device) return device;
193
+
194
+ setPlatformsBaseUrl('/src/config/platforms/');
195
+ setRegistryUrl('/src/config/kernels/registry.json');
196
+
197
+ device = await initDevice();
198
+ if (!device) {
199
+ throw new Error('WebGPU not available');
200
+ }
201
+
202
+ setActiveKernelPath(null, 'none');
203
+
204
+ initialized = true;
205
+ return device;
206
+ }
207
+
208
+
209
+ async function getGPU() {
210
+ if (!device) {
211
+ await initGPU();
212
+ }
213
+ return { device: device, queue: device.queue };
214
+ }
215
+
216
+
217
+ function makeBuffer(
218
+ data,
219
+ usage = GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC
220
+ ) {
221
+ const byteLength = data instanceof ArrayBuffer ? data.byteLength : data.byteLength;
222
+ const buffer = device.createBuffer({
223
+ size: byteLength,
224
+ usage: usage | GPUBufferUsage.COPY_DST,
225
+ mappedAtCreation: true,
226
+ });
227
+
228
+ const mappedRange = buffer.getMappedRange();
229
+ if (data instanceof Float32Array) {
230
+ new Float32Array(mappedRange).set(data);
231
+ } else if (data instanceof Uint32Array) {
232
+ new Uint32Array(mappedRange).set(data);
233
+ } else if (data instanceof Int32Array) {
234
+ new Int32Array(mappedRange).set(data);
235
+ } else if (data instanceof Uint16Array) {
236
+ new Uint16Array(mappedRange).set(data);
237
+ } else if (data instanceof Uint8Array) {
238
+ new Uint8Array(mappedRange).set(data);
239
+ } else {
240
+ new Uint8Array(mappedRange).set(new Uint8Array(data));
241
+ }
242
+ buffer.unmap();
243
+
244
+ return buffer;
245
+ }
246
+
247
+
248
+ async function readBufferData(buffer, size) {
249
+ const stagingBuffer = device.createBuffer({
250
+ size,
251
+ usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
252
+ });
253
+
254
+ const encoder = device.createCommandEncoder();
255
+ encoder.copyBufferToBuffer(buffer, 0, stagingBuffer, 0, size);
256
+ device.queue.submit([encoder.finish()]);
257
+
258
+ await stagingBuffer.mapAsync(GPUMapMode.READ);
259
+ const data = new Uint8Array(stagingBuffer.getMappedRange()).slice();
260
+ stagingBuffer.unmap();
261
+ stagingBuffer.destroy();
262
+
263
+ return data.buffer;
264
+ }
265
+
266
+ // ============================================================================
267
+ // Test Harness - Exposed to window for browser automation
268
+ // ============================================================================
269
+
270
+ const testHarness = {
271
+ // Core
272
+ getGPU,
273
+ device: () => device,
274
+
275
+ // Reference implementations
276
+ references,
277
+ softmax: references.softmaxRef,
278
+ topkRef: references.topkRef,
279
+ softmaxTopkRef: references.softmaxTopkRef,
280
+ matmulRef: references.matmulRef,
281
+ scatterAddRef: references.scatterAddRef,
282
+
283
+ // Utilities
284
+ generateTestData,
285
+ compareArrays,
286
+ makeBuffer,
287
+ readBufferData,
288
+ toF16RoundedFloat32,
289
+ KERNEL_TOLERANCES,
290
+ selectDequantKernel,
291
+
292
+ async runMatmulTransposeA(dev, A, B, M, N, K, options = {}) {
293
+ const bufA = makeBuffer(A);
294
+ const bufB = makeBuffer(B);
295
+ const tensorA = createTensor(bufA, 'f32', [K, M], 'matmul_ta_a');
296
+ const tensorB = createTensor(bufB, 'f32', [K, N], 'matmul_ta_b');
297
+
298
+ const resultTensor = await runMatmulTransposeA(tensorA, tensorB, M, N, K, options);
299
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
300
+
301
+ bufA.destroy();
302
+ bufB.destroy();
303
+ resultTensor.buffer.destroy();
304
+
305
+ return result;
306
+ },
307
+
308
+ // ============================================================================
309
+ // Kernel Runners (match expected interface from tests)
310
+ // ============================================================================
311
+
312
+
313
+ async runMatmul(dev, A, B, M, N, K, alpha = 1.0) {
314
+ if (!runMatmul) {
315
+ // Fallback to reference implementation
316
+ return references.matmulRef(A, B, M, N, K, alpha);
317
+ }
318
+
319
+ const bufA = makeBuffer(A);
320
+ const tensorA = createTensor(bufA, 'f32', [M, K], 'matmul_a');
321
+ const bufB = makeBuffer(B);
322
+
323
+ // Test uses standard layout B [K, N], so transposeB = false
324
+ // (GPU kernel defaults to transposeB=true for SafeTensors [N, K] layout)
325
+ const resultTensor = await runMatmul(tensorA, bufB, M, N, K, {
326
+ alpha,
327
+ transposeB: false,
328
+ bDtype: 'f32',
329
+ outputDtype: 'f32',
330
+ });
331
+
332
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
333
+
334
+ bufA.destroy();
335
+ bufB.destroy();
336
+ resultTensor.buffer.destroy();
337
+
338
+ return result;
339
+ },
340
+
341
+
342
+ async runBatchMatmul(dev, A, B, batch, M, N, K) {
343
+ // Always use reference - batch matmul kernel may not be implemented
344
+ return references.batchMatmulRef(A, B, batch, M, N, K);
345
+ },
346
+
347
+
348
+ async runMatvec(dev, A, x, M, K) {
349
+ // Always use reference - matvec kernel may not be implemented
350
+ return references.matvecRef(A, x, M, K);
351
+ },
352
+
353
+
354
+ async runMatmulQ4K(dev, A, B_q4k, M, N, K, alpha = 1.0) {
355
+ if (!runMatmul) {
356
+ throw new Error('runMatmul kernel not available');
357
+ }
358
+
359
+ // Create A buffer (activations)
360
+ const bufA = makeBuffer(A);
361
+ const tensorA = createTensor(bufA, 'f32', [M, K], 'matmul_q4k_a');
362
+
363
+ // Create B buffer and pass q4k dtype to trigger fused Q4K selection
364
+ const bufB = makeBuffer(B_q4k, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC);
365
+
366
+ // Run matmul - kernel auto-detects q4k and uses fused variant
367
+ // transposeB is implicit for Q4K (weight matrix stored as [N, K])
368
+ const resultTensor = await runMatmul(tensorA, bufB, M, N, K, {
369
+ alpha,
370
+ bDtype: 'q4k',
371
+ outputDtype: 'f32',
372
+ });
373
+
374
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
375
+
376
+ bufA.destroy();
377
+ bufB.destroy();
378
+ resultTensor.buffer.destroy();
379
+
380
+ return result;
381
+ },
382
+
383
+
384
+ async runSoftmax(dev, input, innerSize, outerSize, temperature = 1.0) {
385
+ if (!runSoftmax) {
386
+ return references.softmaxRef(input, innerSize, outerSize, temperature);
387
+ }
388
+
389
+ const inputBuf = makeBuffer(input);
390
+ const inputTensor = createTensor(inputBuf, 'f32', [outerSize, innerSize], 'softmax_input');
391
+
392
+ const resultTensor = await runSoftmax(inputTensor, -1, {
393
+ batchSize: outerSize,
394
+ size: innerSize,
395
+ temperature,
396
+ });
397
+
398
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, input.length * 4));
399
+
400
+ inputBuf.destroy();
401
+ resultTensor.buffer.destroy();
402
+
403
+ return result;
404
+ },
405
+
406
+
407
+ async runSoftmaxTopK(dev, logits, numTokens, numExperts, topK, options = {}) {
408
+ if (!runSoftmaxTopK) {
409
+ return references.softmaxTopkRef(logits, numTokens, numExperts, topK, options.normalize !== false);
410
+ }
411
+
412
+ const inputBuf = makeBuffer(logits);
413
+
414
+ const { indices: indicesBuf, weights: weightsBuf } = await runSoftmaxTopK(
415
+ inputBuf,
416
+ numTokens,
417
+ numExperts,
418
+ topK,
419
+ {
420
+ normalize: options.normalize !== false,
421
+ inputDtype: options.inputDtype ?? 'f32',
422
+ weightsDtype: options.weightsDtype ?? 'f32',
423
+ }
424
+ );
425
+
426
+ const indices = new Uint32Array(await readBufferData(indicesBuf, numTokens * topK * 4));
427
+ const weights = new Float32Array(await readBufferData(weightsBuf, numTokens * topK * 4));
428
+
429
+ inputBuf.destroy();
430
+ indicesBuf.destroy();
431
+ weightsBuf.destroy();
432
+
433
+ return { indices, weights };
434
+ },
435
+
436
+
437
+ async runTopK(dev, probs, numTokens, numExperts, topK, options = {}) {
438
+ const inputBuf = makeBuffer(probs);
439
+
440
+ const { indices: indicesBuf, weights: weightsBuf } = await runTopK(
441
+ inputBuf,
442
+ numTokens,
443
+ numExperts,
444
+ topK,
445
+ { normalize: options.normalize !== false }
446
+ );
447
+
448
+ const indices = new Uint32Array(await readBufferData(indicesBuf, numTokens * topK * 4));
449
+ const weights = new Float32Array(await readBufferData(weightsBuf, numTokens * topK * 4));
450
+
451
+ inputBuf.destroy();
452
+ indicesBuf.destroy();
453
+ weightsBuf.destroy();
454
+
455
+ return { indices, weights };
456
+ },
457
+
458
+
459
+ async runScatterAdd(dev, expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK) {
460
+ if (!runScatterAdd) {
461
+ return references.scatterAddRef(expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK);
462
+ }
463
+
464
+ const expertBuf = makeBuffer(expertOutputs);
465
+ const indicesBuf = makeBuffer(indices);
466
+ const weightsBuf = makeBuffer(weights);
467
+
468
+ // Wrap expertBuf in Tensor (MoE kernels now use Tensor abstraction)
469
+ const expertTensor = createTensor(expertBuf, 'f32', [numExperts, numTokens, hiddenSize], 'expert_outputs');
470
+ const resultTensor = await runScatterAdd(
471
+ expertTensor,
472
+ indicesBuf,
473
+ weightsBuf,
474
+ numTokens,
475
+ hiddenSize,
476
+ numExperts,
477
+ topK
478
+ );
479
+
480
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, numTokens * hiddenSize * 4));
481
+
482
+ expertBuf.destroy();
483
+ indicesBuf.destroy();
484
+ weightsBuf.destroy();
485
+ resultTensor.buffer.destroy();
486
+
487
+ return result;
488
+ },
489
+
490
+
491
+ async runRMSNorm(dev, input, weight, numTokens, hiddenSize, eps = 1e-6, options = {}) {
492
+ if (!runRMSNorm) {
493
+ return references.rmsNormRef(input, weight, numTokens, hiddenSize, eps);
494
+ }
495
+
496
+ const inputBuf = makeBuffer(input);
497
+ const weightBuf = makeBuffer(weight);
498
+ const inputTensor = createTensor(inputBuf, 'f32', [numTokens, hiddenSize], 'rmsnorm_input');
499
+
500
+ const resultTensor = await runRMSNorm(inputTensor, weightBuf, eps, {
501
+ batchSize: numTokens,
502
+ hiddenSize,
503
+ ...options,
504
+ });
505
+
506
+ let result;
507
+ if (resultTensor.dtype === 'f16') {
508
+ const rawData = await readBufferData(resultTensor.buffer, numTokens * hiddenSize * 2);
509
+ const u16View = new Uint16Array(rawData);
510
+ result = new Float32Array(u16View.length);
511
+ for (let i = 0; i < u16View.length; i++) {
512
+ result[i] = f16ToF32(u16View[i]);
513
+ }
514
+ } else {
515
+ result = new Float32Array(
516
+ await readBufferData(resultTensor.buffer, numTokens * hiddenSize * 4)
517
+ );
518
+ }
519
+
520
+ inputBuf.destroy();
521
+ weightBuf.destroy();
522
+ resultTensor.buffer.destroy();
523
+
524
+ return result;
525
+ },
526
+
527
+
528
+ async runRoPE(dev, input, seqLen, numHeads, headDim, startPos = 0) {
529
+ const { cos, sin } = references.computeRopeFreqs(headDim, seqLen + startPos);
530
+
531
+ if (!runRoPE) {
532
+ return references.ropeRef(input, cos, sin, seqLen, numHeads, headDim, startPos);
533
+ }
534
+
535
+ const inputBuf = makeBuffer(input);
536
+ const cosBuf = makeBuffer(cos);
537
+ const sinBuf = makeBuffer(sin);
538
+
539
+ const inputTensor = createTensor(inputBuf, 'f32', [seqLen, numHeads, headDim], 'rope_input');
540
+
541
+ await runRoPE(inputTensor, cosBuf, sinBuf, seqLen, {
542
+ numHeads,
543
+ headDim,
544
+ startPos,
545
+ });
546
+
547
+ const result = new Float32Array(
548
+ await readBufferData(inputBuf, seqLen * numHeads * headDim * 4)
549
+ );
550
+
551
+ inputBuf.destroy();
552
+ cosBuf.destroy();
553
+ sinBuf.destroy();
554
+
555
+ return result;
556
+ },
557
+
558
+
559
+ async runSiLU(dev, input) {
560
+ if (!runSiLU) {
561
+ return references.siluRef(input);
562
+ }
563
+
564
+ const inputBuf = makeBuffer(input);
565
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'silu_input');
566
+
567
+ const resultTensor = await runSiLU(inputTensor, { size: input.length, swigluLimit: null });
568
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, input.length * 4));
569
+
570
+ inputBuf.destroy();
571
+ resultTensor.buffer.destroy();
572
+
573
+ return result;
574
+ },
575
+
576
+
577
+ async runSiLUGated(dev, gate, up) {
578
+ if (!runSiLU) {
579
+ return references.siluGatedRef(gate, up);
580
+ }
581
+
582
+ const gateBuf = makeBuffer(gate);
583
+ const upBuf = makeBuffer(up);
584
+
585
+ const gateTensor = createTensor(gateBuf, 'f32', [gate.length], 'silu_gate');
586
+ const upTensor = createTensor(upBuf, 'f32', [up.length], 'silu_up');
587
+
588
+ const resultTensor = await runSiLU(upTensor, {
589
+ size: up.length,
590
+ gate: gateTensor,
591
+ swigluLimit: null,
592
+ inputActivation: 'identity',
593
+ });
594
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, up.length * 4));
595
+
596
+ gateBuf.destroy();
597
+ upBuf.destroy();
598
+ resultTensor.buffer.destroy();
599
+
600
+ return result;
601
+ },
602
+
603
+
604
+ async runGather(dev, embeddings, indices, vocabSize, embedDim) {
605
+ if (!runGather) {
606
+ return references.gatherRef(embeddings, indices, vocabSize, embedDim);
607
+ }
608
+
609
+ const embBuf = makeBuffer(embeddings);
610
+ const idxBuf = makeBuffer(indices);
611
+ const numTokens = indices.length;
612
+
613
+ const embTensor = createTensor(embBuf, 'f32', [vocabSize, embedDim], 'gather_embeddings');
614
+ const idxTensor = createTensor(idxBuf, 'u32', [numTokens], 'gather_indices');
615
+
616
+ // Test data uses standard [vocab_size, hidden_size] layout, not GGUF [hidden_size, vocab_size]
617
+ const resultTensor = await runGather(idxTensor.buffer, embTensor.buffer, numTokens, embedDim, vocabSize, { transpose: false, embeddingDtype: 'f32', outputDtype: 'f32' });
618
+ let result;
619
+ if (resultTensor.dtype === 'f16') {
620
+ const rawData = await readBufferData(resultTensor.buffer, numTokens * embedDim * 2);
621
+ const u16View = new Uint16Array(rawData);
622
+ result = new Float32Array(u16View.length);
623
+ for (let i = 0; i < u16View.length; i++) {
624
+ result[i] = f16ToF32(u16View[i]);
625
+ }
626
+ } else {
627
+ result = new Float32Array(
628
+ await readBufferData(resultTensor.buffer, numTokens * embedDim * 4)
629
+ );
630
+ }
631
+
632
+ embBuf.destroy();
633
+ idxBuf.destroy();
634
+ resultTensor.buffer.destroy();
635
+
636
+ return result;
637
+ },
638
+
639
+
640
+ async runResidual(dev, x, residual, options = {}) {
641
+ if (!runResidualAdd) {
642
+ return options.outputScale == null
643
+ ? references.residualAddRef(x, residual)
644
+ : references.outputScaledResidualAddRef(x, residual, options.outputScale);
645
+ }
646
+
647
+ const xBuf = makeBuffer(x);
648
+ const resBuf = makeBuffer(residual);
649
+ const size = x.length;
650
+
651
+ const xTensor = createTensor(xBuf, 'f32', [size], 'residual_x');
652
+ const resTensor = createTensor(resBuf, 'f32', [size], 'residual_res');
653
+
654
+ const resultTensor = await runResidualAdd(xTensor, resTensor, size, {
655
+ outputScale: options.outputScale,
656
+ });
657
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, size * 4));
658
+
659
+ xBuf.destroy();
660
+ resBuf.destroy();
661
+ resultTensor.buffer.destroy();
662
+
663
+ return result;
664
+ },
665
+
666
+
667
+ async runBiasAdd(dev, data, bias, numTokens, dim) {
668
+ if (!runBiasAdd) {
669
+ const result = new Float32Array(data);
670
+ for (let t = 0; t < numTokens; t++) {
671
+ const rowOffset = t * dim;
672
+ for (let d = 0; d < dim; d++) {
673
+ result[rowOffset + d] += bias[d];
674
+ }
675
+ }
676
+ return result;
677
+ }
678
+
679
+ const dataBuf = makeBuffer(data);
680
+ const biasBuf = makeBuffer(bias);
681
+ const dataTensor = createTensor(dataBuf, 'f32', [numTokens, dim], 'bias_add_data');
682
+ const biasTensor = createTensor(biasBuf, 'f32', [dim], 'bias_add_bias');
683
+
684
+ const resultTensor = await runBiasAdd(dataTensor, biasTensor, numTokens, dim);
685
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, numTokens * dim * 4));
686
+
687
+ dataBuf.destroy();
688
+ biasBuf.destroy();
689
+ if (resultTensor.buffer !== dataBuf) {
690
+ resultTensor.buffer.destroy();
691
+ }
692
+
693
+ return result;
694
+ },
695
+
696
+
697
+ async runLayerNorm(dev, input, weight, bias, batchSize, hiddenSize, eps = 1e-5) {
698
+ if (!runLayerNorm) {
699
+ const output = new Float32Array(batchSize * hiddenSize);
700
+ for (let b = 0; b < batchSize; b++) {
701
+ const base = b * hiddenSize;
702
+ let mean = 0;
703
+ for (let i = 0; i < hiddenSize; i++) {
704
+ mean += input[base + i];
705
+ }
706
+ mean /= hiddenSize;
707
+ let varSum = 0;
708
+ for (let i = 0; i < hiddenSize; i++) {
709
+ const diff = input[base + i] - mean;
710
+ varSum += diff * diff;
711
+ }
712
+ const invStd = 1 / Math.sqrt(varSum / hiddenSize + eps);
713
+ for (let i = 0; i < hiddenSize; i++) {
714
+ const norm = (input[base + i] - mean) * invStd;
715
+ output[base + i] = norm * weight[i] + bias[i];
716
+ }
717
+ }
718
+ return output;
719
+ }
720
+
721
+ const inputBuf = makeBuffer(input);
722
+ const weightBuf = makeBuffer(weight);
723
+ const biasBuf = makeBuffer(bias);
724
+ const inputTensor = createTensor(inputBuf, 'f32', [batchSize, hiddenSize], 'layernorm_input');
725
+
726
+ const resultTensor = await runLayerNorm(inputTensor, weightBuf, biasBuf, eps, {
727
+ batchSize,
728
+ hiddenSize,
729
+ });
730
+ const result = await readTensorToFloat32(resultTensor, batchSize * hiddenSize);
731
+
732
+ inputBuf.destroy();
733
+
734
+ weightBuf.destroy();
735
+
736
+ biasBuf.destroy();
737
+
738
+ resultTensor.buffer.destroy();
739
+
740
+
741
+
742
+ return result;
743
+
744
+ },
745
+
746
+
747
+
748
+ async runLayerNormBackward(dev, input, weight, gradOutput, numTokens, hiddenSize, eps = 1e-5) {
749
+
750
+ if (!runLayerNormBackward) {
751
+
752
+ // Reference implementation
753
+
754
+ const output = new Float32Array(numTokens * hiddenSize);
755
+
756
+ for (let b = 0; b < numTokens; b++) {
757
+
758
+ const base = b * hiddenSize;
759
+
760
+ let mean = 0;
761
+
762
+ for (let i = 0; i < hiddenSize; i++) {
763
+
764
+ mean += input[base + i];
765
+
766
+ }
767
+
768
+ mean /= hiddenSize;
769
+
770
+ let varSum = 0;
771
+
772
+ let sumGY = 0;
773
+
774
+ let sumGYX = 0;
775
+
776
+ for (let i = 0; i < hiddenSize; i++) {
777
+
778
+ const x = input[base + i];
779
+
780
+ const diff = x - mean;
781
+
782
+ varSum += diff * diff;
783
+
784
+ const gy = gradOutput[base + i] * weight[i];
785
+
786
+ sumGY += gy;
787
+
788
+ sumGYX += gy * diff;
789
+
790
+ }
791
+
792
+ const invStd = 1 / Math.sqrt(varSum / hiddenSize + eps);
793
+
794
+ const invStd2 = invStd * invStd;
795
+
796
+ for (let i = 0; i < hiddenSize; i++) {
797
+
798
+ const x = input[base + i];
799
+
800
+ const gy = gradOutput[base + i] * weight[i];
801
+
802
+ output[base + i] = invStd * (gy - (sumGY + (x - mean) * invStd2 * sumGYX) / hiddenSize);
803
+
804
+ }
805
+
806
+ }
807
+
808
+ return output;
809
+
810
+ }
811
+
812
+
813
+
814
+ const inputBuf = makeBuffer(input);
815
+
816
+ const weightBuf = makeBuffer(weight);
817
+
818
+ const gradBuf = makeBuffer(gradOutput);
819
+
820
+ const inputTensor = createTensor(inputBuf, 'f32', [numTokens, hiddenSize], 'layernorm_bw_input');
821
+
822
+ const weightTensor = createTensor(weightBuf, 'f32', [hiddenSize], 'layernorm_bw_weight');
823
+
824
+ const gradTensor = createTensor(gradBuf, 'f32', [numTokens, hiddenSize], 'layernorm_bw_grad');
825
+
826
+
827
+
828
+ const result = await runLayerNormBackward(inputTensor, weightTensor, gradTensor, {
829
+ numTokens,
830
+ hiddenSize,
831
+ eps,
832
+ });
833
+ const resultTensor = result?.gradInput || result;
834
+ const resultData = await readTensorToFloat32(resultTensor, numTokens * hiddenSize);
835
+
836
+ inputBuf.destroy();
837
+ weightBuf.destroy();
838
+ gradBuf.destroy();
839
+ if (result?.gradInput?.buffer) {
840
+ result.gradInput.buffer.destroy();
841
+ } else if (result?.buffer) {
842
+ result.buffer.destroy();
843
+ }
844
+ result?.gradWeight?.buffer?.destroy();
845
+ result?.gradBias?.buffer?.destroy();
846
+
847
+ return resultData;
848
+
849
+ },
850
+
851
+
852
+
853
+ async runGroupNorm(dev, input, weight, bias, channels, height, width, numGroups, eps = 1e-5) {
854
+ if (!runGroupNorm) {
855
+ const output = new Float32Array(channels * height * width);
856
+ const channelsPerGroup = Math.floor(channels / numGroups);
857
+ for (let g = 0; g < numGroups; g++) {
858
+ const cStart = g * channelsPerGroup;
859
+ const cEnd = cStart + channelsPerGroup;
860
+ let mean = 0;
861
+ let count = 0;
862
+ for (let c = cStart; c < cEnd; c++) {
863
+ const base = c * height * width;
864
+ for (let i = 0; i < height * width; i++) {
865
+ mean += input[base + i];
866
+ count++;
867
+ }
868
+ }
869
+ mean /= count;
870
+ let varSum = 0;
871
+ for (let c = cStart; c < cEnd; c++) {
872
+ const base = c * height * width;
873
+ for (let i = 0; i < height * width; i++) {
874
+ const diff = input[base + i] - mean;
875
+ varSum += diff * diff;
876
+ }
877
+ }
878
+ const invStd = 1 / Math.sqrt(varSum / count + eps);
879
+ for (let c = cStart; c < cEnd; c++) {
880
+ const base = c * height * width;
881
+ for (let i = 0; i < height * width; i++) {
882
+ const norm = (input[base + i] - mean) * invStd;
883
+ output[base + i] = norm * weight[c] + bias[c];
884
+ }
885
+ }
886
+ }
887
+ return output;
888
+ }
889
+
890
+ const inputBuf = makeBuffer(input);
891
+ const weightBuf = makeBuffer(weight);
892
+ const biasBuf = makeBuffer(bias);
893
+ const inputTensor = createTensor(inputBuf, 'f32', [channels, height, width], 'groupnorm_input');
894
+
895
+ const resultTensor = await runGroupNorm(inputTensor, weightBuf, biasBuf, {
896
+ channels,
897
+ height,
898
+ width,
899
+ numGroups,
900
+ eps,
901
+ });
902
+ const result = await readTensorToFloat32(resultTensor, channels * height * width);
903
+
904
+ inputBuf.destroy();
905
+ weightBuf.destroy();
906
+ biasBuf.destroy();
907
+ resultTensor.buffer.destroy();
908
+
909
+ return result;
910
+ },
911
+
912
+
913
+ async runModulate(dev, input, mod, numTokens, hiddenSize, options = {}) {
914
+ const { scaleOffset = 0, shiftOffset = hiddenSize, gateOffset = hiddenSize * 2, hasGate = false, addOne = true } = options;
915
+ if (!runModulate) {
916
+ const output = new Float32Array(numTokens * hiddenSize);
917
+ for (let t = 0; t < numTokens; t++) {
918
+ const base = t * hiddenSize;
919
+ for (let d = 0; d < hiddenSize; d++) {
920
+ const rawScale = mod[scaleOffset + d];
921
+ const shift = mod[shiftOffset + d];
922
+ const scale = addOne ? 1 + rawScale : rawScale;
923
+ let value = input[base + d] * scale + shift;
924
+ if (hasGate) {
925
+ value *= mod[gateOffset + d];
926
+ }
927
+ output[base + d] = value;
928
+ }
929
+ }
930
+ return output;
931
+ }
932
+
933
+ const inputBuf = makeBuffer(input);
934
+ const modBuf = makeBuffer(mod);
935
+ const inputTensor = createTensor(inputBuf, 'f32', [numTokens, hiddenSize], 'modulate_input');
936
+ const modTensor = createTensor(modBuf, 'f32', [mod.length], 'modulate_params');
937
+
938
+ const resultTensor = await runModulate(inputTensor, modTensor, {
939
+ numTokens,
940
+ hiddenSize,
941
+ scaleOffset,
942
+ shiftOffset,
943
+ gateOffset,
944
+ hasGate,
945
+ addOne,
946
+ });
947
+ const result = await readTensorToFloat32(resultTensor, numTokens * hiddenSize);
948
+
949
+ inputBuf.destroy();
950
+ modBuf.destroy();
951
+ resultTensor.buffer.destroy();
952
+
953
+ return result;
954
+ },
955
+
956
+
957
+ async runConv2D(dev, input, weight, bias, options = {}) {
958
+ if (!runConv2D) {
959
+ const {
960
+ inChannels,
961
+ outChannels,
962
+ height,
963
+ width,
964
+ kernelH,
965
+ kernelW,
966
+ stride = 1,
967
+ pad = 0,
968
+ } = options;
969
+ const outHeight = Math.floor((height + pad * 2 - kernelH) / stride) + 1;
970
+ const outWidth = Math.floor((width + pad * 2 - kernelW) / stride) + 1;
971
+ const output = new Float32Array(outChannels * outHeight * outWidth);
972
+ for (let oc = 0; oc < outChannels; oc++) {
973
+ for (let oy = 0; oy < outHeight; oy++) {
974
+ for (let ox = 0; ox < outWidth; ox++) {
975
+ let sum = bias ? bias[oc] : 0;
976
+ for (let ic = 0; ic < inChannels; ic++) {
977
+ for (let ky = 0; ky < kernelH; ky++) {
978
+ const inY = oy * stride + ky - pad;
979
+ if (inY < 0 || inY >= height) continue;
980
+ for (let kx = 0; kx < kernelW; kx++) {
981
+ const inX = ox * stride + kx - pad;
982
+ if (inX < 0 || inX >= width) continue;
983
+ const inputIdx = (ic * height + inY) * width + inX;
984
+ const weightIdx = (((oc * inChannels + ic) * kernelH + ky) * kernelW + kx);
985
+ sum += input[inputIdx] * weight[weightIdx];
986
+ }
987
+ }
988
+ }
989
+ output[(oc * outHeight + oy) * outWidth + ox] = sum;
990
+ }
991
+ }
992
+ }
993
+ return output;
994
+ }
995
+
996
+ const inputBuf = makeBuffer(input);
997
+ const weightBuf = makeBuffer(weight);
998
+ const biasBuf = bias ? makeBuffer(bias) : null;
999
+ const inputTensor = createTensor(inputBuf, 'f32', [options.inChannels, options.height, options.width], 'conv2d_input');
1000
+
1001
+ const resultTensor = await runConv2D(inputTensor, weightBuf, biasBuf, options);
1002
+ const stride = options.stride ?? 1;
1003
+ const pad = options.pad ?? 0;
1004
+ const outHeight = Math.floor((options.height + pad * 2 - options.kernelH) / stride) + 1;
1005
+ const outWidth = Math.floor((options.width + pad * 2 - options.kernelW) / stride) + 1;
1006
+ const result = await readTensorToFloat32(resultTensor, options.outChannels * outHeight * outWidth);
1007
+
1008
+ inputBuf.destroy();
1009
+ weightBuf.destroy();
1010
+ biasBuf?.destroy();
1011
+ resultTensor.buffer.destroy();
1012
+
1013
+ return result;
1014
+ },
1015
+
1016
+
1017
+ async runPixelShuffle(dev, input, options = {}) {
1018
+ if (!runPixelShuffle) {
1019
+ const {
1020
+ outChannels,
1021
+ outHeight,
1022
+ outWidth,
1023
+ gridWidth,
1024
+ patchSize,
1025
+ patchChannels,
1026
+ } = options;
1027
+ const output = new Float32Array(outChannels * outHeight * outWidth);
1028
+ const spatial = outHeight * outWidth;
1029
+ for (let idx = 0; idx < output.length; idx++) {
1030
+ const c = Math.floor(idx / spatial);
1031
+ const rem = idx - c * spatial;
1032
+ const y = Math.floor(rem / outWidth);
1033
+ const x = rem - y * outWidth;
1034
+ const gridY = Math.floor(y / patchSize);
1035
+ const gridX = Math.floor(x / patchSize);
1036
+ const subY = y - gridY * patchSize;
1037
+ const subX = x - gridX * patchSize;
1038
+ const tokenIdx = gridY * gridWidth + gridX;
1039
+ const patchIdx = (subY * patchSize + subX) * outChannels + c;
1040
+ const inputIdx = tokenIdx * patchChannels + patchIdx;
1041
+ output[idx] = input[inputIdx];
1042
+ }
1043
+ return output;
1044
+ }
1045
+
1046
+ const inputBuf = makeBuffer(input);
1047
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'pixel_shuffle_input');
1048
+ const resultTensor = await runPixelShuffle(inputTensor, options);
1049
+ const result = await readTensorToFloat32(resultTensor, options.outChannels * options.outHeight * options.outWidth);
1050
+
1051
+ inputBuf.destroy();
1052
+ resultTensor.buffer.destroy();
1053
+
1054
+ return result;
1055
+ },
1056
+
1057
+
1058
+ async runUpsample2D(dev, input, options = {}) {
1059
+ if (!runUpsample2D) {
1060
+ const { channels, inHeight, inWidth, outHeight, outWidth, scale } = options;
1061
+ const output = new Float32Array(channels * outHeight * outWidth);
1062
+ for (let c = 0; c < channels; c++) {
1063
+ for (let oy = 0; oy < outHeight; oy++) {
1064
+ for (let ox = 0; ox < outWidth; ox++) {
1065
+ const inY = Math.floor(oy / scale);
1066
+ const inX = Math.floor(ox / scale);
1067
+ const inIdx = (c * inHeight + inY) * inWidth + inX;
1068
+ const outIdx = (c * outHeight + oy) * outWidth + ox;
1069
+ output[outIdx] = input[inIdx];
1070
+ }
1071
+ }
1072
+ }
1073
+ return output;
1074
+ }
1075
+
1076
+ const height = Number.isFinite(options.height) ? options.height : options.inHeight;
1077
+ const width = Number.isFinite(options.width) ? options.width : options.inWidth;
1078
+ const inputBuf = makeBuffer(input);
1079
+ const inputTensor = createTensor(inputBuf, 'f32', [options.channels, height, width], 'upsample2d_input');
1080
+ const resultTensor = await runUpsample2D(inputTensor, options);
1081
+ const result = await readTensorToFloat32(resultTensor, options.channels * options.outHeight * options.outWidth);
1082
+
1083
+ inputBuf.destroy();
1084
+ resultTensor.buffer.destroy();
1085
+
1086
+ return result;
1087
+ },
1088
+
1089
+
1090
+ async runTranspose(dev, input, rows, cols) {
1091
+ if (!runTranspose) {
1092
+ const output = new Float32Array(rows * cols);
1093
+ for (let r = 0; r < rows; r++) {
1094
+ for (let c = 0; c < cols; c++) {
1095
+ output[c * rows + r] = input[r * cols + c];
1096
+ }
1097
+ }
1098
+ return output;
1099
+ }
1100
+
1101
+ const inputBuf = makeBuffer(input);
1102
+ const inputTensor = createTensor(inputBuf, 'f32', [rows, cols], 'transpose_input');
1103
+ const resultTensor = await runTranspose(inputTensor, rows, cols);
1104
+ const result = await readTensorToFloat32(resultTensor, rows * cols);
1105
+
1106
+ inputBuf.destroy();
1107
+ resultTensor.buffer.destroy();
1108
+
1109
+ return result;
1110
+ },
1111
+
1112
+
1113
+ async runKVQuantize(dev, keys, values, numKVHeads, headDim, numTokens, mode = 'int8') {
1114
+ if (!runKVQuantize) {
1115
+ const packedStride = Math.ceil(headDim / 4);
1116
+ return {
1117
+ packedStride,
1118
+ outputK: new Uint32Array(numTokens * numKVHeads * packedStride),
1119
+ outputV: new Uint32Array(numTokens * numKVHeads * packedStride),
1120
+ scalesK: new Uint16Array(numTokens * numKVHeads),
1121
+ scalesV: new Uint16Array(numTokens * numKVHeads),
1122
+ };
1123
+ }
1124
+
1125
+ const packedStride = Math.ceil(headDim / 4);
1126
+ const keyBuf = makeBuffer(keys, GPUBufferUsage.STORAGE);
1127
+ const valBuf = makeBuffer(values, GPUBufferUsage.STORAGE);
1128
+ const outputSize = numTokens * numKVHeads * packedStride * 4;
1129
+ const scalesSize = numTokens * numKVHeads * 2;
1130
+ const scalesSizeAligned = Math.ceil(scalesSize / 4) * 4;
1131
+ const outputK = makeBuffer(new Uint8Array(outputSize));
1132
+ const outputV = makeBuffer(new Uint8Array(outputSize));
1133
+ const scalesK = makeBuffer(new Uint8Array(scalesSizeAligned));
1134
+ const scalesV = makeBuffer(new Uint8Array(scalesSizeAligned));
1135
+
1136
+ await runKVQuantize(
1137
+ keyBuf,
1138
+ valBuf,
1139
+ outputK,
1140
+ outputV,
1141
+ scalesK,
1142
+ scalesV,
1143
+ {
1144
+ numKVHeads,
1145
+ headDim,
1146
+ startPos: 0,
1147
+ numTokens,
1148
+ packedStride,
1149
+ mode,
1150
+ }
1151
+ );
1152
+
1153
+ const outK = new Uint32Array(await readBufferData(outputK, outputSize));
1154
+ const outV = new Uint32Array(await readBufferData(outputV, outputSize));
1155
+ const outScalesKFull = new Uint16Array(await readBufferData(scalesK, scalesSizeAligned));
1156
+ const outScalesVFull = new Uint16Array(await readBufferData(scalesV, scalesSizeAligned));
1157
+ const outScalesK = outScalesKFull.slice(0, numTokens * numKVHeads);
1158
+ const outScalesV = outScalesVFull.slice(0, numTokens * numKVHeads);
1159
+
1160
+ keyBuf.destroy();
1161
+ valBuf.destroy();
1162
+ outputK.destroy();
1163
+ outputV.destroy();
1164
+ scalesK.destroy();
1165
+ scalesV.destroy();
1166
+
1167
+ return { packedStride, outputK: outK, outputV: outV, scalesK: outScalesK, scalesV: outScalesV };
1168
+ },
1169
+
1170
+
1171
+ async runCrossEntropyLoss(dev, softmax, targets, numTokens, vocabSize) {
1172
+ if (!runCrossEntropyLoss) {
1173
+ const output = new Float32Array(numTokens);
1174
+ for (let t = 0; t < numTokens; t++) {
1175
+ const target = targets[t];
1176
+ if (target >= vocabSize) {
1177
+ output[t] = 0;
1178
+ continue;
1179
+ }
1180
+ const p = Math.max(softmax[t * vocabSize + target], 1e-9);
1181
+ output[t] = -Math.log(p);
1182
+ }
1183
+ return output;
1184
+ }
1185
+
1186
+ const softmaxBuf = makeBuffer(softmax);
1187
+ const targetsBuf = makeBuffer(targets);
1188
+ const softmaxTensor = createTensor(softmaxBuf, 'f32', [numTokens, vocabSize], 'cross_entropy_softmax');
1189
+ const targetsTensor = createTensor(targetsBuf, 'u32', [numTokens], 'cross_entropy_targets');
1190
+
1191
+ const resultTensor = await runCrossEntropyLoss(softmaxTensor, targetsTensor, { numTokens, vocabSize });
1192
+ const result = await readTensorToFloat32(resultTensor, numTokens);
1193
+
1194
+ softmaxBuf.destroy();
1195
+ targetsBuf.destroy();
1196
+ resultTensor.buffer.destroy();
1197
+
1198
+ return result;
1199
+ },
1200
+
1201
+
1202
+ async runCrossEntropyBackward(dev, softmax, targets, gradOutput, numTokens, vocabSize) {
1203
+ if (!runCrossEntropyBackward) {
1204
+ const output = new Float32Array(numTokens * vocabSize);
1205
+ for (let t = 0; t < numTokens; t++) {
1206
+ const target = targets[t];
1207
+ for (let c = 0; c < vocabSize; c++) {
1208
+ let grad = softmax[t * vocabSize + c];
1209
+ if (c === target) grad -= 1;
1210
+ output[t * vocabSize + c] = grad * gradOutput[t];
1211
+ }
1212
+ }
1213
+ return output;
1214
+ }
1215
+
1216
+ const softmaxBuf = makeBuffer(softmax);
1217
+ const targetsBuf = makeBuffer(targets);
1218
+ const gradBuf = makeBuffer(gradOutput);
1219
+ const softmaxTensor = createTensor(softmaxBuf, 'f32', [numTokens, vocabSize], 'cross_entropy_softmax');
1220
+ const targetsTensor = createTensor(targetsBuf, 'u32', [numTokens], 'cross_entropy_targets');
1221
+ const gradTensor = createTensor(gradBuf, 'f32', [numTokens], 'cross_entropy_grad');
1222
+
1223
+ const resultTensor = await runCrossEntropyBackward(softmaxTensor, targetsTensor, gradTensor, { numTokens, vocabSize });
1224
+ const result = await readTensorToFloat32(resultTensor, numTokens * vocabSize);
1225
+
1226
+ softmaxBuf.destroy();
1227
+ targetsBuf.destroy();
1228
+ gradBuf.destroy();
1229
+ resultTensor.buffer.destroy();
1230
+
1231
+ return result;
1232
+ },
1233
+
1234
+ async runBiasAddBackward(dev, gradOutput, numTokens, dim) {
1235
+ if (!runBiasAddBackward) return null;
1236
+ const gBuf = makeBuffer(gradOutput);
1237
+ const resultTensor = await runBiasAddBackward(createTensor(gBuf, 'f32', [numTokens, dim], 'bias_bw_in'), { numTokens, dim });
1238
+ const result = await readTensorToFloat32(resultTensor, dim);
1239
+ gBuf.destroy();
1240
+ resultTensor.buffer.destroy();
1241
+ return result;
1242
+ },
1243
+
1244
+ async runUpsample2DBackward(dev, gradOutput, options) {
1245
+ if (!runUpsample2DBackward) return null;
1246
+ const gBuf = makeBuffer(gradOutput);
1247
+ const resultTensor = await runUpsample2DBackward(createTensor(gBuf, 'f32', [options.channels, options.outHeight, options.outWidth], 'upsample_bw_in'), options);
1248
+ const result = await readTensorToFloat32(resultTensor, options.channels * options.inHeight * options.inWidth);
1249
+ gBuf.destroy();
1250
+ resultTensor.buffer.destroy();
1251
+ return result;
1252
+ },
1253
+
1254
+ async runPixelShuffleBackward(dev, gradOutput, options) {
1255
+ if (!runPixelShuffleBackward) return null;
1256
+ const gBuf = makeBuffer(gradOutput);
1257
+ const resultTensor = await runPixelShuffleBackward(createTensor(gBuf, 'f32', [options.outChannels, options.outHeight, options.outWidth], 'pixel_bw_in'), options);
1258
+ const result = await readTensorToFloat32(resultTensor, options.gridWidth * options.gridHeight * options.patchChannels);
1259
+ gBuf.destroy();
1260
+ resultTensor.buffer.destroy();
1261
+ return result;
1262
+ },
1263
+
1264
+ async runGroupNormBackward(dev, input, weight, gradOutput, options) {
1265
+ if (!runGroupNormBackward) return null;
1266
+ const iBuf = makeBuffer(input);
1267
+ const wBuf = makeBuffer(weight);
1268
+ const gBuf = makeBuffer(gradOutput);
1269
+ const resultTensor = await runGroupNormBackward(
1270
+ createTensor(iBuf, 'f32', [options.channels, options.height, options.width], 'gn_bw_i'),
1271
+ createTensor(wBuf, 'f32', [options.channels], 'gn_bw_w'),
1272
+ createTensor(gBuf, 'f32', [options.channels, options.height, options.width], 'gn_bw_g'),
1273
+ options
1274
+ );
1275
+ const result = await readTensorToFloat32(resultTensor, options.channels * options.height * options.width);
1276
+ iBuf.destroy();
1277
+ wBuf.destroy();
1278
+ gBuf.destroy();
1279
+ resultTensor.buffer.destroy();
1280
+ return result;
1281
+ },
1282
+
1283
+ async runConv2DBackward(dev, input, weight, gradOutput, options) {
1284
+ if (!runConv2DBackward) return null;
1285
+ const iBuf = makeBuffer(input);
1286
+ const wBuf = makeBuffer(weight);
1287
+ const gBuf = makeBuffer(gradOutput);
1288
+ const result = await runConv2DBackward(
1289
+ createTensor(iBuf, 'f32', [options.inChannels, options.height, options.width], 'conv_bw_i'),
1290
+ createTensor(wBuf, 'f32', [options.outChannels, options.inChannels, options.kernelH, options.kernelW], 'conv_bw_w'),
1291
+ createTensor(gBuf, 'f32', [options.outChannels, options.outHeight, options.outWidth], 'conv_bw_g'),
1292
+ options
1293
+ );
1294
+ const gradInput = await readTensorToFloat32(result.gradInput, options.inChannels * options.height * options.width);
1295
+ const gradWeight = await readTensorToFloat32(result.gradWeight, options.outChannels * options.inChannels * options.kernelH * options.kernelW);
1296
+ iBuf.destroy();
1297
+ wBuf.destroy();
1298
+ gBuf.destroy();
1299
+ result.gradInput.buffer.destroy();
1300
+ result.gradWeight.buffer.destroy();
1301
+ return { gradInput, gradWeight };
1302
+ },
1303
+
1304
+ async runDequantQ4K(dev, quantized, numBlocks) {
1305
+ if (!dequantize) {
1306
+ throw new Error('dequantize kernel not available');
1307
+ }
1308
+
1309
+ const qBuf = makeBuffer(quantized, GPUBufferUsage.STORAGE);
1310
+ const outTensor = await dequantize(qBuf, numBlocks, { outputDtype: 'f32', useVec4: false });
1311
+ const out = new Float32Array(await readBufferData(outTensor.buffer, numBlocks * 256 * 4));
1312
+
1313
+ qBuf.destroy();
1314
+ outTensor.buffer.destroy();
1315
+
1316
+ return out;
1317
+ },
1318
+
1319
+
1320
+ async runDequantQ4K_Vec4(dev, quantized, numBlocks) {
1321
+ if (!dequantize) {
1322
+ throw new Error('dequantize kernel not available');
1323
+ }
1324
+
1325
+ const qBuf = makeBuffer(quantized, GPUBufferUsage.STORAGE);
1326
+ const outTensor = await dequantize(qBuf, numBlocks, { outputDtype: 'f32', useVec4: true });
1327
+ const out = new Float32Array(await readBufferData(outTensor.buffer, numBlocks * 256 * 4));
1328
+
1329
+ qBuf.destroy();
1330
+ outTensor.buffer.destroy();
1331
+
1332
+ return out;
1333
+ },
1334
+
1335
+ async runDequantQ4K_F16(dev, quantized, numBlocks) {
1336
+ if (!dequantize) {
1337
+ throw new Error('dequantize kernel not available');
1338
+ }
1339
+
1340
+ const qBuf = makeBuffer(quantized, GPUBufferUsage.STORAGE);
1341
+ // Use F16 output and vec4=true (default) to match production loader path
1342
+ const outTensor = await dequantize(qBuf, numBlocks, { outputDtype: 'f16', useVec4: true });
1343
+
1344
+ // Read back F16 data and convert to F32 for comparison
1345
+ const f16Bytes = numBlocks * 256 * 2; // F16 = 2 bytes per element
1346
+ const rawData = await readBufferData(outTensor.buffer, f16Bytes);
1347
+ const u16 = new Uint16Array(rawData);
1348
+ const out = new Float32Array(numBlocks * 256);
1349
+
1350
+ // Convert F16 to F32
1351
+ for (let i = 0; i < u16.length; i++) {
1352
+ const h = u16[i];
1353
+ const sign = (h >> 15) & 1;
1354
+ const exp = (h >> 10) & 0x1F;
1355
+ const mant = h & 0x3FF;
1356
+ let f;
1357
+ if (exp === 0) {
1358
+ f = mant === 0 ? 0 : Math.pow(2, -14) * (mant / 1024);
1359
+ } else if (exp === 31) {
1360
+ f = mant === 0 ? Infinity : NaN;
1361
+ } else {
1362
+ f = Math.pow(2, exp - 15) * (1 + mant / 1024);
1363
+ }
1364
+ out[i] = sign ? -f : f;
1365
+ }
1366
+
1367
+ qBuf.destroy();
1368
+ outTensor.buffer.destroy();
1369
+
1370
+ return out;
1371
+ },
1372
+
1373
+
1374
+ async runAttention(dev, Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim, mask = null) {
1375
+ if (!runAttention) {
1376
+ // Fallback to reference if kernel not available
1377
+ return references.attentionRef(Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim, mask);
1378
+ }
1379
+
1380
+ // Create GPU buffers
1381
+ const qBuf = makeBuffer(Q);
1382
+ const kBuf = makeBuffer(K);
1383
+ const vBuf = makeBuffer(V);
1384
+ const maskBuf = mask ? makeBuffer(mask) : null;
1385
+ const isCausal = !!mask;
1386
+
1387
+ const qTensor = createTensor(qBuf, 'f32', [seqLen, numHeads, headDim], 'attn_q');
1388
+ const kTensor = createTensor(kBuf, 'f32', [kvLen, numKVHeads, headDim], 'attn_k');
1389
+ const vTensor = createTensor(vBuf, 'f32', [kvLen, numKVHeads, headDim], 'attn_v');
1390
+
1391
+ // Run attention via kernel selector (handles tier selection automatically)
1392
+ const resultTensor = await runAttention(qTensor, kTensor, vTensor, maskBuf, numHeads, headDim, {
1393
+ seqLen,
1394
+ kvLen,
1395
+ numKVHeads,
1396
+ scale: 1 / Math.sqrt(headDim),
1397
+ causal: isCausal,
1398
+ });
1399
+
1400
+ // Read back result
1401
+ const out = new Float32Array(await readBufferData(resultTensor.buffer, seqLen * numHeads * headDim * 4));
1402
+ qBuf.destroy();
1403
+ kBuf.destroy();
1404
+ vBuf.destroy();
1405
+ maskBuf?.destroy();
1406
+ resultTensor.buffer.destroy();
1407
+ return out;
1408
+ },
1409
+
1410
+
1411
+
1412
+ async runAttentionTiered(dev, Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim) {
1413
+ if (!runAttentionTiered) {
1414
+ return references.attentionRef(Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim, null);
1415
+ }
1416
+
1417
+ const qBuf = makeBuffer(toF16Array(Q));
1418
+ const kBuf = makeBuffer(toF16Array(K));
1419
+ const vBuf = makeBuffer(toF16Array(V));
1420
+ const qTensor = createTensor(qBuf, 'f16', [seqLen, numHeads, headDim], 'tiered_q');
1421
+ const hotK = createTensor(kBuf, 'f16', [kvLen, numKVHeads, headDim], 'tiered_hot_k');
1422
+ const hotV = createTensor(vBuf, 'f16', [kvLen, numKVHeads, headDim], 'tiered_hot_v');
1423
+ const coldK = createTensor(makeBuffer(new Uint16Array(2)), 'f16', [1], 'tiered_cold_k');
1424
+ const coldV = createTensor(makeBuffer(new Uint16Array(2)), 'f16', [1], 'tiered_cold_v');
1425
+
1426
+ const resultTensor = await runAttentionTiered(
1427
+ qTensor,
1428
+ hotK,
1429
+ hotV,
1430
+ coldK,
1431
+ coldV,
1432
+ numHeads,
1433
+ headDim,
1434
+ {
1435
+ seqLen,
1436
+ hotLen: kvLen,
1437
+ coldLen: 0,
1438
+ numKVHeads,
1439
+ causal: false,
1440
+ }
1441
+ );
1442
+ const result = await readTensorToFloat32(resultTensor, seqLen * numHeads * headDim);
1443
+
1444
+ qBuf.destroy();
1445
+ kBuf.destroy();
1446
+ vBuf.destroy();
1447
+ coldK.buffer.destroy();
1448
+ coldV.buffer.destroy();
1449
+ resultTensor.buffer.destroy();
1450
+
1451
+ return result;
1452
+ },
1453
+
1454
+
1455
+ async runAttentionTieredQuant(dev, Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim) {
1456
+ if (!runAttentionTieredQuant) {
1457
+ return references.attentionRef(Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim, null);
1458
+ }
1459
+
1460
+ const qBuf = makeBuffer(Q);
1461
+ const kBuf = makeBuffer(toF16Array(K));
1462
+ const vBuf = makeBuffer(toF16Array(V));
1463
+ const qTensor = createTensor(qBuf, 'f32', [seqLen, numHeads, headDim], 'tieredq_q');
1464
+ const hotK = createTensor(kBuf, 'f16', [kvLen, numKVHeads, headDim], 'tieredq_hot_k');
1465
+ const hotV = createTensor(vBuf, 'f16', [kvLen, numKVHeads, headDim], 'tieredq_hot_v');
1466
+
1467
+ const coldPackedK = makeBuffer(new Uint8Array(4));
1468
+ const coldPackedV = makeBuffer(new Uint8Array(4));
1469
+ const coldScalesK = makeBuffer(new Uint8Array(4));
1470
+ const coldScalesV = makeBuffer(new Uint8Array(4));
1471
+
1472
+ const resultTensor = await runAttentionTieredQuant(
1473
+ qTensor,
1474
+ hotK,
1475
+ hotV,
1476
+ coldPackedK,
1477
+ coldPackedV,
1478
+ coldScalesK,
1479
+ coldScalesV,
1480
+ numHeads,
1481
+ headDim,
1482
+ {
1483
+ seqLen,
1484
+ hotLen: kvLen,
1485
+ coldLen: 0,
1486
+ numKVHeads,
1487
+ packedStride: 1,
1488
+ mode: 'int8',
1489
+ causal: false,
1490
+ }
1491
+ );
1492
+ const result = await readTensorToFloat32(resultTensor, seqLen * numHeads * headDim);
1493
+
1494
+ qBuf.destroy();
1495
+ kBuf.destroy();
1496
+ vBuf.destroy();
1497
+ coldPackedK.destroy();
1498
+ coldPackedV.destroy();
1499
+ coldScalesK.destroy();
1500
+ coldScalesV.destroy();
1501
+ resultTensor.buffer.destroy();
1502
+
1503
+ return result;
1504
+ },
1505
+
1506
+
1507
+ async benchmarkAttentionDecodeVariant(dev, options = {}) {
1508
+ if (!runAttention) {
1509
+ throw new Error('runAttention kernel not available');
1510
+ }
1511
+
1512
+ const {
1513
+ kernel = 'attention_decode_chunked_f16.wgsl',
1514
+ kvLens = [128, 256, 512, 1024, 1536, 2048],
1515
+ headDim = 256,
1516
+ numHeads = 8,
1517
+ numKVHeads = 4,
1518
+ warmupRuns = 5,
1519
+ timedRuns = 20,
1520
+ } = options;
1521
+
1522
+ const { device } = await getGPU();
1523
+ const benchmark = new KernelBenchmark(device);
1524
+ const seqLen = 1;
1525
+
1526
+ const qData = fillDeterministic(new Float32Array(numHeads * headDim));
1527
+ const qBuf = makeBuffer(toF16Array(qData));
1528
+ const qTensor = createTensor(qBuf, 'f16', [seqLen, numHeads, headDim], 'bench_q');
1529
+
1530
+ setActiveKernelPath(buildAttentionKernelPath(`bench-${kernel}`, kernel), 'runtime');
1531
+
1532
+ const results = [];
1533
+ for (const kvLen of kvLens) {
1534
+ const kData = fillDeterministic(new Float32Array(kvLen * numKVHeads * headDim));
1535
+ const vData = fillDeterministic(new Float32Array(kvLen * numKVHeads * headDim), 0.02);
1536
+ const kBuf = makeBuffer(toF16Array(kData));
1537
+ const vBuf = makeBuffer(toF16Array(vData));
1538
+
1539
+ const kTensor = createTensor(kBuf, 'f16', [kvLen, numKVHeads, headDim], 'bench_k');
1540
+ const vTensor = createTensor(vBuf, 'f16', [kvLen, numKVHeads, headDim], 'bench_v');
1541
+
1542
+ const outputSize = seqLen * numHeads * headDim * 2;
1543
+ const outputBuffer = device.createBuffer({
1544
+ label: 'bench_attention_output',
1545
+ size: outputSize,
1546
+ usage: GPUBufferUsage.STORAGE,
1547
+ });
1548
+
1549
+ const stats = await benchmark.runBenchmark(
1550
+ async () => {
1551
+ await runAttention(qTensor, kTensor, vTensor, null, numHeads, headDim, {
1552
+ seqLen,
1553
+ kvLen,
1554
+ numKVHeads,
1555
+ scale: 1 / Math.sqrt(headDim),
1556
+ causal: true,
1557
+ outputBuffer,
1558
+ });
1559
+ },
1560
+ {
1561
+ warmupRuns,
1562
+ timedRuns,
1563
+ label: `kv${kvLen}`,
1564
+ }
1565
+ );
1566
+
1567
+ const metrics = computeMetrics(stats, {
1568
+ operation: 'attention',
1569
+ seqLen,
1570
+ kvLen,
1571
+ numHeads,
1572
+ headDim,
1573
+ elementSize: 2,
1574
+ });
1575
+
1576
+ results.push({
1577
+ kvLen,
1578
+ stats: metrics,
1579
+ });
1580
+
1581
+ kBuf.destroy();
1582
+ vBuf.destroy();
1583
+ outputBuffer.destroy();
1584
+ }
1585
+
1586
+ qBuf.destroy();
1587
+ setActiveKernelPath(null, 'none');
1588
+
1589
+ return {
1590
+ kernel,
1591
+ headDim,
1592
+ numHeads,
1593
+ numKVHeads,
1594
+ warmupRuns,
1595
+ timedRuns,
1596
+ results,
1597
+ };
1598
+ },
1599
+
1600
+
1601
+ async runMoEGather(dev, tokens, expertIndices, numTokens, hiddenSize, numExperts, topK) {
1602
+ if (!runMoEGather) {
1603
+ // Fallback to reference if kernel not available
1604
+ const result = references.moeGatherRef(tokens, expertIndices, numTokens, hiddenSize, numExperts, topK);
1605
+ return {
1606
+ gatheredTokens: result.gatheredTokens,
1607
+ tokenCounts: result.tokenCounts,
1608
+ };
1609
+ }
1610
+
1611
+ // Create GPU buffers
1612
+ const tokensBuf = makeBuffer(tokens);
1613
+ const indicesBuf = makeBuffer(expertIndices);
1614
+
1615
+ // Wrap tokensBuf in Tensor (MoE kernels now use Tensor abstraction)
1616
+ const tokensTensor = createTensor(tokensBuf, 'f32', [numTokens, hiddenSize], 'moe_input');
1617
+
1618
+ // Run MoE gather via kernel selector
1619
+ const result = await runMoEGather(tokensTensor, indicesBuf, numTokens, hiddenSize, numExperts, topK);
1620
+
1621
+ // Read back results (result.gathered is now a Tensor)
1622
+ const maxTokensPerExpert = result.maxTokensPerExpert;
1623
+ const gatheredTokens = new Float32Array(await readBufferData(result.gathered.buffer, numExperts * maxTokensPerExpert * hiddenSize * 4));
1624
+ const tokenCounts = new Uint32Array(await readBufferData(result.tokenCounts, numExperts * 4));
1625
+
1626
+ tokensBuf.destroy();
1627
+ indicesBuf.destroy();
1628
+ result.gathered.buffer.destroy();
1629
+ result.tokenCounts.destroy();
1630
+ result.tokenMap.destroy();
1631
+
1632
+ return {
1633
+ gatheredTokens,
1634
+ tokenCounts,
1635
+ };
1636
+ },
1637
+
1638
+
1639
+ async runArgmax(dev, logits) {
1640
+ const logitsBuf = makeBuffer(logits);
1641
+ const tokenId = await sampleKernel.runArgmax(logitsBuf, logits.length, {
1642
+ logitsDtype: 'f32',
1643
+ padTokenId: null,
1644
+ logitSoftcap: 0,
1645
+ outputIndex: 0,
1646
+ });
1647
+ logitsBuf.destroy();
1648
+ return tokenId;
1649
+ },
1650
+
1651
+
1652
+ async runSampleTopK(dev, logits, temperature, topK, randomValue) {
1653
+ const logitsBuf = makeBuffer(logits);
1654
+ const tokenId = await sampleKernel.runGPUSample(logitsBuf, logits.length, {
1655
+ temperature,
1656
+ topK,
1657
+ randomSeed: randomValue * 10000, // Convert to seed
1658
+ logitsDtype: 'f32',
1659
+ padTokenId: null,
1660
+ logitSoftcap: 0,
1661
+ outputIndex: 0,
1662
+ greedyThreshold: 0.01,
1663
+ });
1664
+ logitsBuf.destroy();
1665
+ return tokenId;
1666
+ },
1667
+
1668
+
1669
+ async runSwiGLU(dev, gate, up, gateBias, upBias) {
1670
+ // For testing, we pre-add bias to gate and up, then use SiLU with gating
1671
+ const size = gate.length;
1672
+
1673
+ // Add biases
1674
+ const gateWithBias = new Float32Array(size);
1675
+ const upWithBias = new Float32Array(size);
1676
+ for (let i = 0; i < size; i++) {
1677
+ gateWithBias[i] = gate[i] + gateBias[i];
1678
+ upWithBias[i] = up[i] + upBias[i];
1679
+ }
1680
+
1681
+ if (!runSiLU) {
1682
+ // Fallback to reference implementation
1683
+ const result = new Float32Array(size);
1684
+ for (let i = 0; i < size; i++) {
1685
+ const silu = gateWithBias[i] / (1 + Math.exp(-gateWithBias[i]));
1686
+ result[i] = silu * upWithBias[i];
1687
+ }
1688
+ return result;
1689
+ }
1690
+
1691
+ const gateBuf = makeBuffer(gateWithBias);
1692
+ const upBuf = makeBuffer(upWithBias);
1693
+
1694
+ const gateTensor = createTensor(gateBuf, 'f32', [size], 'swiglu_gate');
1695
+ const upTensor = createTensor(upBuf, 'f32', [size], 'swiglu_up');
1696
+
1697
+ // runSiLU with gate option: output = silu(gate) * up
1698
+ const resultTensor = await runSiLU(upTensor, {
1699
+ size,
1700
+ gate: gateTensor,
1701
+ swigluLimit: null,
1702
+ inputActivation: 'identity',
1703
+ });
1704
+
1705
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, size * 4));
1706
+
1707
+ gateBuf.destroy();
1708
+ upBuf.destroy();
1709
+ resultTensor.buffer.destroy();
1710
+
1711
+ return result;
1712
+ },
1713
+
1714
+
1715
+ async runScale(dev, input, scale) {
1716
+ if (!runScale) {
1717
+ // Fallback to reference implementation
1718
+ const result = new Float32Array(input.length);
1719
+ for (let i = 0; i < input.length; i++) {
1720
+ result[i] = input[i] * scale;
1721
+ }
1722
+ return result;
1723
+ }
1724
+
1725
+ const inputBuf = makeBuffer(input);
1726
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'scale_input');
1727
+
1728
+ const resultTensor = await runScale(inputTensor, scale, { count: input.length });
1729
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, input.length * 4));
1730
+
1731
+ inputBuf.destroy();
1732
+ resultTensor.buffer.destroy();
1733
+
1734
+ return result;
1735
+ },
1736
+
1737
+
1738
+ async runGeLU(dev, input) {
1739
+ if (!runGeLU) {
1740
+ return references.geluRef(input);
1741
+ }
1742
+
1743
+ const inputBuf = makeBuffer(input);
1744
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'gelu_input');
1745
+
1746
+ const resultTensor = await runGeLU(inputTensor, { size: input.length });
1747
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, input.length * 4));
1748
+
1749
+ inputBuf.destroy();
1750
+ resultTensor.buffer.destroy();
1751
+
1752
+ return result;
1753
+ },
1754
+
1755
+
1756
+ async runGeGLU(dev, gate, up) {
1757
+ if (!runGeLU) {
1758
+ return references.gegluRef(gate, up);
1759
+ }
1760
+
1761
+ const gateBuf = makeBuffer(gate);
1762
+ const upBuf = makeBuffer(up);
1763
+
1764
+ const gateTensor = createTensor(gateBuf, 'f32', [gate.length], 'geglu_gate');
1765
+ const upTensor = createTensor(upBuf, 'f32', [up.length], 'geglu_up');
1766
+
1767
+ const resultTensor = await runGeLU(upTensor, { size: up.length, gate: gateTensor });
1768
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, up.length * 4));
1769
+
1770
+ gateBuf.destroy();
1771
+ upBuf.destroy();
1772
+ resultTensor.buffer.destroy();
1773
+
1774
+ return result;
1775
+ },
1776
+
1777
+
1778
+ async runSplitQKV(dev, qkv, numTokens, qSize, kSize, vSize) {
1779
+ if (!runSplitQKV) {
1780
+ return references.splitQkvRef(qkv, numTokens, qSize, kSize, vSize);
1781
+ }
1782
+
1783
+ const qkvBuf = makeBuffer(qkv);
1784
+ const qkvTensor = createTensor(qkvBuf, 'f32', [numTokens, qSize + kSize + vSize], 'split_qkv_input');
1785
+
1786
+ const { Q: qTensor, K: kTensor, V: vTensor } = await runSplitQKV(qkvTensor, {
1787
+ numTokens,
1788
+ qSize,
1789
+ kSize,
1790
+ vSize,
1791
+ });
1792
+
1793
+ const Q = new Float32Array(await readBufferData(qTensor.buffer, numTokens * qSize * 4));
1794
+ const K = new Float32Array(await readBufferData(kTensor.buffer, numTokens * kSize * 4));
1795
+ const V = new Float32Array(await readBufferData(vTensor.buffer, numTokens * vSize * 4));
1796
+
1797
+ qkvBuf.destroy();
1798
+ qTensor.buffer.destroy();
1799
+ kTensor.buffer.destroy();
1800
+ vTensor.buffer.destroy();
1801
+
1802
+ return { Q, K, V };
1803
+ },
1804
+
1805
+
1806
+ async runF16ToF32(dev, input) {
1807
+ if (!castF16ToF32) {
1808
+ const out = new Float32Array(input.length);
1809
+ for (let i = 0; i < input.length; i++) {
1810
+ out[i] = f16ToF32(input[i]);
1811
+ }
1812
+ return out;
1813
+ }
1814
+
1815
+ const inputBuf = makeBuffer(input);
1816
+ const inputTensor = createTensor(inputBuf, 'f16', [input.length], 'f16_to_f32_input');
1817
+ const outTensor = await castF16ToF32(inputTensor);
1818
+ const out = new Float32Array(await readBufferData(outTensor.buffer, input.length * 4));
1819
+
1820
+ inputBuf.destroy();
1821
+ outTensor.buffer.destroy();
1822
+
1823
+ return out;
1824
+ },
1825
+
1826
+
1827
+ async runSoftmaxBackward(dev, softmax, gradOutput, rows, cols) {
1828
+ if (!runSoftmaxBackward) {
1829
+ const output = new Float32Array(rows * cols);
1830
+ for (let r = 0; r < rows; r++) {
1831
+ const base = r * cols;
1832
+ let sum = 0;
1833
+ for (let c = 0; c < cols; c++) {
1834
+ sum += softmax[base + c] * gradOutput[base + c];
1835
+ }
1836
+ for (let c = 0; c < cols; c++) {
1837
+ const idx = base + c;
1838
+ output[idx] = softmax[idx] * (gradOutput[idx] - sum);
1839
+ }
1840
+ }
1841
+ return output;
1842
+ }
1843
+
1844
+ const softmaxBuf = makeBuffer(softmax);
1845
+ const gradBuf = makeBuffer(gradOutput);
1846
+ const softmaxTensor = createTensor(softmaxBuf, 'f32', [rows, cols], 'softmax_backward_input');
1847
+ const gradTensor = createTensor(gradBuf, 'f32', [rows, cols], 'softmax_backward_grad');
1848
+ const resultTensor = await runSoftmaxBackward(softmaxTensor, gradTensor, { rows, cols });
1849
+ const result = await readTensorToFloat32(resultTensor, rows * cols);
1850
+
1851
+ softmaxBuf.destroy();
1852
+ gradBuf.destroy();
1853
+ resultTensor.buffer.destroy();
1854
+
1855
+ return result;
1856
+ },
1857
+
1858
+
1859
+ async runSiluBackward(dev, input, gradOutput) {
1860
+ if (!runSiluBackward) {
1861
+ const output = new Float32Array(input.length);
1862
+ for (let i = 0; i < input.length; i++) {
1863
+ const x = input[i];
1864
+ const sigmoid = 1 / (1 + Math.exp(-x));
1865
+ const deriv = sigmoid * (1 + x * (1 - sigmoid));
1866
+ output[i] = gradOutput[i] * deriv;
1867
+ }
1868
+ return output;
1869
+ }
1870
+
1871
+ const inputBuf = makeBuffer(input);
1872
+ const gradBuf = makeBuffer(gradOutput);
1873
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'silu_backward_input');
1874
+ const gradTensor = createTensor(gradBuf, 'f32', [input.length], 'silu_backward_grad');
1875
+ const resultTensor = await runSiluBackward(inputTensor, gradTensor, { count: input.length });
1876
+ const result = await readTensorToFloat32(resultTensor, input.length);
1877
+
1878
+ inputBuf.destroy();
1879
+ gradBuf.destroy();
1880
+ resultTensor.buffer.destroy();
1881
+
1882
+ return result;
1883
+ },
1884
+
1885
+
1886
+ async runGeluBackward(dev, input, gradOutput) {
1887
+ if (!runGeluBackward) {
1888
+ const output = new Float32Array(input.length);
1889
+ for (let i = 0; i < input.length; i++) {
1890
+ const x = input[i];
1891
+ const sqrt2pi = 0.7978845608;
1892
+ const c = 0.044715;
1893
+ const x3 = x * x * x;
1894
+ const inner = sqrt2pi * (x + c * x3);
1895
+ const innerClamped = Math.max(-15, Math.min(15, inner));
1896
+ const tanhInner = Math.tanh(innerClamped);
1897
+ const sech2 = 1 - tanhInner * tanhInner;
1898
+ const innerDeriv = sqrt2pi * (1 + 3 * c * x * x);
1899
+ const deriv = 0.5 * (1 + tanhInner) + 0.5 * x * sech2 * innerDeriv;
1900
+ output[i] = gradOutput[i] * deriv;
1901
+ }
1902
+ return output;
1903
+ }
1904
+
1905
+ const inputBuf = makeBuffer(input);
1906
+ const gradBuf = makeBuffer(gradOutput);
1907
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'gelu_backward_input');
1908
+ const gradTensor = createTensor(gradBuf, 'f32', [input.length], 'gelu_backward_grad');
1909
+ const resultTensor = await runGeluBackward(inputTensor, gradTensor, { count: input.length });
1910
+ const result = await readTensorToFloat32(resultTensor, input.length);
1911
+
1912
+ inputBuf.destroy();
1913
+ gradBuf.destroy();
1914
+ resultTensor.buffer.destroy();
1915
+
1916
+ return result;
1917
+ },
1918
+
1919
+
1920
+ async runScaleBackward(dev, input, gradOutput, scale) {
1921
+ if (!runScaleBackward) {
1922
+ const output = new Float32Array(input.length);
1923
+ for (let i = 0; i < input.length; i++) {
1924
+ output[i] = gradOutput[i] * scale;
1925
+ }
1926
+ return output;
1927
+ }
1928
+
1929
+ const gradBuf = makeBuffer(gradOutput);
1930
+ const gradTensor = createTensor(gradBuf, 'f32', [input.length], 'scale_backward_grad');
1931
+ const resultTensor = await runScaleBackward(gradTensor, gradTensor, { count: input.length, scale });
1932
+ const result = await readTensorToFloat32(resultTensor, input.length);
1933
+
1934
+ gradBuf.destroy();
1935
+ resultTensor.buffer.destroy();
1936
+
1937
+ return result;
1938
+ },
1939
+
1940
+
1941
+ async runRoPEBackward(dev, gradOutput, cos, sin, seqLen, numHeads, headDim, startPos = 0) {
1942
+ if (!runRoPEBackward) {
1943
+ const output = new Float32Array(gradOutput.length);
1944
+ const halfDim = headDim / 2;
1945
+ for (let pos = 0; pos < seqLen; pos++) {
1946
+ for (let h = 0; h < numHeads; h++) {
1947
+ const base = (pos * numHeads + h) * headDim;
1948
+ const freqBase = (startPos + pos) * halfDim;
1949
+ for (let i = 0; i < halfDim; i++) {
1950
+ const dy0 = gradOutput[base + i];
1951
+ const dy1 = gradOutput[base + i + halfDim];
1952
+ const c = cos[freqBase + i];
1953
+ const s = sin[freqBase + i];
1954
+ output[base + i] = dy0 * c + dy1 * s;
1955
+ output[base + i + halfDim] = -dy0 * s + dy1 * c;
1956
+ }
1957
+ }
1958
+ }
1959
+ return output;
1960
+ }
1961
+
1962
+ const gradBuf = makeBuffer(gradOutput);
1963
+ const cosBuf = makeBuffer(cos);
1964
+ const sinBuf = makeBuffer(sin);
1965
+ const gradTensor = createTensor(gradBuf, 'f32', [seqLen, numHeads, headDim], 'rope_backward_grad');
1966
+ const cosTensor = createTensor(cosBuf, 'f32', [cos.length], 'rope_cos');
1967
+ const sinTensor = createTensor(sinBuf, 'f32', [sin.length], 'rope_sin');
1968
+ const resultTensor = await runRoPEBackward(gradTensor, cosTensor, sinTensor, {
1969
+ seqLen,
1970
+ numHeads,
1971
+ headDim,
1972
+ startPos,
1973
+ });
1974
+ const result = await readTensorToFloat32(resultTensor, seqLen * numHeads * headDim);
1975
+
1976
+ gradBuf.destroy();
1977
+ cosBuf.destroy();
1978
+ sinBuf.destroy();
1979
+ resultTensor.buffer.destroy();
1980
+
1981
+ return result;
1982
+ },
1983
+
1984
+
1985
+ async runRmsNormBackward(dev, input, weight, gradOutput, numTokens, hiddenSize, eps = 1e-6) {
1986
+ if (!runRmsNormBackward) {
1987
+ const output = new Float32Array(numTokens * hiddenSize);
1988
+ for (let t = 0; t < numTokens; t++) {
1989
+ const base = t * hiddenSize;
1990
+ let sumSq = 0;
1991
+ let sumGX = 0;
1992
+ for (let i = 0; i < hiddenSize; i++) {
1993
+ const x = input[base + i];
1994
+ const g = gradOutput[base + i] * weight[i];
1995
+ sumSq += x * x;
1996
+ sumGX += g * x;
1997
+ }
1998
+ const invRms = 1 / Math.sqrt(sumSq / hiddenSize + eps);
1999
+ const invRms3 = invRms * invRms * invRms;
2000
+ const coeff = (sumGX / hiddenSize) * invRms3;
2001
+ for (let i = 0; i < hiddenSize; i++) {
2002
+ const x = input[base + i];
2003
+ const g = gradOutput[base + i] * weight[i];
2004
+ output[base + i] = g * invRms - x * coeff;
2005
+ }
2006
+ }
2007
+ return output;
2008
+ }
2009
+
2010
+ const inputBuf = makeBuffer(input);
2011
+ const weightBuf = makeBuffer(weight);
2012
+ const gradBuf = makeBuffer(gradOutput);
2013
+ const inputTensor = createTensor(inputBuf, 'f32', [numTokens, hiddenSize], 'rmsnorm_backward_input');
2014
+ const weightTensor = createTensor(weightBuf, 'f32', [hiddenSize], 'rmsnorm_backward_weight');
2015
+ const gradTensor = createTensor(gradBuf, 'f32', [numTokens, hiddenSize], 'rmsnorm_backward_grad');
2016
+ const resultTensor = await runRmsNormBackward(inputTensor, weightTensor, gradTensor, { numTokens, hiddenSize, eps });
2017
+ const result = await readTensorToFloat32(resultTensor, numTokens * hiddenSize);
2018
+
2019
+ inputBuf.destroy();
2020
+ weightBuf.destroy();
2021
+ gradBuf.destroy();
2022
+ resultTensor.buffer.destroy();
2023
+
2024
+ return result;
2025
+ },
2026
+
2027
+
2028
+ async runEmbedBackward(dev, input, gradOutput, vocabSize = null) {
2029
+ if (!runEmbedBackward) {
2030
+ const fallbackVocabSize = Number.isFinite(vocabSize) && vocabSize > 0
2031
+ ? Math.max(1, Math.floor(vocabSize))
2032
+ : Math.max(1, Math.max(...input, 0) + 1);
2033
+ const numTokens = input.length;
2034
+ const hiddenSize = gradOutput.length / numTokens;
2035
+ const output = new Float32Array(fallbackVocabSize * hiddenSize);
2036
+ for (let i = 0; i < numTokens; i += 1) {
2037
+ const token = input[i];
2038
+ const outputBase = token * hiddenSize;
2039
+ const gradBase = i * hiddenSize;
2040
+ for (let d = 0; d < hiddenSize; d += 1) {
2041
+ output[outputBase + d] += gradOutput[gradBase + d];
2042
+ }
2043
+ }
2044
+ return output;
2045
+ }
2046
+
2047
+ const inputBuf = makeBuffer(input);
2048
+ const gradBuf = makeBuffer(gradOutput);
2049
+ const numTokens = input.length;
2050
+ const inputTensor = createTensor(inputBuf, 'u32', [input.length], 'embed_backward_input');
2051
+ const gradTensor = createTensor(gradBuf, 'f32', [gradOutput.length], 'embed_backward_grad');
2052
+ const maxInput = input.length > 0 ? Math.max(...input, 0) : 0;
2053
+ const hiddenSize = Math.max(1, gradOutput.length / numTokens);
2054
+ const resolvedVocabSize = Math.max(1, Math.floor(vocabSize ?? Math.ceil(maxInput + 1)));
2055
+ const resultTensor = await runEmbedBackward(inputTensor, gradTensor, {
2056
+ numTokens,
2057
+ hiddenSize,
2058
+ vocabSize: resolvedVocabSize,
2059
+ transpose: false,
2060
+ indexOffset: 0,
2061
+ });
2062
+ const result = await readTensorToFloat32(resultTensor, resolvedVocabSize * hiddenSize);
2063
+
2064
+ inputBuf.destroy();
2065
+ gradBuf.destroy();
2066
+ resultTensor.buffer.destroy();
2067
+
2068
+ return result;
2069
+ },
2070
+
2071
+
2072
+ async runAttentionBackward(dev, q, k, v, softmax, gradOutput, seqLen, numHeads, headDim, scale = 1.0) {
2073
+ if (!runAttentionBackward) {
2074
+ return null;
2075
+ }
2076
+
2077
+ const elementsPerHead = seqLen * headDim;
2078
+ const totalElements = seqLen * numHeads * headDim;
2079
+
2080
+ const packByHead = (source) => {
2081
+ const packed = new Float32Array(totalElements);
2082
+ for (let h = 0; h < numHeads; h++) {
2083
+ const headBase = h * elementsPerHead;
2084
+ for (let t = 0; t < seqLen; t++) {
2085
+ const sourceBase = (t * numHeads + h) * headDim;
2086
+ const packedBase = headBase + t * headDim;
2087
+ for (let d = 0; d < headDim; d++) {
2088
+ packed[packedBase + d] = source[sourceBase + d];
2089
+ }
2090
+ }
2091
+ }
2092
+ return packed;
2093
+ };
2094
+
2095
+ const unpackByHead = (source) => {
2096
+ const unpacked = new Float32Array(totalElements);
2097
+ for (let h = 0; h < numHeads; h++) {
2098
+ const headBase = h * elementsPerHead;
2099
+ for (let t = 0; t < seqLen; t++) {
2100
+ const targetBase = (t * numHeads + h) * headDim;
2101
+ const sourceBase = headBase + t * headDim;
2102
+ for (let d = 0; d < headDim; d++) {
2103
+ unpacked[targetBase + d] = source[sourceBase + d];
2104
+ }
2105
+ }
2106
+ }
2107
+ return unpacked;
2108
+ };
2109
+
2110
+ const qPacked = packByHead(q);
2111
+ const kPacked = packByHead(k);
2112
+ const vPacked = packByHead(v);
2113
+ const gPacked = packByHead(gradOutput);
2114
+
2115
+ const qBuf = makeBuffer(qPacked);
2116
+ const kBuf = makeBuffer(kPacked);
2117
+ const vBuf = makeBuffer(vPacked);
2118
+ const sBuf = makeBuffer(softmax);
2119
+ const gBuf = makeBuffer(gPacked);
2120
+ const qTensor = createTensor(qBuf, 'f32', [numHeads, seqLen, headDim], 'attn_bw_q');
2121
+ const kTensor = createTensor(kBuf, 'f32', [numHeads, seqLen, headDim], 'attn_bw_k');
2122
+ const vTensor = createTensor(vBuf, 'f32', [numHeads, seqLen, headDim], 'attn_bw_v');
2123
+ const sTensor = createTensor(sBuf, 'f32', [numHeads, seqLen, seqLen], 'attn_bw_softmax');
2124
+ const gTensor = createTensor(gBuf, 'f32', [numHeads, seqLen, headDim], 'attn_bw_grad');
2125
+
2126
+ const result = await runAttentionBackward(qTensor, kTensor, vTensor, sTensor, gTensor, {
2127
+ seqLen,
2128
+ numHeads,
2129
+ headDim,
2130
+ scale,
2131
+ causal: false,
2132
+ });
2133
+
2134
+ const gradQPacked = await readTensorToFloat32(result.gradQ, seqLen * numHeads * headDim);
2135
+ const gradKPacked = await readTensorToFloat32(result.gradK, seqLen * numHeads * headDim);
2136
+ const gradVPacked = await readTensorToFloat32(result.gradV, seqLen * numHeads * headDim);
2137
+ const gradQ = unpackByHead(gradQPacked);
2138
+ const gradK = unpackByHead(gradKPacked);
2139
+ const gradV = unpackByHead(gradVPacked);
2140
+
2141
+ qBuf.destroy();
2142
+ kBuf.destroy();
2143
+ vBuf.destroy();
2144
+ sBuf.destroy();
2145
+ gBuf.destroy();
2146
+ result.gradQ.buffer.destroy();
2147
+ result.gradK.buffer.destroy();
2148
+ result.gradV.buffer.destroy();
2149
+
2150
+ return { gradQ, gradK, gradV };
2151
+ },
2152
+
2153
+
2154
+ async runAdam(dev, params, grads, moment1, moment2, options) {
2155
+ if (!runAdam) {
2156
+ const outParams = new Float32Array(params);
2157
+ const outM1 = new Float32Array(moment1);
2158
+ const outM2 = new Float32Array(moment2);
2159
+ const { count, step, lr, beta1, beta2, eps } = options;
2160
+ for (let i = 0; i < count; i++) {
2161
+ const g = grads[i];
2162
+ outM1[i] = beta1 * outM1[i] + (1 - beta1) * g;
2163
+ outM2[i] = beta2 * outM2[i] + (1 - beta2) * g * g;
2164
+ const mHat = outM1[i] / (1 - Math.pow(beta1, step));
2165
+ const vHat = outM2[i] / (1 - Math.pow(beta2, step));
2166
+ outParams[i] = outParams[i] - lr * mHat / (Math.sqrt(vHat) + eps);
2167
+ }
2168
+ return { params: outParams, moment1: outM1, moment2: outM2 };
2169
+ }
2170
+
2171
+ const paramBuf = makeBuffer(params);
2172
+ const gradBuf = makeBuffer(grads);
2173
+ const m1Buf = makeBuffer(moment1);
2174
+ const m2Buf = makeBuffer(moment2);
2175
+ const paramTensor = createTensor(paramBuf, 'f32', [params.length], 'adam_params');
2176
+ const gradTensor = createTensor(gradBuf, 'f32', [grads.length], 'adam_grads');
2177
+ const m1Tensor = createTensor(m1Buf, 'f32', [moment1.length], 'adam_m1');
2178
+ const m2Tensor = createTensor(m2Buf, 'f32', [moment2.length], 'adam_m2');
2179
+
2180
+ const resultTensor = await runAdam(paramTensor, gradTensor, m1Tensor, m2Tensor, options);
2181
+ const outParams = await readTensorToFloat32(resultTensor, params.length);
2182
+ const outM1 = new Float32Array(await readBufferData(m1Buf, moment1.length * 4));
2183
+ const outM2 = new Float32Array(await readBufferData(m2Buf, moment2.length * 4));
2184
+
2185
+ paramBuf.destroy();
2186
+ gradBuf.destroy();
2187
+ m1Buf.destroy();
2188
+ m2Buf.destroy();
2189
+ resultTensor.buffer.destroy();
2190
+
2191
+ return { params: outParams, moment1: outM1, moment2: outM2 };
2192
+ },
2193
+
2194
+
2195
+ async runBF16ToF32(dev, input) {
2196
+ if (!runBF16ToF32) {
2197
+ const out = new Float32Array(input.length);
2198
+ for (let i = 0; i < input.length; i++) {
2199
+ const view = new DataView(new ArrayBuffer(4));
2200
+ view.setUint32(0, input[i] << 16, true);
2201
+ out[i] = view.getFloat32(0, true);
2202
+ }
2203
+ return out;
2204
+ }
2205
+
2206
+ const inputBuf = makeBuffer(input, GPUBufferUsage.STORAGE);
2207
+ const outTensor = await runBF16ToF32(inputBuf, [input.length], 'bf16_to_f32_test');
2208
+ const out = new Float32Array(await readBufferData(outTensor.buffer, input.length * 4));
2209
+
2210
+ inputBuf.destroy();
2211
+ outTensor.buffer.destroy();
2212
+
2213
+ return out;
2214
+ },
2215
+
2216
+
2217
+ async runF32ToF16(dev, input) {
2218
+ if (!castF32ToF16) {
2219
+ const out = new Uint16Array(input.length);
2220
+ for (let i = 0; i < input.length; i++) {
2221
+ const view = new DataView(new ArrayBuffer(4));
2222
+ view.setFloat32(0, input[i], true);
2223
+ const bits = view.getUint32(0, true);
2224
+ const sign = (bits >> 31) & 0x1;
2225
+ const exp = (bits >> 23) & 0xff;
2226
+ const mant = bits & 0x7fffff;
2227
+
2228
+ let hExp = 0;
2229
+ let hMant = 0;
2230
+ if (exp === 0xff) {
2231
+ hExp = 0x1f;
2232
+ hMant = mant ? 0x200 : 0;
2233
+ } else if (exp !== 0) {
2234
+ const newExp = exp - 127 + 15;
2235
+ if (newExp >= 0x1f) {
2236
+ hExp = 0x1f;
2237
+ } else if (newExp > 0) {
2238
+ hExp = newExp;
2239
+ hMant = mant >> 13;
2240
+ }
2241
+ }
2242
+ out[i] = (sign << 15) | (hExp << 10) | hMant;
2243
+ }
2244
+ return out;
2245
+ }
2246
+
2247
+ const inputBuf = makeBuffer(input);
2248
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'f32_to_f16_input');
2249
+ const outTensor = await castF32ToF16(inputTensor);
2250
+ const out = new Uint16Array(await readBufferData(outTensor.buffer, input.length * 2));
2251
+
2252
+ inputBuf.destroy();
2253
+ outTensor.buffer.destroy();
2254
+
2255
+ return out;
2256
+ },
2257
+
2258
+
2259
+ async runBF16ToF16(dev, input) {
2260
+ if (!runBF16ToF16) {
2261
+ const out = new Uint16Array(input.length);
2262
+ for (let i = 0; i < input.length; i++) {
2263
+ const view = new DataView(new ArrayBuffer(4));
2264
+ view.setUint32(0, input[i] << 16, true);
2265
+ const bits = view.getUint32(0, true);
2266
+ const sign = (bits >> 31) & 0x1;
2267
+ const exp = (bits >> 23) & 0xff;
2268
+ const mant = bits & 0x7fffff;
2269
+
2270
+ let hExp = 0;
2271
+ let hMant = 0;
2272
+ if (exp === 0xff) {
2273
+ hExp = 0x1f;
2274
+ hMant = mant ? 0x200 : 0;
2275
+ } else if (exp !== 0) {
2276
+ const newExp = exp - 127 + 15;
2277
+ if (newExp >= 0x1f) {
2278
+ hExp = 0x1f;
2279
+ } else if (newExp > 0) {
2280
+ hExp = newExp;
2281
+ hMant = mant >> 13;
2282
+ }
2283
+ }
2284
+ out[i] = (sign << 15) | (hExp << 10) | hMant;
2285
+ }
2286
+ return out;
2287
+ }
2288
+
2289
+ const inputBuf = makeBuffer(input, GPUBufferUsage.STORAGE);
2290
+ const outTensor = await runBF16ToF16(inputBuf, [input.length], 'bf16_to_f16_test');
2291
+ const out = new Uint16Array(await readBufferData(outTensor.buffer, input.length * 2));
2292
+
2293
+ inputBuf.destroy();
2294
+ outTensor.buffer.destroy();
2295
+
2296
+ return out;
2297
+ },
2298
+
2299
+
2300
+ async runDequantQ6K(dev, quantized, numBlocks) {
2301
+ if (!dequantizeQ6K) {
2302
+ throw new Error('dequantizeQ6K kernel not available');
2303
+ }
2304
+
2305
+ const blockSize = 256; // Q6_K: 256 elements per block
2306
+ const quantizedBytes = quantized instanceof Uint8Array
2307
+ ? quantized
2308
+ : new Uint8Array(quantized.buffer ?? quantized);
2309
+ const alignedBytes = quantizedBytes.byteLength % 4 === 0
2310
+ ? quantizedBytes
2311
+ : (() => {
2312
+ const padded = new Uint8Array(Math.ceil(quantizedBytes.byteLength / 4) * 4);
2313
+ padded.set(quantizedBytes);
2314
+ return padded;
2315
+ })();
2316
+ const qBuf = makeBuffer(alignedBytes, GPUBufferUsage.STORAGE);
2317
+ const outTensor = await dequantizeQ6K(qBuf, numBlocks, { outputOffset: 0, outputDtype: 'f16' });
2318
+
2319
+ const outBuf = outTensor.buffer;
2320
+ // Q6K outputs f16 - read raw bytes and convert
2321
+ const rawData = await readBufferData(outBuf, numBlocks * blockSize * 2); // f16 = 2 bytes
2322
+ const u16View = new Uint16Array(rawData);
2323
+ const out = new Float32Array(u16View.length);
2324
+
2325
+ // Convert f16 to f32
2326
+ for (let i = 0; i < u16View.length; i++) {
2327
+ out[i] = f16ToF32(u16View[i]);
2328
+ }
2329
+
2330
+ qBuf.destroy();
2331
+ outBuf.destroy();
2332
+
2333
+ return out;
2334
+ },
2335
+
2336
+
2337
+ async runMatmulF16W(dev, A, B_f16, M, N, K) {
2338
+ if (!runMatmul) {
2339
+ throw new Error('runMatmul kernel not available');
2340
+ }
2341
+
2342
+ // Create A buffer (F32 activations)
2343
+ const bufA = makeBuffer(A);
2344
+ const tensorA = createTensor(bufA, 'f32', [M, K], 'matmul_f16w_a');
2345
+
2346
+ // Create B buffer (F16 weights) - pass raw Uint16Array
2347
+ const bufB = makeBuffer(B_f16, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC);
2348
+
2349
+ // Run matmul with bDtype='f16' and preferF16=true to trigger f16w_f32a kernel
2350
+ // transposeB=true because B is [N, K] format (weight matrix layout)
2351
+ const resultTensor = await runMatmul(tensorA, bufB, M, N, K, {
2352
+ bDtype: 'f16',
2353
+ preferF16: true,
2354
+ transposeB: true,
2355
+ outputDtype: 'f32',
2356
+ });
2357
+
2358
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
2359
+
2360
+ bufA.destroy();
2361
+ bufB.destroy();
2362
+ resultTensor.buffer.destroy();
2363
+
2364
+ return result;
2365
+ },
2366
+
2367
+
2368
+ async runDequantAndMatmulF16W(dev, A, B_q4k, M, N, K, numBlocks) {
2369
+ if (!runMatmul || !dequantize) {
2370
+ throw new Error('runMatmul or dequantize kernel not available');
2371
+ }
2372
+
2373
+ // Dequant Q4K -> F16 on GPU
2374
+ const qBuf = makeBuffer(B_q4k, GPUBufferUsage.STORAGE);
2375
+ const dequantTensor = await dequantize(qBuf, numBlocks, { outputDtype: 'f16', useVec4: true });
2376
+
2377
+ // Create A buffer (F32 activations)
2378
+ const bufA = makeBuffer(A);
2379
+ const tensorA = createTensor(bufA, 'f32', [M, K], 'dequant_matmul_a');
2380
+
2381
+ // Run matmul with F16 weights (stays on GPU, no CPU round-trip)
2382
+ // transposeB=true because dequanted weights are [N, K] format
2383
+ const resultTensor = await runMatmul(tensorA, dequantTensor.buffer, M, N, K, {
2384
+ bDtype: 'f16',
2385
+ preferF16: true,
2386
+ transposeB: true,
2387
+ outputDtype: 'f32',
2388
+ });
2389
+
2390
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
2391
+
2392
+ qBuf.destroy();
2393
+ bufA.destroy();
2394
+ dequantTensor.buffer.destroy();
2395
+ resultTensor.buffer.destroy();
2396
+
2397
+ return result;
2398
+ },
2399
+
2400
+
2401
+ async runCheckStop(dev, sampledToken, eosTokenId, maxTokens, currentPos) {
2402
+ // Create buffer for sampled token
2403
+ const tokenBuffer = makeBuffer(new Uint32Array([sampledToken]));
2404
+
2405
+ const shouldStop = await checkStop({
2406
+ sampledTokenBuffer: tokenBuffer,
2407
+ eosTokenId,
2408
+ maxTokens,
2409
+ currentPos,
2410
+ });
2411
+
2412
+ tokenBuffer.destroy();
2413
+
2414
+ return shouldStop;
2415
+ },
2416
+
2417
+
2418
+ checkStopRef(sampledToken, eosTokenId, maxTokens, currentPos) {
2419
+ const isEOS = sampledToken === eosTokenId;
2420
+ const reachedMax = currentPos >= maxTokens;
2421
+ return isEOS || reachedMax;
2422
+ },
2423
+
2424
+
2425
+ async runFusedMatmulResidual(dev, input, weight, residual, N, K, alpha = 1.0) {
2426
+ const inputBuf = makeBuffer(input);
2427
+ const weightF16 = toF16Array(weight);
2428
+ const weightBuf = makeBuffer(weightF16);
2429
+ const residualBuf = makeBuffer(residual);
2430
+
2431
+ const inputTensor = createTensor(inputBuf, 'f32', [1, K], 'fused_matmul_res_input');
2432
+ const residualTensor = createTensor(residualBuf, 'f32', [1, N], 'fused_matmul_res_residual');
2433
+
2434
+ const resultTensor = await runMatmulResidualFused(inputTensor, weightBuf, residualTensor, {
2435
+ N,
2436
+ K,
2437
+ alpha,
2438
+ });
2439
+
2440
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, N * 4));
2441
+
2442
+ inputBuf.destroy();
2443
+ weightBuf.destroy();
2444
+ residualBuf.destroy();
2445
+ resultTensor.buffer.destroy();
2446
+
2447
+ return result;
2448
+ },
2449
+
2450
+
2451
+ fusedMatmulResidualRef(input, weight, residual, N, K, alpha = 1.0) {
2452
+ // input: [1, K], weight: [N, K] (row-major, transposeB=true), residual: [1, N]
2453
+ const weightF16 = toF16RoundedFloat32(weight);
2454
+ const output = new Float32Array(N);
2455
+ for (let n = 0; n < N; n++) {
2456
+ let sum = 0;
2457
+ for (let k = 0; k < K; k++) {
2458
+ sum += input[k] * weightF16[n * K + k];
2459
+ }
2460
+ output[n] = sum * alpha + residual[n];
2461
+ }
2462
+ return output;
2463
+ },
2464
+
2465
+
2466
+ async runFusedMatmulRMSNorm(dev, input, weight, normWeight, N, K, eps = 1e-5, residual = null) {
2467
+ const inputBuf = makeBuffer(input);
2468
+ const weightBuf = makeBuffer(weight);
2469
+ const normWeightBuf = makeBuffer(normWeight);
2470
+ const residualBuf = residual ? makeBuffer(residual) : null;
2471
+
2472
+ const inputTensor = createTensor(inputBuf, 'f32', [1, K], 'fused_matmul_rmsnorm_input');
2473
+
2474
+ const resultTensor = await runMatmulRMSNormFused(inputTensor, weightBuf, normWeightBuf, {
2475
+ N,
2476
+ K,
2477
+ eps,
2478
+ residual: residualBuf,
2479
+ transposeB: true,
2480
+ });
2481
+
2482
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, N * 4));
2483
+
2484
+ inputBuf.destroy();
2485
+ weightBuf.destroy();
2486
+ normWeightBuf.destroy();
2487
+ if (residualBuf) residualBuf.destroy();
2488
+ resultTensor.buffer.destroy();
2489
+
2490
+ return result;
2491
+ },
2492
+
2493
+
2494
+ fusedMatmulRMSNormRef(input, weight, normWeight, N, K, eps = 1e-5, residual = null) {
2495
+ // Step 1: matmul input[1,K] @ weight[N,K]^T -> intermediate[1,N]
2496
+ const intermediate = new Float32Array(N);
2497
+ for (let n = 0; n < N; n++) {
2498
+ let sum = 0;
2499
+ for (let k = 0; k < K; k++) {
2500
+ sum += input[k] * weight[n * K + k];
2501
+ }
2502
+ intermediate[n] = sum;
2503
+ }
2504
+
2505
+ // Step 2: RMSNorm
2506
+ let sumSq = 0;
2507
+ for (let i = 0; i < N; i++) {
2508
+ sumSq += intermediate[i] * intermediate[i];
2509
+ }
2510
+ const rms = Math.sqrt(sumSq / N + eps);
2511
+
2512
+ const output = new Float32Array(N);
2513
+ for (let i = 0; i < N; i++) {
2514
+ const normalized = (intermediate[i] / rms) * normWeight[i];
2515
+ output[i] = residual ? normalized + residual[i] : normalized;
2516
+ }
2517
+
2518
+ return output;
2519
+ },
2520
+
2521
+
2522
+ async runFusedFFN(dev, input, W_gate, W_up, hiddenSize, intermediateSize, activation = 'silu') {
2523
+ const inputBuf = makeBuffer(input);
2524
+ const gateBuf = makeBuffer(W_gate);
2525
+ const upBuf = makeBuffer(W_up);
2526
+ const gateWeight = createWeightBuffer(gateBuf, 'f32', 'row', [intermediateSize, hiddenSize], 'fused_ffn_gate');
2527
+ const upWeight = createWeightBuffer(upBuf, 'f32', 'row', [intermediateSize, hiddenSize], 'fused_ffn_up');
2528
+
2529
+ const inputTensor = createTensor(inputBuf, 'f32', [1, hiddenSize], 'fused_ffn_input');
2530
+
2531
+ const resultTensor = await runFusedFFN(inputTensor, gateWeight, upWeight, hiddenSize, intermediateSize, {
2532
+ batchSize: 1,
2533
+ activation,
2534
+ alpha: 1.0,
2535
+ swigluLimit: null,
2536
+ });
2537
+
2538
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, intermediateSize * 4));
2539
+
2540
+ inputBuf.destroy();
2541
+ gateBuf.destroy();
2542
+ upBuf.destroy();
2543
+ resultTensor.buffer.destroy();
2544
+
2545
+ return result;
2546
+ },
2547
+
2548
+
2549
+ fusedFFNRef(input, W_gate, W_up, hiddenSize, intermediateSize, activation = 'silu') {
2550
+ // Step 1: gate = input @ W_gate^T (input[1,K] @ W_gate[N,K]^T -> gate[1,N])
2551
+ const gate = new Float32Array(intermediateSize);
2552
+ for (let n = 0; n < intermediateSize; n++) {
2553
+ let sum = 0;
2554
+ for (let k = 0; k < hiddenSize; k++) {
2555
+ sum += input[k] * W_gate[n * hiddenSize + k];
2556
+ }
2557
+ gate[n] = sum;
2558
+ }
2559
+
2560
+ // Step 2: up = input @ W_up^T
2561
+ const up = new Float32Array(intermediateSize);
2562
+ for (let n = 0; n < intermediateSize; n++) {
2563
+ let sum = 0;
2564
+ for (let k = 0; k < hiddenSize; k++) {
2565
+ sum += input[k] * W_up[n * hiddenSize + k];
2566
+ }
2567
+ up[n] = sum;
2568
+ }
2569
+
2570
+ // Step 3: Apply activation to gate and multiply by up
2571
+ const output = new Float32Array(intermediateSize);
2572
+ for (let i = 0; i < intermediateSize; i++) {
2573
+ let activated;
2574
+ if (activation === 'silu') {
2575
+ // SiLU(x) = x * sigmoid(x) = x / (1 + exp(-x))
2576
+ activated = gate[i] / (1 + Math.exp(-gate[i]));
2577
+ } else {
2578
+ // GELU(x) = x * 0.5 * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))
2579
+ const x = gate[i];
2580
+ const sqrt2pi = Math.sqrt(2 / Math.PI);
2581
+ const cdf = 0.5 * (1 + Math.tanh(sqrt2pi * (x + 0.044715 * x * x * x)));
2582
+ activated = x * cdf;
2583
+ }
2584
+ output[i] = activated * up[i];
2585
+ }
2586
+
2587
+ return output;
2588
+ },
2589
+ };
2590
+
2591
+ // Expose to window for browser automation
2592
+ window.testHarness = testHarness;
2593
+ window.gpuReady = false;
2594
+ window.gpuError = undefined;
2595
+
2596
+ // Auto-initialize on load
2597
+ window.addEventListener('DOMContentLoaded', async () => {
2598
+ try {
2599
+ await initGPU();
2600
+ console.log('WebGPU initialized successfully');
2601
+ window.gpuReady = true;
2602
+
2603
+ // Display status
2604
+ const status = document.getElementById('status');
2605
+ if (status) {
2606
+ const caps = getKernelCapabilities();
2607
+ status.innerHTML = `
2608
+ <strong>WebGPU Ready</strong><br>
2609
+ Adapter: ${caps?.adapterInfo || 'Unknown'}<br>
2610
+ F16 Support: ${caps?.hasF16 ? 'Yes' : 'No'}<br>
2611
+ Subgroups: ${caps?.hasSubgroups ? 'Yes' : 'No'}
2612
+ `;
2613
+ status.style.color = 'green';
2614
+ }
2615
+ } catch (e) {
2616
+ console.error('Failed to initialize WebGPU:', e);
2617
+ window.gpuReady = false;
2618
+ window.gpuError = e.message;
2619
+
2620
+ const status = document.getElementById('status');
2621
+ if (status) {
2622
+ status.innerHTML = `<strong>WebGPU Error:</strong> ${e.message}`;
2623
+ status.style.color = 'red';
2624
+ }
2625
+ }
2626
+ });
2627
+
2628
+ // Export for module usage
2629
+ export { testHarness, initGPU, getGPU };