doppler-gpu 0.4.6 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (705) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +98 -137
  3. package/assets/doppler.svg +1 -3
  4. package/models/catalog.json +1575 -0
  5. package/package.json +59 -26
  6. package/src/cli/doppler-cli.js +28 -7
  7. package/src/cli/doppler-quickstart.js +1 -1
  8. package/src/cli/doppler-serve.js +2 -2
  9. package/src/client/doppler-registry.js +12 -0
  10. package/src/client/doppler-registry.json +52 -3
  11. package/src/client/runtime/index.js +25 -2
  12. package/src/client/runtime/model-session.d.ts +31 -1
  13. package/src/client/runtime/model-session.js +31 -0
  14. package/src/client/runtime/model-source.d.ts +19 -2
  15. package/src/client/runtime/model-source.js +45 -5
  16. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  17. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  22. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  23. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +50 -9
  24. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +35 -5
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  26. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  27. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  28. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  29. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  30. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  32. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  33. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  34. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  36. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  38. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  39. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  41. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  42. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  43. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  44. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  45. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +548 -0
  46. package/src/config/execution-contract-check.js +14 -10
  47. package/src/config/kernel-path-loader.js +1 -0
  48. package/src/config/kernels/kernel-ref-digests.js +26 -17
  49. package/src/config/kernels/registry.json +448 -27
  50. package/src/config/merge.js +41 -0
  51. package/src/config/param-validator.js +67 -8
  52. package/src/config/required-inference-fields-contract-check.js +14 -0
  53. package/src/config/runtime/default.json +3 -1
  54. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  55. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  56. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  57. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  58. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  59. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  60. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  61. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  62. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  63. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  64. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  65. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  66. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  67. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  68. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  69. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  70. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  71. package/src/config/schema/debug.schema.d.ts +2 -0
  72. package/src/config/schema/debug.schema.js +1 -0
  73. package/src/config/schema/doppler.schema.d.ts +5 -1
  74. package/src/config/schema/doppler.schema.js +117 -292
  75. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  76. package/src/config/schema/execution-v1.schema.js +10 -0
  77. package/src/config/schema/index.d.ts +3 -1
  78. package/src/config/schema/inference.schema.d.ts +17 -3
  79. package/src/config/schema/kvcache.schema.d.ts +19 -3
  80. package/src/config/schema/kvcache.schema.js +2 -1
  81. package/src/config/schema/manifest.schema.d.ts +38 -1
  82. package/src/config/schema/manifest.schema.js +18 -0
  83. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  84. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  85. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  86. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  87. package/src/config/transforms/capability-transform-resolver.js +7 -1
  88. package/src/config/transforms/execution-graph-transforms.js +10 -2
  89. package/src/converter/conversion-plan.js +2 -0
  90. package/src/converter/parsers/transformer.js +27 -1
  91. package/src/converter/quantization-info.js +16 -4
  92. package/src/converter/rope-config.js +55 -3
  93. package/src/debug/config.d.ts +4 -0
  94. package/src/experimental/orchestration/index.d.ts +10 -1
  95. package/src/experimental/training/distillation/student-fixture.js +1 -1
  96. package/src/experimental/training/lora-pipeline.js +1 -1
  97. package/src/experimental/training/lora.js +2 -2
  98. package/src/experimental/training/suite.js +2 -2
  99. package/src/formats/rdrr/classification.js +3 -0
  100. package/src/formats/rdrr/integrity.d.ts +0 -2
  101. package/src/formats/rdrr/integrity.js +0 -2
  102. package/src/formats/rdrr/types.d.ts +1 -1
  103. package/src/gpu/kernels/attention.d.ts +2 -0
  104. package/src/gpu/kernels/attention.js +75 -16
  105. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  106. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  107. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  108. package/src/gpu/kernels/dequant.d.ts +6 -6
  109. package/src/gpu/kernels/dequant.js +17 -6
  110. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  111. package/src/gpu/kernels/fused_ffn.js +540 -19
  112. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  113. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  114. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  115. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  116. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  117. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  118. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  119. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  120. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  121. package/src/gpu/kernels/index.d.ts +21 -0
  122. package/src/gpu/kernels/index.js +13 -0
  123. package/src/gpu/kernels/linear-attention-core.js +414 -21
  124. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  125. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  126. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  127. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  128. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  129. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  130. package/src/gpu/kernels/matmul-selection.js +9 -2
  131. package/src/gpu/kernels/matmul.d.ts +1 -1
  132. package/src/gpu/kernels/matmul.js +24 -6
  133. package/src/gpu/kernels/moe.d.ts +5 -1
  134. package/src/gpu/kernels/moe.js +5 -2
  135. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  136. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  137. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  138. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  139. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  140. package/src/gpu/kernels/silu.wgsl +7 -1
  141. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  142. package/src/gpu/kernels/softmax.d.ts +3 -3
  143. package/src/gpu/kernels/softmax.js +9 -2
  144. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  145. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  146. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  147. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  148. package/src/gpu/kernels/types.d.ts +1 -1
  149. package/src/gpu/profiler.js +11 -1
  150. package/src/gpu/tensor.d.ts +1 -0
  151. package/src/index.d.ts +1 -1
  152. package/src/index.js +1 -1
  153. package/src/inference/browser-harness-model-helpers.js +4 -1
  154. package/src/inference/browser-harness-suite-helpers.js +1 -1
  155. package/src/inference/browser-harness-text-helpers.js +589 -117
  156. package/src/inference/browser-harness.d.ts +3 -3
  157. package/src/inference/browser-harness.js +486 -13
  158. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  159. package/src/inference/kv-cache/base.js +42 -1
  160. package/src/inference/kv-cache/quantized.js +16 -2
  161. package/src/inference/kv-cache/tiered.js +50 -15
  162. package/src/inference/moe-router.js +38 -7
  163. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  164. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  165. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  166. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  167. package/src/inference/pipelines/text/attention/projections.js +174 -4
  168. package/src/inference/pipelines/text/attention/record.js +44 -21
  169. package/src/inference/pipelines/text/attention/run.js +18 -3
  170. package/src/inference/pipelines/text/config.d.ts +23 -1
  171. package/src/inference/pipelines/text/config.js +235 -112
  172. package/src/inference/pipelines/text/embed.js +39 -12
  173. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  174. package/src/inference/pipelines/text/execution-plan.js +45 -2
  175. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  176. package/src/inference/pipelines/text/execution-v1.js +122 -8
  177. package/src/inference/pipelines/text/ffn/dense.js +254 -25
  178. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  179. package/src/inference/pipelines/text/ffn/standard.js +145 -39
  180. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  181. package/src/inference/pipelines/text/generator-helpers.js +4 -1
  182. package/src/inference/pipelines/text/generator-runtime.js +54 -20
  183. package/src/inference/pipelines/text/generator-steps.js +5 -2
  184. package/src/inference/pipelines/text/generator.d.ts +18 -1
  185. package/src/inference/pipelines/text/generator.js +303 -20
  186. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  187. package/src/inference/pipelines/text/init-rope.js +54 -6
  188. package/src/inference/pipelines/text/init.d.ts +22 -3
  189. package/src/inference/pipelines/text/init.js +65 -17
  190. package/src/inference/pipelines/text/layer.js +111 -5
  191. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  192. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  193. package/src/inference/pipelines/text/linear-attention.js +185 -24
  194. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  195. package/src/inference/pipelines/text/logits/gpu.js +78 -115
  196. package/src/inference/pipelines/text/logits/index.js +105 -87
  197. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  198. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  199. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  200. package/src/inference/pipelines/text/logits/utils.js +11 -0
  201. package/src/inference/pipelines/text/lora-apply.js +6 -4
  202. package/src/inference/pipelines/text/model-load.js +6 -0
  203. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  204. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  205. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  206. package/src/inference/pipelines/text/ops.js +48 -1
  207. package/src/inference/pipelines/text/per-layer-inputs.js +105 -21
  208. package/src/inference/pipelines/text/probes.js +9 -4
  209. package/src/inference/pipelines/text/state.d.ts +4 -4
  210. package/src/inference/pipelines/text/state.js +4 -0
  211. package/src/inference/pipelines/text/types.d.ts +27 -2
  212. package/src/inference/pipelines/text/weights.js +7 -2
  213. package/src/inference/pipelines/text.d.ts +21 -1
  214. package/src/inference/pipelines/text.js +60 -14
  215. package/src/inference/tokenizers/behavior-flags.js +23 -2
  216. package/src/inference/tokenizers/bundled.js +45 -23
  217. package/src/inference/tokenizers/types.d.ts +7 -4
  218. package/src/loader/doppler-loader.js +25 -2
  219. package/src/loader/final-weights-loader.js +12 -1
  220. package/src/loader/index.d.ts +3 -1
  221. package/src/loader/index.js +2 -0
  222. package/src/loader/loader-types.d.ts +3 -1
  223. package/src/loader/manifest-config.d.ts +18 -0
  224. package/src/loader/manifest-config.js +11 -3
  225. package/src/loader/shard-cache.d.ts +2 -0
  226. package/src/loader/shard-cache.js +29 -1
  227. package/src/models/gemma4.js +8 -0
  228. package/src/rules/execution-rules-contract-check.js +12 -3
  229. package/src/rules/inference/capability-transforms.rules.json +159 -22
  230. package/src/rules/inference/config.rules.json +26 -2
  231. package/src/rules/inference/dtype.rules.json +12 -5
  232. package/src/rules/inference/execution.rules.json +5 -19
  233. package/src/rules/inference/ffn.rules.json +33 -0
  234. package/src/rules/inference/moe.rules.json +97 -0
  235. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  236. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  237. package/src/rules/kernels/matmul.rules.json +1 -0
  238. package/src/rules/kernels/silu.rules.json +1 -0
  239. package/src/rules/rule-registry.js +2 -0
  240. package/src/storage/artifact-storage-context.js +36 -0
  241. package/src/tooling/command-api-constants.js +2 -2
  242. package/src/tooling/command-api-family-normalizers.js +2 -1
  243. package/src/tooling/command-api.d.ts +1 -1
  244. package/src/tooling/command-runner-shared.js +3 -1
  245. package/src/tooling/node-browser-command-runner.js +42 -7
  246. package/src/tooling/node-converter.js +20 -0
  247. package/src/tooling/node-source-runtime.js +5 -1
  248. package/src/tooling/opfs-cache.d.ts +21 -1
  249. package/src/tooling/opfs-cache.js +168 -32
  250. package/src/tooling-exports/storage.d.ts +1 -1
  251. package/src/tooling-exports/storage.js +1 -1
  252. package/src/types/gpu.d.ts +25 -0
  253. package/src/version.js +1 -1
  254. package/tests/kernels/browser/kernel-suite.js +1593 -0
  255. package/tests/kernels/browser/test-page.js +2629 -0
  256. package/tests/kernels/harness/benchmark.js +225 -0
  257. package/tests/kernels/harness/buffer-utils.js +88 -0
  258. package/tests/kernels/harness/tolerance.js +174 -0
  259. package/tests/kernels/reference/adam.js +15 -0
  260. package/tests/kernels/reference/attention.js +158 -0
  261. package/tests/kernels/reference/backward.js +96 -0
  262. package/tests/kernels/reference/bias-add.js +20 -0
  263. package/tests/kernels/reference/conv2d.js +58 -0
  264. package/tests/kernels/reference/cross-entropy.js +26 -0
  265. package/tests/kernels/reference/dequant.js +312 -0
  266. package/tests/kernels/reference/embed.js +15 -0
  267. package/tests/kernels/reference/gather.js +61 -0
  268. package/tests/kernels/reference/gelu.js +37 -0
  269. package/tests/kernels/reference/groupnorm.js +36 -0
  270. package/tests/kernels/reference/index.js +89 -0
  271. package/tests/kernels/reference/layernorm.js +22 -0
  272. package/tests/kernels/reference/matmul.js +57 -0
  273. package/tests/kernels/reference/modulate.js +24 -0
  274. package/tests/kernels/reference/moe-gather.js +83 -0
  275. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  276. package/tests/kernels/reference/residual.js +55 -0
  277. package/tests/kernels/reference/rmsnorm.js +52 -0
  278. package/tests/kernels/reference/rope.js +79 -0
  279. package/tests/kernels/reference/sample.js +97 -0
  280. package/tests/kernels/reference/scatter-add.js +58 -0
  281. package/tests/kernels/reference/silu.js +52 -0
  282. package/tests/kernels/reference/softmax.js +86 -0
  283. package/tests/kernels/reference/split-qkv.js +58 -0
  284. package/tests/kernels/reference/topk.js +88 -0
  285. package/tests/kernels/reference/upsample2d.js +37 -0
  286. package/BRANDING.md +0 -14
  287. package/SECURITY.md +0 -19
  288. package/assets/doppler-readme-hero.svg +0 -84
  289. package/assets/doppler-runtime-map.svg +0 -84
  290. package/src/bootstrap.js +0 -30
  291. package/src/cli/doppler-cli.d.ts +0 -46
  292. package/src/cli/doppler-quickstart.d.ts +0 -45
  293. package/src/cli/doppler-serve.d.ts +0 -210
  294. package/src/cli/runtime-profiles.d.ts +0 -40
  295. package/src/client/doppler-api.browser.d.ts +0 -1
  296. package/src/client/doppler-api.browser.js +0 -25
  297. package/src/client/doppler-registry.d.ts +0 -35
  298. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  299. package/src/client/runtime/source-runtime.d.ts +0 -26
  300. package/src/client/runtime/storage.d.ts +0 -5
  301. package/src/config/README.md +0 -61
  302. package/src/config/conversion/README.md +0 -58
  303. package/src/config/goal-completion-matrix.json +0 -552
  304. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  305. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  306. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  307. package/src/config/kernels/registry.d.ts +0 -51
  308. package/src/config/merge-contract-check.d.ts +0 -16
  309. package/src/config/merge-contract-check.js +0 -318
  310. package/src/config/merge-helpers.d.ts +0 -60
  311. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  312. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  313. package/src/config/param-categories.d.ts +0 -8
  314. package/src/config/param-validator.d.ts +0 -28
  315. package/src/config/platforms/loader.d.ts +0 -38
  316. package/src/config/quantization-contract-check.d.ts +0 -12
  317. package/src/config/quantization-contract-check.js +0 -91
  318. package/src/config/runtime/README.md +0 -28
  319. package/src/config/runtime-merge.d.ts +0 -5
  320. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  321. package/src/config/schema/command-surface-contract.schema.json +0 -81
  322. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  323. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  324. package/src/config/schema/platform.schema.js +0 -1
  325. package/src/config/schema/policy-schema-registry.json +0 -26
  326. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  327. package/src/config/support-tiers/subsystems.json +0 -502
  328. package/src/config/supported-operations.d.ts +0 -14
  329. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  330. package/src/config/transforms/execution-graph-transforms.d.ts +0 -300
  331. package/src/converter/conversion-plan.d.ts +0 -56
  332. package/src/converter/index.d.ts +0 -98
  333. package/src/converter/index.js +0 -59
  334. package/src/converter/parsers/diffusion.d.ts +0 -50
  335. package/src/converter/parsers/gguf.d.ts +0 -31
  336. package/src/converter/parsers/index.d.ts +0 -21
  337. package/src/converter/parsers/index.js +0 -12
  338. package/src/converter/parsers/transformer.d.ts +0 -31
  339. package/src/converter/quantization-info.d.ts +0 -38
  340. package/src/converter/rope-config.d.ts +0 -16
  341. package/src/converter/tokenizer-utils.d.ts +0 -12
  342. package/src/debug/reference/README.md +0 -114
  343. package/src/debug/reference/hf_attn_debug.py +0 -114
  344. package/src/debug/reference/hf_embed_check.py +0 -89
  345. package/src/debug/reference/hf_layer_out.py +0 -100
  346. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  347. package/src/debug/reference/hf_rope_check.py +0 -116
  348. package/src/debug/reference/hf_weights.py +0 -75
  349. package/src/errors/doppler-error.d.ts +0 -20
  350. package/src/experimental/bridge/extension/background.d.ts +0 -14
  351. package/src/experimental/bridge/extension/background.js +0 -168
  352. package/src/experimental/bridge/extension/manifest.json +0 -34
  353. package/src/experimental/bridge/extension-client.d.ts +0 -114
  354. package/src/experimental/bridge/index.d.ts +0 -69
  355. package/src/experimental/bridge/node-bridge.js +0 -24
  356. package/src/experimental/bridge/protocol.d.ts +0 -96
  357. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  358. package/src/experimental/browser/gguf-importer.js +0 -532
  359. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  360. package/src/experimental/browser/quantization.d.ts +0 -74
  361. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  362. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  363. package/src/experimental/browser/shard-io-browser.js +0 -89
  364. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  365. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  366. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  367. package/src/experimental/distribution/p2p-transport.js +0 -75
  368. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  369. package/src/experimental/hotswap/manifest.d.ts +0 -42
  370. package/src/experimental/hotswap/model-swap.js +0 -42
  371. package/src/experimental/hotswap/runtime.d.ts +0 -31
  372. package/src/experimental/training/README.md +0 -164
  373. package/src/experimental/training/artifacts.d.ts +0 -160
  374. package/src/experimental/training/attention-backward.d.ts +0 -30
  375. package/src/experimental/training/attention-forward.d.ts +0 -22
  376. package/src/experimental/training/attention-forward.js +0 -82
  377. package/src/experimental/training/autograd.d.ts +0 -51
  378. package/src/experimental/training/backward-kernels.js +0 -39
  379. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  380. package/src/experimental/training/checkpoint.d.ts +0 -36
  381. package/src/experimental/training/clip.d.ts +0 -9
  382. package/src/experimental/training/dataloader.d.ts +0 -8
  383. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  384. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  385. package/src/experimental/training/datasets/index.d.ts +0 -24
  386. package/src/experimental/training/datasets/index.js +0 -13
  387. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  388. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  389. package/src/experimental/training/datasets/reploid.js +0 -36
  390. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  391. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  392. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  393. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  394. package/src/experimental/training/datasets/token-batch.js +0 -52
  395. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  396. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  397. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  398. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  399. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  400. package/src/experimental/training/distillation/eval.d.ts +0 -34
  401. package/src/experimental/training/distillation/index.d.ts +0 -29
  402. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  403. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  404. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  405. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  406. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  407. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  408. package/src/experimental/training/export.d.ts +0 -41
  409. package/src/experimental/training/index.d.ts +0 -74
  410. package/src/experimental/training/index.js +0 -63
  411. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  412. package/src/experimental/training/lora.d.ts +0 -19
  413. package/src/experimental/training/loss-scaling.d.ts +0 -21
  414. package/src/experimental/training/loss.d.ts +0 -10
  415. package/src/experimental/training/objectives/base.d.ts +0 -58
  416. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  417. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  418. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  419. package/src/experimental/training/objectives/index.d.ts +0 -12
  420. package/src/experimental/training/objectives/index.js +0 -6
  421. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  422. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  423. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  424. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  425. package/src/experimental/training/operator-command.d.ts +0 -5
  426. package/src/experimental/training/operator-eval.d.ts +0 -48
  427. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  428. package/src/experimental/training/optimizer.d.ts +0 -22
  429. package/src/experimental/training/runner.d.ts +0 -249
  430. package/src/experimental/training/suite.d.ts +0 -299
  431. package/src/experimental/training/tensor-factory.d.ts +0 -9
  432. package/src/experimental/training/trainer.d.ts +0 -89
  433. package/src/experimental/training/ul_dataset.d.ts +0 -47
  434. package/src/experimental/training/ul_schedule.d.ts +0 -6
  435. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  436. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  437. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  438. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  439. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  440. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  441. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  442. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  443. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  444. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  445. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  446. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  447. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  448. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  449. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  450. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  451. package/src/experimental/training/workload-packs/registry.json +0 -278
  452. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  453. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  454. package/src/experimental/training/workloads.d.ts +0 -201
  455. package/src/formats/gguf/types.d.ts +0 -74
  456. package/src/formats/index.d.ts +0 -75
  457. package/src/formats/index.js +0 -19
  458. package/src/formats/litert/types.d.ts +0 -74
  459. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  460. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  461. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  462. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  463. package/src/formats/safetensors/types.d.ts +0 -67
  464. package/src/formats/tflite/types.d.ts +0 -61
  465. package/src/formats/tokenizer/index.d.ts +0 -5
  466. package/src/formats/tokenizer/index.js +0 -3
  467. package/src/formats/tokenizer/types.d.ts +0 -9
  468. package/src/gpu/kernel-runtime.d.ts +0 -13
  469. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  470. package/src/gpu/kernel-selection-log.d.ts +0 -4
  471. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  472. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  473. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  474. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  475. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  476. package/src/gpu/kernel-tuner/types.js +0 -4
  477. package/src/gpu/kernel-tuner.d.ts +0 -33
  478. package/src/gpu/kernels/README.md +0 -159
  479. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  480. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  481. package/src/gpu/kernels/check-stop.d.ts +0 -31
  482. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  483. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  484. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  485. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  486. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  487. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  488. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  489. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  490. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  491. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  492. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  493. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  494. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  495. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  496. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  497. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  498. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  499. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  500. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  501. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  502. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  503. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  504. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  505. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  506. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  507. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  508. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  509. package/src/gpu/kernels/constants.d.ts +0 -129
  510. package/src/gpu/kernels/dispatch.d.ts +0 -72
  511. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  512. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  513. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  514. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  515. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  516. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  517. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  518. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  519. package/src/gpu/kernels/types.js +0 -4
  520. package/src/gpu/perf-guards.d.ts +0 -13
  521. package/src/gpu/profiler.d.ts +0 -93
  522. package/src/gpu/readback-utils.d.ts +0 -16
  523. package/src/gpu/readback-utils.js +0 -49
  524. package/src/gpu/submit-probe.d.ts +0 -12
  525. package/src/index-browser.d.ts +0 -2
  526. package/src/index-browser.js +0 -2
  527. package/src/inference/README.md +0 -593
  528. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  529. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  530. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  531. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  532. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  533. package/src/inference/browser-harness-text-helpers.d.ts +0 -124
  534. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  535. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  536. package/src/inference/network-evolution.js +0 -79
  537. package/src/inference/pipelines/context.d.ts +0 -21
  538. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  539. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  540. package/src/inference/pipelines/diffusion/index.js +0 -3
  541. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  542. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  543. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  544. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  545. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  546. package/src/inference/pipelines/diffusion/types.js +0 -1
  547. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  548. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  549. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  550. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  551. package/src/inference/pipelines/dream/pipeline.js +0 -8
  552. package/src/inference/pipelines/energy/index.d.ts +0 -1
  553. package/src/inference/pipelines/energy/index.js +0 -1
  554. package/src/inference/pipelines/energy/types.js +0 -1
  555. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  556. package/src/inference/pipelines/energy-head/index.js +0 -6
  557. package/src/inference/pipelines/factory.d.ts +0 -10
  558. package/src/inference/pipelines/index.d.ts +0 -22
  559. package/src/inference/pipelines/index.js +0 -19
  560. package/src/inference/pipelines/registry.d.ts +0 -15
  561. package/src/inference/pipelines/rng.d.ts +0 -2
  562. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  563. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  564. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  565. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  566. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  567. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  568. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  569. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  570. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  571. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  572. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  573. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  574. package/src/inference/pipelines/text/attention.d.ts +0 -23
  575. package/src/inference/pipelines/text/attention.js +0 -12
  576. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  577. package/src/inference/pipelines/text/buffer-types.js +0 -4
  578. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  579. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  580. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  581. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  582. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  583. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  584. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  585. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  586. package/src/inference/pipelines/text/embed.d.ts +0 -68
  587. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  588. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  589. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  590. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  591. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  592. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  593. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -25
  595. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  596. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  597. package/src/inference/pipelines/text/ffn.js +0 -18
  598. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  599. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  600. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  601. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  602. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  603. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -89
  604. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  605. package/src/inference/pipelines/text/index.d.ts +0 -5
  606. package/src/inference/pipelines/text/index.js +0 -6
  607. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  608. package/src/inference/pipelines/text/layer.d.ts +0 -118
  609. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  610. package/src/inference/pipelines/text/logits/types.js +0 -4
  611. package/src/inference/pipelines/text/logits.d.ts +0 -27
  612. package/src/inference/pipelines/text/logits.js +0 -16
  613. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  614. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  615. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  616. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  617. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  618. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  619. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -40
  620. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  621. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  622. package/src/inference/pipelines/text/ops.d.ts +0 -219
  623. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  624. package/src/inference/pipelines/text/probes.d.ts +0 -28
  625. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  626. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  627. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  628. package/src/inference/pipelines/text/types.js +0 -4
  629. package/src/inference/tokenizers/base.d.ts +0 -44
  630. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  631. package/src/inference/tokenizers/bpe.d.ts +0 -27
  632. package/src/inference/tokenizers/bundled.d.ts +0 -67
  633. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  634. package/src/inference/tokenizers/types.js +0 -7
  635. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  636. package/src/loader/per-layer-input-loader.d.ts +0 -30
  637. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  638. package/src/loader/tensors/source-transform.d.ts +0 -65
  639. package/src/memory/address-table.d.ts +0 -96
  640. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  641. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  642. package/src/rules/rule-registry.d.ts +0 -77
  643. package/src/storage/artifact-storage-context.d.ts +0 -63
  644. package/src/storage/backends/idb-store.d.ts +0 -52
  645. package/src/storage/backends/memory-store.d.ts +0 -36
  646. package/src/storage/backends/opfs-store.d.ts +0 -41
  647. package/src/storage/blake3.d.ts +0 -17
  648. package/src/storage/download-types.d.ts +0 -157
  649. package/src/storage/downloader.d.ts +0 -103
  650. package/src/storage/index.d.ts +0 -274
  651. package/src/storage/index.js +0 -201
  652. package/src/storage/preflight.d.ts +0 -144
  653. package/src/storage/preflight.js +0 -360
  654. package/src/storage/quickstart-downloader.d.ts +0 -157
  655. package/src/storage/quickstart-downloader.js +0 -289
  656. package/src/storage/source-artifact-store.d.ts +0 -59
  657. package/src/sw.js +0 -184
  658. package/src/tooling/command-api-constants.d.ts +0 -9
  659. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  660. package/src/tooling/command-api-helpers.d.ts +0 -23
  661. package/src/tooling/command-runner-shared.d.ts +0 -89
  662. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  663. package/src/tooling/conversion-config-materializer.js +0 -96
  664. package/src/tooling/diagnose-runner.d.ts +0 -28
  665. package/src/tooling/hf-registry-utils.d.ts +0 -55
  666. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  667. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  668. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  669. package/src/tooling/kernel-path-builder/index.js +0 -1361
  670. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  671. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  672. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  673. package/src/tooling/lean-execution-contract-runner.js +0 -158
  674. package/src/tooling/lean-execution-contract.d.ts +0 -16
  675. package/src/tooling/lean-execution-contract.js +0 -81
  676. package/src/tooling/litert-package-runtime.d.ts +0 -46
  677. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  678. package/src/tooling/node-convert-worker.d.ts +0 -1
  679. package/src/tooling/node-converter.d.ts +0 -1
  680. package/src/tooling/node-file-fetch.d.ts +0 -1
  681. package/src/tooling/node-source-runtime.d.ts +0 -24
  682. package/src/tooling/node-webgpu.d.ts +0 -17
  683. package/src/tooling/precision-replay-math.js +0 -174
  684. package/src/tooling/program-bundle-host.d.ts +0 -16
  685. package/src/tooling/program-bundle-host.js +0 -20
  686. package/src/tooling/runtime-input-composition.d.ts +0 -37
  687. package/src/tooling/source-package-profiles.d.ts +0 -50
  688. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  689. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  690. package/src/tooling/source-runtime-materializer.js +0 -89
  691. package/src/tooling-exports.browser.d.ts +0 -7
  692. package/src/types/chrome.d.ts +0 -36
  693. package/src/types/chrome.js +0 -1
  694. package/src/types/index.d.ts +0 -1
  695. package/src/types/index.js +0 -1
  696. package/src/utils/array-buffer.d.ts +0 -4
  697. package/src/utils/canonical-hash.d.ts +0 -7
  698. package/src/utils/clone-json.d.ts +0 -1
  699. package/src/utils/hf-resolve-url.d.ts +0 -16
  700. package/src/utils/index.d.ts +0 -10
  701. package/src/utils/index.js +0 -10
  702. package/src/utils/load-json.d.ts +0 -5
  703. package/src/utils/plain-object.d.ts +0 -1
  704. package/src/utils/sha256.d.ts +0 -5
  705. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -0,0 +1,2629 @@
1
+
2
+
3
+ // Import from main doppler repo (relative path from tests/kernels/browser/)
4
+ // When served from doppler/, paths are relative to that root
5
+ import { initDevice, getKernelCapabilities, getDeviceLimits, destroyDevice } from '../../../src/gpu/device.js';
6
+
7
+ // Import tensor abstraction for Tensor-based kernels
8
+ import { createTensor } from '../../../src/gpu/tensor.js';
9
+
10
+ // Ensure platform/registry lookups resolve to the main config paths when bundled
11
+ import { setPlatformsBaseUrl } from '../../../src/config/platforms/loader.js';
12
+ import { setRegistryUrl } from '../../../src/config/kernels/registry.js';
13
+ import { createPipeline, createUniformBufferWithView } from '../../../src/gpu/kernels/utils.js';
14
+ import { dispatch } from '../../../src/gpu/kernels/dispatch.js';
15
+
16
+ // Kernel path can be injected via URL param for targeted tests
17
+ import { resolveKernelPath, setActiveKernelPath } from '../../../src/config/kernel-path-loader.js';
18
+
19
+ import {
20
+ runBackwardKernel,
21
+ recordBackwardKernel,
22
+ runMatmulTransposeA,
23
+ } from '../../../src/gpu/kernels/backward/utils.js';
24
+ import * as kernelSelector from '../../../src/gpu/kernel-selector.js';
25
+
26
+ // Destructure available functions with defaults
27
+ const {
28
+ runMatmul = null,
29
+ runSoftmax = null,
30
+ runTopK = null,
31
+ runSoftmaxTopK = null,
32
+ runScatterAdd = null,
33
+ runMoEGather = null,
34
+ runRMSNorm = null,
35
+ runRoPE = null,
36
+ runSiLU = null,
37
+ runSwiGLURowsplitBias = null,
38
+ runScale = null,
39
+ runGather = null,
40
+ runResidualAdd = null,
41
+ runBiasAdd = null,
42
+ runAttention = null,
43
+ runAttentionTiered = null,
44
+ runAttentionTieredQuant = null,
45
+ dequantize = null,
46
+ dequantizeQ6K = null,
47
+ runBF16ToF32 = null,
48
+ runBF16ToF16 = null,
49
+ castF32ToF16 = null,
50
+ castF16ToF32 = null,
51
+ runGeLU = null,
52
+ runSplitQKV = null,
53
+ runConv2D = null,
54
+ runGroupNorm = null,
55
+ runLayerNorm = null,
56
+ runLayerNormBackward = null,
57
+ runModulate = null,
58
+ runPixelShuffle = null,
59
+ runUpsample2D = null,
60
+ runTranspose = null,
61
+ runKVQuantize = null,
62
+ runCrossEntropyLoss = null,
63
+ runCrossEntropyBackward = null,
64
+ runBiasAddBackward = null,
65
+ runUpsample2DBackward = null,
66
+ runPixelShuffleBackward = null,
67
+ runGroupNormBackward = null,
68
+ runConv2DBackward = null,
69
+ runEmbedBackward = null,
70
+ runGeluBackward = null,
71
+ runRmsNormBackward = null,
72
+ runRoPEBackward = null,
73
+ runScaleBackward = null,
74
+ runSiluBackward = null,
75
+ runSoftmaxBackward = null,
76
+ runAttentionBackward = null,
77
+ runAdam = null,
78
+ } = kernelSelector;
79
+
80
+ // Import dequant variant selector for test diagnostics
81
+ import { selectDequantKernel } from '../../../src/gpu/kernels/dequant.js';
82
+
83
+ // Import sample kernel
84
+ import * as sampleKernel from '../../../src/gpu/kernels/sample.js';
85
+
86
+ // Import check-stop kernel
87
+ import { checkStop } from '../../../src/gpu/kernels/check-stop.js';
88
+
89
+ // Import fused kernels
90
+ import { runMatmulResidualFused } from '../../../src/gpu/kernels/fused_matmul_residual.js';
91
+ import { runMatmulRMSNormFused } from '../../../src/gpu/kernels/fused_matmul_rmsnorm.js';
92
+ import { runFusedFFN } from '../../../src/gpu/kernels/fused_ffn.js';
93
+
94
+ // Optional buffer pool
95
+ let bufferPool = null;
96
+ try {
97
+ bufferPool = await import('../../../src/memory/buffer-pool.js');
98
+ } catch (e) {
99
+ console.warn('Buffer pool not available:', e.message);
100
+ }
101
+
102
+ // Import reference implementations
103
+ import * as references from '../reference/index.js';
104
+ import { compareArrays, generateTestData, KERNEL_TOLERANCES } from '../harness/tolerance.js';
105
+ import { createBuffer, readGPUBuffer, readAsFloat32, readAsUint32 } from '../harness/buffer-utils.js';
106
+ import { KernelBenchmark, computeMetrics } from '../harness/benchmark.js';
107
+ import { createWeightBuffer } from '../../../src/gpu/weight-buffer.js';
108
+
109
+ // Global state
110
+ let device = null;
111
+ let initialized = false;
112
+
113
+
114
+ function f16ToF32(h) {
115
+ const sign = (h & 0x8000) >> 15;
116
+ const exponent = (h & 0x7C00) >> 10;
117
+ const mantissa = h & 0x03FF;
118
+
119
+ if (exponent === 0) {
120
+ // Denormalized or zero
121
+ if (mantissa === 0) return sign ? -0 : 0;
122
+ return (sign ? -1 : 1) * Math.pow(2, -14) * (mantissa / 1024);
123
+ } else if (exponent === 31) {
124
+ // Infinity or NaN
125
+ return mantissa === 0 ? (sign ? -Infinity : Infinity) : NaN;
126
+ }
127
+
128
+ // Normalized
129
+ return (sign ? -1 : 1) * Math.pow(2, exponent - 15) * (1 + mantissa / 1024);
130
+ }
131
+
132
+ function f32ToF16Bits(value) {
133
+ return references.float32ToFloat16(value);
134
+ }
135
+
136
+ function toF16Array(values) {
137
+ const out = new Uint16Array(values.length);
138
+ for (let i = 0; i < values.length; i++) {
139
+ out[i] = f32ToF16Bits(values[i]);
140
+ }
141
+ return out;
142
+ }
143
+
144
+ function toF16RoundedFloat32(values) {
145
+ const out = new Float32Array(values.length);
146
+ for (let i = 0; i < values.length; i++) {
147
+ out[i] = f16ToF32(f32ToF16Bits(values[i]));
148
+ }
149
+ return out;
150
+ }
151
+
152
+ async function readTensorToFloat32(tensor, count) {
153
+ if (tensor.dtype === 'f16') {
154
+ const rawData = await readBufferData(tensor.buffer, count * 2);
155
+ const u16View = new Uint16Array(rawData);
156
+ const out = new Float32Array(u16View.length);
157
+ for (let i = 0; i < u16View.length; i++) {
158
+ out[i] = f16ToF32(u16View[i]);
159
+ }
160
+ return out;
161
+ }
162
+ return new Float32Array(await readBufferData(tensor.buffer, count * 4));
163
+ }
164
+
165
+ function buildAttentionKernelPath(id, kernelFile) {
166
+ return {
167
+ id,
168
+ name: id,
169
+ activationDtype: 'f16',
170
+ decode: {
171
+ steps: [
172
+ {
173
+ op: 'attention',
174
+ kernel: kernelFile,
175
+ entry: 'main',
176
+ constants: { SOFTCAP: 50.0 },
177
+ },
178
+ ],
179
+ },
180
+ };
181
+ }
182
+
183
+ function fillDeterministic(values, scale = 0.01) {
184
+ for (let i = 0; i < values.length; i++) {
185
+ values[i] = Math.sin(i * 0.13) * scale;
186
+ }
187
+ return values;
188
+ }
189
+
190
+
191
+ async function initGPU() {
192
+ if (device) return device;
193
+
194
+ setPlatformsBaseUrl('/src/config/platforms/');
195
+ setRegistryUrl('/src/config/kernels/registry.json');
196
+
197
+ device = await initDevice();
198
+ if (!device) {
199
+ throw new Error('WebGPU not available');
200
+ }
201
+
202
+ setActiveKernelPath(null, 'none');
203
+
204
+ initialized = true;
205
+ return device;
206
+ }
207
+
208
+
209
+ async function getGPU() {
210
+ if (!device) {
211
+ await initGPU();
212
+ }
213
+ return { device: device, queue: device.queue };
214
+ }
215
+
216
+
217
+ function makeBuffer(
218
+ data,
219
+ usage = GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC
220
+ ) {
221
+ const byteLength = data instanceof ArrayBuffer ? data.byteLength : data.byteLength;
222
+ const buffer = device.createBuffer({
223
+ size: byteLength,
224
+ usage: usage | GPUBufferUsage.COPY_DST,
225
+ mappedAtCreation: true,
226
+ });
227
+
228
+ const mappedRange = buffer.getMappedRange();
229
+ if (data instanceof Float32Array) {
230
+ new Float32Array(mappedRange).set(data);
231
+ } else if (data instanceof Uint32Array) {
232
+ new Uint32Array(mappedRange).set(data);
233
+ } else if (data instanceof Int32Array) {
234
+ new Int32Array(mappedRange).set(data);
235
+ } else if (data instanceof Uint16Array) {
236
+ new Uint16Array(mappedRange).set(data);
237
+ } else if (data instanceof Uint8Array) {
238
+ new Uint8Array(mappedRange).set(data);
239
+ } else {
240
+ new Uint8Array(mappedRange).set(new Uint8Array(data));
241
+ }
242
+ buffer.unmap();
243
+
244
+ return buffer;
245
+ }
246
+
247
+
248
+ async function readBufferData(buffer, size) {
249
+ const stagingBuffer = device.createBuffer({
250
+ size,
251
+ usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
252
+ });
253
+
254
+ const encoder = device.createCommandEncoder();
255
+ encoder.copyBufferToBuffer(buffer, 0, stagingBuffer, 0, size);
256
+ device.queue.submit([encoder.finish()]);
257
+
258
+ await stagingBuffer.mapAsync(GPUMapMode.READ);
259
+ const data = new Uint8Array(stagingBuffer.getMappedRange()).slice();
260
+ stagingBuffer.unmap();
261
+ stagingBuffer.destroy();
262
+
263
+ return data.buffer;
264
+ }
265
+
266
+ // ============================================================================
267
+ // Test Harness - Exposed to window for browser automation
268
+ // ============================================================================
269
+
270
+ const testHarness = {
271
+ // Core
272
+ getGPU,
273
+ device: () => device,
274
+
275
+ // Reference implementations
276
+ references,
277
+ softmax: references.softmaxRef,
278
+ topkRef: references.topkRef,
279
+ softmaxTopkRef: references.softmaxTopkRef,
280
+ matmulRef: references.matmulRef,
281
+ scatterAddRef: references.scatterAddRef,
282
+
283
+ // Utilities
284
+ generateTestData,
285
+ compareArrays,
286
+ makeBuffer,
287
+ readBufferData,
288
+ toF16RoundedFloat32,
289
+ KERNEL_TOLERANCES,
290
+ selectDequantKernel,
291
+
292
+ async runMatmulTransposeA(dev, A, B, M, N, K, options = {}) {
293
+ const bufA = makeBuffer(A);
294
+ const bufB = makeBuffer(B);
295
+ const tensorA = createTensor(bufA, 'f32', [K, M], 'matmul_ta_a');
296
+ const tensorB = createTensor(bufB, 'f32', [K, N], 'matmul_ta_b');
297
+
298
+ const resultTensor = await runMatmulTransposeA(tensorA, tensorB, M, N, K, options);
299
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
300
+
301
+ bufA.destroy();
302
+ bufB.destroy();
303
+ resultTensor.buffer.destroy();
304
+
305
+ return result;
306
+ },
307
+
308
+ // ============================================================================
309
+ // Kernel Runners (match expected interface from tests)
310
+ // ============================================================================
311
+
312
+
313
+ async runMatmul(dev, A, B, M, N, K, alpha = 1.0) {
314
+ if (!runMatmul) {
315
+ // Fallback to reference implementation
316
+ return references.matmulRef(A, B, M, N, K, alpha);
317
+ }
318
+
319
+ const bufA = makeBuffer(A);
320
+ const tensorA = createTensor(bufA, 'f32', [M, K], 'matmul_a');
321
+ const bufB = makeBuffer(B);
322
+
323
+ // Test uses standard layout B [K, N], so transposeB = false
324
+ // (GPU kernel defaults to transposeB=true for SafeTensors [N, K] layout)
325
+ const resultTensor = await runMatmul(tensorA, bufB, M, N, K, {
326
+ alpha,
327
+ transposeB: false,
328
+ bDtype: 'f32',
329
+ outputDtype: 'f32',
330
+ });
331
+
332
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
333
+
334
+ bufA.destroy();
335
+ bufB.destroy();
336
+ resultTensor.buffer.destroy();
337
+
338
+ return result;
339
+ },
340
+
341
+
342
+ async runBatchMatmul(dev, A, B, batch, M, N, K) {
343
+ // Always use reference - batch matmul kernel may not be implemented
344
+ return references.batchMatmulRef(A, B, batch, M, N, K);
345
+ },
346
+
347
+
348
+ async runMatvec(dev, A, x, M, K) {
349
+ // Always use reference - matvec kernel may not be implemented
350
+ return references.matvecRef(A, x, M, K);
351
+ },
352
+
353
+
354
+ async runMatmulQ4K(dev, A, B_q4k, M, N, K, alpha = 1.0) {
355
+ if (!runMatmul) {
356
+ throw new Error('runMatmul kernel not available');
357
+ }
358
+
359
+ // Create A buffer (activations)
360
+ const bufA = makeBuffer(A);
361
+ const tensorA = createTensor(bufA, 'f32', [M, K], 'matmul_q4k_a');
362
+
363
+ // Create B buffer and pass q4k dtype to trigger fused Q4K selection
364
+ const bufB = makeBuffer(B_q4k, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC);
365
+
366
+ // Run matmul - kernel auto-detects q4k and uses fused variant
367
+ // transposeB is implicit for Q4K (weight matrix stored as [N, K])
368
+ const resultTensor = await runMatmul(tensorA, bufB, M, N, K, {
369
+ alpha,
370
+ bDtype: 'q4k',
371
+ outputDtype: 'f32',
372
+ });
373
+
374
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
375
+
376
+ bufA.destroy();
377
+ bufB.destroy();
378
+ resultTensor.buffer.destroy();
379
+
380
+ return result;
381
+ },
382
+
383
+
384
+ async runSoftmax(dev, input, innerSize, outerSize, temperature = 1.0) {
385
+ if (!runSoftmax) {
386
+ return references.softmaxRef(input, innerSize, outerSize, temperature);
387
+ }
388
+
389
+ const inputBuf = makeBuffer(input);
390
+ const inputTensor = createTensor(inputBuf, 'f32', [outerSize, innerSize], 'softmax_input');
391
+
392
+ const resultTensor = await runSoftmax(inputTensor, -1, {
393
+ batchSize: outerSize,
394
+ size: innerSize,
395
+ temperature,
396
+ });
397
+
398
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, input.length * 4));
399
+
400
+ inputBuf.destroy();
401
+ resultTensor.buffer.destroy();
402
+
403
+ return result;
404
+ },
405
+
406
+
407
+ async runSoftmaxTopK(dev, logits, numTokens, numExperts, topK, options = {}) {
408
+ if (!runSoftmaxTopK) {
409
+ return references.softmaxTopkRef(logits, numTokens, numExperts, topK, options.normalize !== false);
410
+ }
411
+
412
+ const inputBuf = makeBuffer(logits);
413
+
414
+ const { indices: indicesBuf, weights: weightsBuf } = await runSoftmaxTopK(
415
+ inputBuf,
416
+ numTokens,
417
+ numExperts,
418
+ topK,
419
+ {
420
+ normalize: options.normalize !== false,
421
+ inputDtype: options.inputDtype ?? 'f32',
422
+ weightsDtype: options.weightsDtype ?? 'f32',
423
+ }
424
+ );
425
+
426
+ const indices = new Uint32Array(await readBufferData(indicesBuf, numTokens * topK * 4));
427
+ const weights = new Float32Array(await readBufferData(weightsBuf, numTokens * topK * 4));
428
+
429
+ inputBuf.destroy();
430
+ indicesBuf.destroy();
431
+ weightsBuf.destroy();
432
+
433
+ return { indices, weights };
434
+ },
435
+
436
+
437
+ async runTopK(dev, probs, numTokens, numExperts, topK, options = {}) {
438
+ const inputBuf = makeBuffer(probs);
439
+
440
+ const { indices: indicesBuf, weights: weightsBuf } = await runTopK(
441
+ inputBuf,
442
+ numTokens,
443
+ numExperts,
444
+ topK,
445
+ { normalize: options.normalize !== false }
446
+ );
447
+
448
+ const indices = new Uint32Array(await readBufferData(indicesBuf, numTokens * topK * 4));
449
+ const weights = new Float32Array(await readBufferData(weightsBuf, numTokens * topK * 4));
450
+
451
+ inputBuf.destroy();
452
+ indicesBuf.destroy();
453
+ weightsBuf.destroy();
454
+
455
+ return { indices, weights };
456
+ },
457
+
458
+
459
+ async runScatterAdd(dev, expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK) {
460
+ if (!runScatterAdd) {
461
+ return references.scatterAddRef(expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK);
462
+ }
463
+
464
+ const expertBuf = makeBuffer(expertOutputs);
465
+ const indicesBuf = makeBuffer(indices);
466
+ const weightsBuf = makeBuffer(weights);
467
+
468
+ // Wrap expertBuf in Tensor (MoE kernels now use Tensor abstraction)
469
+ const expertTensor = createTensor(expertBuf, 'f32', [numExperts, numTokens, hiddenSize], 'expert_outputs');
470
+ const resultTensor = await runScatterAdd(
471
+ expertTensor,
472
+ indicesBuf,
473
+ weightsBuf,
474
+ numTokens,
475
+ hiddenSize,
476
+ numExperts,
477
+ topK
478
+ );
479
+
480
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, numTokens * hiddenSize * 4));
481
+
482
+ expertBuf.destroy();
483
+ indicesBuf.destroy();
484
+ weightsBuf.destroy();
485
+ resultTensor.buffer.destroy();
486
+
487
+ return result;
488
+ },
489
+
490
+
491
+ async runRMSNorm(dev, input, weight, numTokens, hiddenSize, eps = 1e-6, options = {}) {
492
+ if (!runRMSNorm) {
493
+ return references.rmsNormRef(input, weight, numTokens, hiddenSize, eps);
494
+ }
495
+
496
+ const inputBuf = makeBuffer(input);
497
+ const weightBuf = makeBuffer(weight);
498
+ const inputTensor = createTensor(inputBuf, 'f32', [numTokens, hiddenSize], 'rmsnorm_input');
499
+
500
+ const resultTensor = await runRMSNorm(inputTensor, weightBuf, eps, {
501
+ batchSize: numTokens,
502
+ hiddenSize,
503
+ ...options,
504
+ });
505
+
506
+ let result;
507
+ if (resultTensor.dtype === 'f16') {
508
+ const rawData = await readBufferData(resultTensor.buffer, numTokens * hiddenSize * 2);
509
+ const u16View = new Uint16Array(rawData);
510
+ result = new Float32Array(u16View.length);
511
+ for (let i = 0; i < u16View.length; i++) {
512
+ result[i] = f16ToF32(u16View[i]);
513
+ }
514
+ } else {
515
+ result = new Float32Array(
516
+ await readBufferData(resultTensor.buffer, numTokens * hiddenSize * 4)
517
+ );
518
+ }
519
+
520
+ inputBuf.destroy();
521
+ weightBuf.destroy();
522
+ resultTensor.buffer.destroy();
523
+
524
+ return result;
525
+ },
526
+
527
+
528
+ async runRoPE(dev, input, seqLen, numHeads, headDim, startPos = 0) {
529
+ const { cos, sin } = references.computeRopeFreqs(headDim, seqLen + startPos);
530
+
531
+ if (!runRoPE) {
532
+ return references.ropeRef(input, cos, sin, seqLen, numHeads, headDim, startPos);
533
+ }
534
+
535
+ const inputBuf = makeBuffer(input);
536
+ const cosBuf = makeBuffer(cos);
537
+ const sinBuf = makeBuffer(sin);
538
+
539
+ const inputTensor = createTensor(inputBuf, 'f32', [seqLen, numHeads, headDim], 'rope_input');
540
+
541
+ await runRoPE(inputTensor, cosBuf, sinBuf, seqLen, {
542
+ numHeads,
543
+ headDim,
544
+ startPos,
545
+ });
546
+
547
+ const result = new Float32Array(
548
+ await readBufferData(inputBuf, seqLen * numHeads * headDim * 4)
549
+ );
550
+
551
+ inputBuf.destroy();
552
+ cosBuf.destroy();
553
+ sinBuf.destroy();
554
+
555
+ return result;
556
+ },
557
+
558
+
559
+ async runSiLU(dev, input) {
560
+ if (!runSiLU) {
561
+ return references.siluRef(input);
562
+ }
563
+
564
+ const inputBuf = makeBuffer(input);
565
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'silu_input');
566
+
567
+ const resultTensor = await runSiLU(inputTensor, { size: input.length, swigluLimit: null });
568
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, input.length * 4));
569
+
570
+ inputBuf.destroy();
571
+ resultTensor.buffer.destroy();
572
+
573
+ return result;
574
+ },
575
+
576
+
577
+ async runSiLUGated(dev, gate, up) {
578
+ if (!runSiLU) {
579
+ return references.siluGatedRef(gate, up);
580
+ }
581
+
582
+ const gateBuf = makeBuffer(gate);
583
+ const upBuf = makeBuffer(up);
584
+
585
+ const gateTensor = createTensor(gateBuf, 'f32', [gate.length], 'silu_gate');
586
+ const upTensor = createTensor(upBuf, 'f32', [up.length], 'silu_up');
587
+
588
+ const resultTensor = await runSiLU(upTensor, {
589
+ size: up.length,
590
+ gate: gateTensor,
591
+ swigluLimit: null,
592
+ inputActivation: 'identity',
593
+ });
594
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, up.length * 4));
595
+
596
+ gateBuf.destroy();
597
+ upBuf.destroy();
598
+ resultTensor.buffer.destroy();
599
+
600
+ return result;
601
+ },
602
+
603
+
604
+ async runGather(dev, embeddings, indices, vocabSize, embedDim) {
605
+ if (!runGather) {
606
+ return references.gatherRef(embeddings, indices, vocabSize, embedDim);
607
+ }
608
+
609
+ const embBuf = makeBuffer(embeddings);
610
+ const idxBuf = makeBuffer(indices);
611
+ const numTokens = indices.length;
612
+
613
+ const embTensor = createTensor(embBuf, 'f32', [vocabSize, embedDim], 'gather_embeddings');
614
+ const idxTensor = createTensor(idxBuf, 'u32', [numTokens], 'gather_indices');
615
+
616
+ // Test data uses standard [vocab_size, hidden_size] layout, not GGUF [hidden_size, vocab_size]
617
+ const resultTensor = await runGather(idxTensor.buffer, embTensor.buffer, numTokens, embedDim, vocabSize, { transpose: false, embeddingDtype: 'f32', outputDtype: 'f32' });
618
+ let result;
619
+ if (resultTensor.dtype === 'f16') {
620
+ const rawData = await readBufferData(resultTensor.buffer, numTokens * embedDim * 2);
621
+ const u16View = new Uint16Array(rawData);
622
+ result = new Float32Array(u16View.length);
623
+ for (let i = 0; i < u16View.length; i++) {
624
+ result[i] = f16ToF32(u16View[i]);
625
+ }
626
+ } else {
627
+ result = new Float32Array(
628
+ await readBufferData(resultTensor.buffer, numTokens * embedDim * 4)
629
+ );
630
+ }
631
+
632
+ embBuf.destroy();
633
+ idxBuf.destroy();
634
+ resultTensor.buffer.destroy();
635
+
636
+ return result;
637
+ },
638
+
639
+
640
+ async runResidual(dev, x, residual, options = {}) {
641
+ if (!runResidualAdd) {
642
+ return options.outputScale == null
643
+ ? references.residualAddRef(x, residual)
644
+ : references.outputScaledResidualAddRef(x, residual, options.outputScale);
645
+ }
646
+
647
+ const xBuf = makeBuffer(x);
648
+ const resBuf = makeBuffer(residual);
649
+ const size = x.length;
650
+
651
+ const xTensor = createTensor(xBuf, 'f32', [size], 'residual_x');
652
+ const resTensor = createTensor(resBuf, 'f32', [size], 'residual_res');
653
+
654
+ const resultTensor = await runResidualAdd(xTensor, resTensor, size, {
655
+ outputScale: options.outputScale,
656
+ });
657
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, size * 4));
658
+
659
+ xBuf.destroy();
660
+ resBuf.destroy();
661
+ resultTensor.buffer.destroy();
662
+
663
+ return result;
664
+ },
665
+
666
+
667
+ async runBiasAdd(dev, data, bias, numTokens, dim) {
668
+ if (!runBiasAdd) {
669
+ const result = new Float32Array(data);
670
+ for (let t = 0; t < numTokens; t++) {
671
+ const rowOffset = t * dim;
672
+ for (let d = 0; d < dim; d++) {
673
+ result[rowOffset + d] += bias[d];
674
+ }
675
+ }
676
+ return result;
677
+ }
678
+
679
+ const dataBuf = makeBuffer(data);
680
+ const biasBuf = makeBuffer(bias);
681
+ const dataTensor = createTensor(dataBuf, 'f32', [numTokens, dim], 'bias_add_data');
682
+ const biasTensor = createTensor(biasBuf, 'f32', [dim], 'bias_add_bias');
683
+
684
+ const resultTensor = await runBiasAdd(dataTensor, biasTensor, numTokens, dim);
685
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, numTokens * dim * 4));
686
+
687
+ dataBuf.destroy();
688
+ biasBuf.destroy();
689
+ if (resultTensor.buffer !== dataBuf) {
690
+ resultTensor.buffer.destroy();
691
+ }
692
+
693
+ return result;
694
+ },
695
+
696
+
697
+ async runLayerNorm(dev, input, weight, bias, batchSize, hiddenSize, eps = 1e-5) {
698
+ if (!runLayerNorm) {
699
+ const output = new Float32Array(batchSize * hiddenSize);
700
+ for (let b = 0; b < batchSize; b++) {
701
+ const base = b * hiddenSize;
702
+ let mean = 0;
703
+ for (let i = 0; i < hiddenSize; i++) {
704
+ mean += input[base + i];
705
+ }
706
+ mean /= hiddenSize;
707
+ let varSum = 0;
708
+ for (let i = 0; i < hiddenSize; i++) {
709
+ const diff = input[base + i] - mean;
710
+ varSum += diff * diff;
711
+ }
712
+ const invStd = 1 / Math.sqrt(varSum / hiddenSize + eps);
713
+ for (let i = 0; i < hiddenSize; i++) {
714
+ const norm = (input[base + i] - mean) * invStd;
715
+ output[base + i] = norm * weight[i] + bias[i];
716
+ }
717
+ }
718
+ return output;
719
+ }
720
+
721
+ const inputBuf = makeBuffer(input);
722
+ const weightBuf = makeBuffer(weight);
723
+ const biasBuf = makeBuffer(bias);
724
+ const inputTensor = createTensor(inputBuf, 'f32', [batchSize, hiddenSize], 'layernorm_input');
725
+
726
+ const resultTensor = await runLayerNorm(inputTensor, weightBuf, biasBuf, eps, {
727
+ batchSize,
728
+ hiddenSize,
729
+ });
730
+ const result = await readTensorToFloat32(resultTensor, batchSize * hiddenSize);
731
+
732
+ inputBuf.destroy();
733
+
734
+ weightBuf.destroy();
735
+
736
+ biasBuf.destroy();
737
+
738
+ resultTensor.buffer.destroy();
739
+
740
+
741
+
742
+ return result;
743
+
744
+ },
745
+
746
+
747
+
748
+ async runLayerNormBackward(dev, input, weight, gradOutput, numTokens, hiddenSize, eps = 1e-5) {
749
+
750
+ if (!runLayerNormBackward) {
751
+
752
+ // Reference implementation
753
+
754
+ const output = new Float32Array(numTokens * hiddenSize);
755
+
756
+ for (let b = 0; b < numTokens; b++) {
757
+
758
+ const base = b * hiddenSize;
759
+
760
+ let mean = 0;
761
+
762
+ for (let i = 0; i < hiddenSize; i++) {
763
+
764
+ mean += input[base + i];
765
+
766
+ }
767
+
768
+ mean /= hiddenSize;
769
+
770
+ let varSum = 0;
771
+
772
+ let sumGY = 0;
773
+
774
+ let sumGYX = 0;
775
+
776
+ for (let i = 0; i < hiddenSize; i++) {
777
+
778
+ const x = input[base + i];
779
+
780
+ const diff = x - mean;
781
+
782
+ varSum += diff * diff;
783
+
784
+ const gy = gradOutput[base + i] * weight[i];
785
+
786
+ sumGY += gy;
787
+
788
+ sumGYX += gy * diff;
789
+
790
+ }
791
+
792
+ const invStd = 1 / Math.sqrt(varSum / hiddenSize + eps);
793
+
794
+ const invStd2 = invStd * invStd;
795
+
796
+ for (let i = 0; i < hiddenSize; i++) {
797
+
798
+ const x = input[base + i];
799
+
800
+ const gy = gradOutput[base + i] * weight[i];
801
+
802
+ output[base + i] = invStd * (gy - (sumGY + (x - mean) * invStd2 * sumGYX) / hiddenSize);
803
+
804
+ }
805
+
806
+ }
807
+
808
+ return output;
809
+
810
+ }
811
+
812
+
813
+
814
+ const inputBuf = makeBuffer(input);
815
+
816
+ const weightBuf = makeBuffer(weight);
817
+
818
+ const gradBuf = makeBuffer(gradOutput);
819
+
820
+ const inputTensor = createTensor(inputBuf, 'f32', [numTokens, hiddenSize], 'layernorm_bw_input');
821
+
822
+ const weightTensor = createTensor(weightBuf, 'f32', [hiddenSize], 'layernorm_bw_weight');
823
+
824
+ const gradTensor = createTensor(gradBuf, 'f32', [numTokens, hiddenSize], 'layernorm_bw_grad');
825
+
826
+
827
+
828
+ const result = await runLayerNormBackward(inputTensor, weightTensor, gradTensor, {
829
+ numTokens,
830
+ hiddenSize,
831
+ eps,
832
+ });
833
+ const resultTensor = result?.gradInput || result;
834
+ const resultData = await readTensorToFloat32(resultTensor, numTokens * hiddenSize);
835
+
836
+ inputBuf.destroy();
837
+ weightBuf.destroy();
838
+ gradBuf.destroy();
839
+ if (result?.gradInput?.buffer) {
840
+ result.gradInput.buffer.destroy();
841
+ } else if (result?.buffer) {
842
+ result.buffer.destroy();
843
+ }
844
+ result?.gradWeight?.buffer?.destroy();
845
+ result?.gradBias?.buffer?.destroy();
846
+
847
+ return resultData;
848
+
849
+ },
850
+
851
+
852
+
853
+ async runGroupNorm(dev, input, weight, bias, channels, height, width, numGroups, eps = 1e-5) {
854
+ if (!runGroupNorm) {
855
+ const output = new Float32Array(channels * height * width);
856
+ const channelsPerGroup = Math.floor(channels / numGroups);
857
+ for (let g = 0; g < numGroups; g++) {
858
+ const cStart = g * channelsPerGroup;
859
+ const cEnd = cStart + channelsPerGroup;
860
+ let mean = 0;
861
+ let count = 0;
862
+ for (let c = cStart; c < cEnd; c++) {
863
+ const base = c * height * width;
864
+ for (let i = 0; i < height * width; i++) {
865
+ mean += input[base + i];
866
+ count++;
867
+ }
868
+ }
869
+ mean /= count;
870
+ let varSum = 0;
871
+ for (let c = cStart; c < cEnd; c++) {
872
+ const base = c * height * width;
873
+ for (let i = 0; i < height * width; i++) {
874
+ const diff = input[base + i] - mean;
875
+ varSum += diff * diff;
876
+ }
877
+ }
878
+ const invStd = 1 / Math.sqrt(varSum / count + eps);
879
+ for (let c = cStart; c < cEnd; c++) {
880
+ const base = c * height * width;
881
+ for (let i = 0; i < height * width; i++) {
882
+ const norm = (input[base + i] - mean) * invStd;
883
+ output[base + i] = norm * weight[c] + bias[c];
884
+ }
885
+ }
886
+ }
887
+ return output;
888
+ }
889
+
890
+ const inputBuf = makeBuffer(input);
891
+ const weightBuf = makeBuffer(weight);
892
+ const biasBuf = makeBuffer(bias);
893
+ const inputTensor = createTensor(inputBuf, 'f32', [channels, height, width], 'groupnorm_input');
894
+
895
+ const resultTensor = await runGroupNorm(inputTensor, weightBuf, biasBuf, {
896
+ channels,
897
+ height,
898
+ width,
899
+ numGroups,
900
+ eps,
901
+ });
902
+ const result = await readTensorToFloat32(resultTensor, channels * height * width);
903
+
904
+ inputBuf.destroy();
905
+ weightBuf.destroy();
906
+ biasBuf.destroy();
907
+ resultTensor.buffer.destroy();
908
+
909
+ return result;
910
+ },
911
+
912
+
913
+ async runModulate(dev, input, mod, numTokens, hiddenSize, options = {}) {
914
+ const { scaleOffset = 0, shiftOffset = hiddenSize, gateOffset = hiddenSize * 2, hasGate = false, addOne = true } = options;
915
+ if (!runModulate) {
916
+ const output = new Float32Array(numTokens * hiddenSize);
917
+ for (let t = 0; t < numTokens; t++) {
918
+ const base = t * hiddenSize;
919
+ for (let d = 0; d < hiddenSize; d++) {
920
+ const rawScale = mod[scaleOffset + d];
921
+ const shift = mod[shiftOffset + d];
922
+ const scale = addOne ? 1 + rawScale : rawScale;
923
+ let value = input[base + d] * scale + shift;
924
+ if (hasGate) {
925
+ value *= mod[gateOffset + d];
926
+ }
927
+ output[base + d] = value;
928
+ }
929
+ }
930
+ return output;
931
+ }
932
+
933
+ const inputBuf = makeBuffer(input);
934
+ const modBuf = makeBuffer(mod);
935
+ const inputTensor = createTensor(inputBuf, 'f32', [numTokens, hiddenSize], 'modulate_input');
936
+ const modTensor = createTensor(modBuf, 'f32', [mod.length], 'modulate_params');
937
+
938
+ const resultTensor = await runModulate(inputTensor, modTensor, {
939
+ numTokens,
940
+ hiddenSize,
941
+ scaleOffset,
942
+ shiftOffset,
943
+ gateOffset,
944
+ hasGate,
945
+ addOne,
946
+ });
947
+ const result = await readTensorToFloat32(resultTensor, numTokens * hiddenSize);
948
+
949
+ inputBuf.destroy();
950
+ modBuf.destroy();
951
+ resultTensor.buffer.destroy();
952
+
953
+ return result;
954
+ },
955
+
956
+
957
+ async runConv2D(dev, input, weight, bias, options = {}) {
958
+ if (!runConv2D) {
959
+ const {
960
+ inChannels,
961
+ outChannels,
962
+ height,
963
+ width,
964
+ kernelH,
965
+ kernelW,
966
+ stride = 1,
967
+ pad = 0,
968
+ } = options;
969
+ const outHeight = Math.floor((height + pad * 2 - kernelH) / stride) + 1;
970
+ const outWidth = Math.floor((width + pad * 2 - kernelW) / stride) + 1;
971
+ const output = new Float32Array(outChannels * outHeight * outWidth);
972
+ for (let oc = 0; oc < outChannels; oc++) {
973
+ for (let oy = 0; oy < outHeight; oy++) {
974
+ for (let ox = 0; ox < outWidth; ox++) {
975
+ let sum = bias ? bias[oc] : 0;
976
+ for (let ic = 0; ic < inChannels; ic++) {
977
+ for (let ky = 0; ky < kernelH; ky++) {
978
+ const inY = oy * stride + ky - pad;
979
+ if (inY < 0 || inY >= height) continue;
980
+ for (let kx = 0; kx < kernelW; kx++) {
981
+ const inX = ox * stride + kx - pad;
982
+ if (inX < 0 || inX >= width) continue;
983
+ const inputIdx = (ic * height + inY) * width + inX;
984
+ const weightIdx = (((oc * inChannels + ic) * kernelH + ky) * kernelW + kx);
985
+ sum += input[inputIdx] * weight[weightIdx];
986
+ }
987
+ }
988
+ }
989
+ output[(oc * outHeight + oy) * outWidth + ox] = sum;
990
+ }
991
+ }
992
+ }
993
+ return output;
994
+ }
995
+
996
+ const inputBuf = makeBuffer(input);
997
+ const weightBuf = makeBuffer(weight);
998
+ const biasBuf = bias ? makeBuffer(bias) : null;
999
+ const inputTensor = createTensor(inputBuf, 'f32', [options.inChannels, options.height, options.width], 'conv2d_input');
1000
+
1001
+ const resultTensor = await runConv2D(inputTensor, weightBuf, biasBuf, options);
1002
+ const stride = options.stride ?? 1;
1003
+ const pad = options.pad ?? 0;
1004
+ const outHeight = Math.floor((options.height + pad * 2 - options.kernelH) / stride) + 1;
1005
+ const outWidth = Math.floor((options.width + pad * 2 - options.kernelW) / stride) + 1;
1006
+ const result = await readTensorToFloat32(resultTensor, options.outChannels * outHeight * outWidth);
1007
+
1008
+ inputBuf.destroy();
1009
+ weightBuf.destroy();
1010
+ biasBuf?.destroy();
1011
+ resultTensor.buffer.destroy();
1012
+
1013
+ return result;
1014
+ },
1015
+
1016
+
1017
+ async runPixelShuffle(dev, input, options = {}) {
1018
+ if (!runPixelShuffle) {
1019
+ const {
1020
+ outChannels,
1021
+ outHeight,
1022
+ outWidth,
1023
+ gridWidth,
1024
+ patchSize,
1025
+ patchChannels,
1026
+ } = options;
1027
+ const output = new Float32Array(outChannels * outHeight * outWidth);
1028
+ const spatial = outHeight * outWidth;
1029
+ for (let idx = 0; idx < output.length; idx++) {
1030
+ const c = Math.floor(idx / spatial);
1031
+ const rem = idx - c * spatial;
1032
+ const y = Math.floor(rem / outWidth);
1033
+ const x = rem - y * outWidth;
1034
+ const gridY = Math.floor(y / patchSize);
1035
+ const gridX = Math.floor(x / patchSize);
1036
+ const subY = y - gridY * patchSize;
1037
+ const subX = x - gridX * patchSize;
1038
+ const tokenIdx = gridY * gridWidth + gridX;
1039
+ const patchIdx = (subY * patchSize + subX) * outChannels + c;
1040
+ const inputIdx = tokenIdx * patchChannels + patchIdx;
1041
+ output[idx] = input[inputIdx];
1042
+ }
1043
+ return output;
1044
+ }
1045
+
1046
+ const inputBuf = makeBuffer(input);
1047
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'pixel_shuffle_input');
1048
+ const resultTensor = await runPixelShuffle(inputTensor, options);
1049
+ const result = await readTensorToFloat32(resultTensor, options.outChannels * options.outHeight * options.outWidth);
1050
+
1051
+ inputBuf.destroy();
1052
+ resultTensor.buffer.destroy();
1053
+
1054
+ return result;
1055
+ },
1056
+
1057
+
1058
+ async runUpsample2D(dev, input, options = {}) {
1059
+ if (!runUpsample2D) {
1060
+ const { channels, inHeight, inWidth, outHeight, outWidth, scale } = options;
1061
+ const output = new Float32Array(channels * outHeight * outWidth);
1062
+ for (let c = 0; c < channels; c++) {
1063
+ for (let oy = 0; oy < outHeight; oy++) {
1064
+ for (let ox = 0; ox < outWidth; ox++) {
1065
+ const inY = Math.floor(oy / scale);
1066
+ const inX = Math.floor(ox / scale);
1067
+ const inIdx = (c * inHeight + inY) * inWidth + inX;
1068
+ const outIdx = (c * outHeight + oy) * outWidth + ox;
1069
+ output[outIdx] = input[inIdx];
1070
+ }
1071
+ }
1072
+ }
1073
+ return output;
1074
+ }
1075
+
1076
+ const height = Number.isFinite(options.height) ? options.height : options.inHeight;
1077
+ const width = Number.isFinite(options.width) ? options.width : options.inWidth;
1078
+ const inputBuf = makeBuffer(input);
1079
+ const inputTensor = createTensor(inputBuf, 'f32', [options.channels, height, width], 'upsample2d_input');
1080
+ const resultTensor = await runUpsample2D(inputTensor, options);
1081
+ const result = await readTensorToFloat32(resultTensor, options.channels * options.outHeight * options.outWidth);
1082
+
1083
+ inputBuf.destroy();
1084
+ resultTensor.buffer.destroy();
1085
+
1086
+ return result;
1087
+ },
1088
+
1089
+
1090
+ async runTranspose(dev, input, rows, cols) {
1091
+ if (!runTranspose) {
1092
+ const output = new Float32Array(rows * cols);
1093
+ for (let r = 0; r < rows; r++) {
1094
+ for (let c = 0; c < cols; c++) {
1095
+ output[c * rows + r] = input[r * cols + c];
1096
+ }
1097
+ }
1098
+ return output;
1099
+ }
1100
+
1101
+ const inputBuf = makeBuffer(input);
1102
+ const inputTensor = createTensor(inputBuf, 'f32', [rows, cols], 'transpose_input');
1103
+ const resultTensor = await runTranspose(inputTensor, rows, cols);
1104
+ const result = await readTensorToFloat32(resultTensor, rows * cols);
1105
+
1106
+ inputBuf.destroy();
1107
+ resultTensor.buffer.destroy();
1108
+
1109
+ return result;
1110
+ },
1111
+
1112
+
1113
+ async runKVQuantize(dev, keys, values, numKVHeads, headDim, numTokens, mode = 'int8') {
1114
+ if (!runKVQuantize) {
1115
+ const packedStride = Math.ceil(headDim / 4);
1116
+ return {
1117
+ packedStride,
1118
+ outputK: new Uint32Array(numTokens * numKVHeads * packedStride),
1119
+ outputV: new Uint32Array(numTokens * numKVHeads * packedStride),
1120
+ scalesK: new Uint16Array(numTokens * numKVHeads),
1121
+ scalesV: new Uint16Array(numTokens * numKVHeads),
1122
+ };
1123
+ }
1124
+
1125
+ const packedStride = Math.ceil(headDim / 4);
1126
+ const keyBuf = makeBuffer(keys, GPUBufferUsage.STORAGE);
1127
+ const valBuf = makeBuffer(values, GPUBufferUsage.STORAGE);
1128
+ const outputSize = numTokens * numKVHeads * packedStride * 4;
1129
+ const scalesSize = numTokens * numKVHeads * 2;
1130
+ const scalesSizeAligned = Math.ceil(scalesSize / 4) * 4;
1131
+ const outputK = makeBuffer(new Uint8Array(outputSize));
1132
+ const outputV = makeBuffer(new Uint8Array(outputSize));
1133
+ const scalesK = makeBuffer(new Uint8Array(scalesSizeAligned));
1134
+ const scalesV = makeBuffer(new Uint8Array(scalesSizeAligned));
1135
+
1136
+ await runKVQuantize(
1137
+ keyBuf,
1138
+ valBuf,
1139
+ outputK,
1140
+ outputV,
1141
+ scalesK,
1142
+ scalesV,
1143
+ {
1144
+ numKVHeads,
1145
+ headDim,
1146
+ startPos: 0,
1147
+ numTokens,
1148
+ packedStride,
1149
+ mode,
1150
+ }
1151
+ );
1152
+
1153
+ const outK = new Uint32Array(await readBufferData(outputK, outputSize));
1154
+ const outV = new Uint32Array(await readBufferData(outputV, outputSize));
1155
+ const outScalesKFull = new Uint16Array(await readBufferData(scalesK, scalesSizeAligned));
1156
+ const outScalesVFull = new Uint16Array(await readBufferData(scalesV, scalesSizeAligned));
1157
+ const outScalesK = outScalesKFull.slice(0, numTokens * numKVHeads);
1158
+ const outScalesV = outScalesVFull.slice(0, numTokens * numKVHeads);
1159
+
1160
+ keyBuf.destroy();
1161
+ valBuf.destroy();
1162
+ outputK.destroy();
1163
+ outputV.destroy();
1164
+ scalesK.destroy();
1165
+ scalesV.destroy();
1166
+
1167
+ return { packedStride, outputK: outK, outputV: outV, scalesK: outScalesK, scalesV: outScalesV };
1168
+ },
1169
+
1170
+
1171
+ async runCrossEntropyLoss(dev, softmax, targets, numTokens, vocabSize) {
1172
+ if (!runCrossEntropyLoss) {
1173
+ const output = new Float32Array(numTokens);
1174
+ for (let t = 0; t < numTokens; t++) {
1175
+ const target = targets[t];
1176
+ if (target >= vocabSize) {
1177
+ output[t] = 0;
1178
+ continue;
1179
+ }
1180
+ const p = Math.max(softmax[t * vocabSize + target], 1e-9);
1181
+ output[t] = -Math.log(p);
1182
+ }
1183
+ return output;
1184
+ }
1185
+
1186
+ const softmaxBuf = makeBuffer(softmax);
1187
+ const targetsBuf = makeBuffer(targets);
1188
+ const softmaxTensor = createTensor(softmaxBuf, 'f32', [numTokens, vocabSize], 'cross_entropy_softmax');
1189
+ const targetsTensor = createTensor(targetsBuf, 'u32', [numTokens], 'cross_entropy_targets');
1190
+
1191
+ const resultTensor = await runCrossEntropyLoss(softmaxTensor, targetsTensor, { numTokens, vocabSize });
1192
+ const result = await readTensorToFloat32(resultTensor, numTokens);
1193
+
1194
+ softmaxBuf.destroy();
1195
+ targetsBuf.destroy();
1196
+ resultTensor.buffer.destroy();
1197
+
1198
+ return result;
1199
+ },
1200
+
1201
+
1202
+ async runCrossEntropyBackward(dev, softmax, targets, gradOutput, numTokens, vocabSize) {
1203
+ if (!runCrossEntropyBackward) {
1204
+ const output = new Float32Array(numTokens * vocabSize);
1205
+ for (let t = 0; t < numTokens; t++) {
1206
+ const target = targets[t];
1207
+ for (let c = 0; c < vocabSize; c++) {
1208
+ let grad = softmax[t * vocabSize + c];
1209
+ if (c === target) grad -= 1;
1210
+ output[t * vocabSize + c] = grad * gradOutput[t];
1211
+ }
1212
+ }
1213
+ return output;
1214
+ }
1215
+
1216
+ const softmaxBuf = makeBuffer(softmax);
1217
+ const targetsBuf = makeBuffer(targets);
1218
+ const gradBuf = makeBuffer(gradOutput);
1219
+ const softmaxTensor = createTensor(softmaxBuf, 'f32', [numTokens, vocabSize], 'cross_entropy_softmax');
1220
+ const targetsTensor = createTensor(targetsBuf, 'u32', [numTokens], 'cross_entropy_targets');
1221
+ const gradTensor = createTensor(gradBuf, 'f32', [numTokens], 'cross_entropy_grad');
1222
+
1223
+ const resultTensor = await runCrossEntropyBackward(softmaxTensor, targetsTensor, gradTensor, { numTokens, vocabSize });
1224
+ const result = await readTensorToFloat32(resultTensor, numTokens * vocabSize);
1225
+
1226
+ softmaxBuf.destroy();
1227
+ targetsBuf.destroy();
1228
+ gradBuf.destroy();
1229
+ resultTensor.buffer.destroy();
1230
+
1231
+ return result;
1232
+ },
1233
+
1234
+ async runBiasAddBackward(dev, gradOutput, numTokens, dim) {
1235
+ if (!runBiasAddBackward) return null;
1236
+ const gBuf = makeBuffer(gradOutput);
1237
+ const resultTensor = await runBiasAddBackward(createTensor(gBuf, 'f32', [numTokens, dim], 'bias_bw_in'), { numTokens, dim });
1238
+ const result = await readTensorToFloat32(resultTensor, dim);
1239
+ gBuf.destroy();
1240
+ resultTensor.buffer.destroy();
1241
+ return result;
1242
+ },
1243
+
1244
+ async runUpsample2DBackward(dev, gradOutput, options) {
1245
+ if (!runUpsample2DBackward) return null;
1246
+ const gBuf = makeBuffer(gradOutput);
1247
+ const resultTensor = await runUpsample2DBackward(createTensor(gBuf, 'f32', [options.channels, options.outHeight, options.outWidth], 'upsample_bw_in'), options);
1248
+ const result = await readTensorToFloat32(resultTensor, options.channels * options.inHeight * options.inWidth);
1249
+ gBuf.destroy();
1250
+ resultTensor.buffer.destroy();
1251
+ return result;
1252
+ },
1253
+
1254
+ async runPixelShuffleBackward(dev, gradOutput, options) {
1255
+ if (!runPixelShuffleBackward) return null;
1256
+ const gBuf = makeBuffer(gradOutput);
1257
+ const resultTensor = await runPixelShuffleBackward(createTensor(gBuf, 'f32', [options.outChannels, options.outHeight, options.outWidth], 'pixel_bw_in'), options);
1258
+ const result = await readTensorToFloat32(resultTensor, options.gridWidth * options.gridHeight * options.patchChannels);
1259
+ gBuf.destroy();
1260
+ resultTensor.buffer.destroy();
1261
+ return result;
1262
+ },
1263
+
1264
+ async runGroupNormBackward(dev, input, weight, gradOutput, options) {
1265
+ if (!runGroupNormBackward) return null;
1266
+ const iBuf = makeBuffer(input);
1267
+ const wBuf = makeBuffer(weight);
1268
+ const gBuf = makeBuffer(gradOutput);
1269
+ const resultTensor = await runGroupNormBackward(
1270
+ createTensor(iBuf, 'f32', [options.channels, options.height, options.width], 'gn_bw_i'),
1271
+ createTensor(wBuf, 'f32', [options.channels], 'gn_bw_w'),
1272
+ createTensor(gBuf, 'f32', [options.channels, options.height, options.width], 'gn_bw_g'),
1273
+ options
1274
+ );
1275
+ const result = await readTensorToFloat32(resultTensor, options.channels * options.height * options.width);
1276
+ iBuf.destroy();
1277
+ wBuf.destroy();
1278
+ gBuf.destroy();
1279
+ resultTensor.buffer.destroy();
1280
+ return result;
1281
+ },
1282
+
1283
+ async runConv2DBackward(dev, input, weight, gradOutput, options) {
1284
+ if (!runConv2DBackward) return null;
1285
+ const iBuf = makeBuffer(input);
1286
+ const wBuf = makeBuffer(weight);
1287
+ const gBuf = makeBuffer(gradOutput);
1288
+ const result = await runConv2DBackward(
1289
+ createTensor(iBuf, 'f32', [options.inChannels, options.height, options.width], 'conv_bw_i'),
1290
+ createTensor(wBuf, 'f32', [options.outChannels, options.inChannels, options.kernelH, options.kernelW], 'conv_bw_w'),
1291
+ createTensor(gBuf, 'f32', [options.outChannels, options.outHeight, options.outWidth], 'conv_bw_g'),
1292
+ options
1293
+ );
1294
+ const gradInput = await readTensorToFloat32(result.gradInput, options.inChannels * options.height * options.width);
1295
+ const gradWeight = await readTensorToFloat32(result.gradWeight, options.outChannels * options.inChannels * options.kernelH * options.kernelW);
1296
+ iBuf.destroy();
1297
+ wBuf.destroy();
1298
+ gBuf.destroy();
1299
+ result.gradInput.buffer.destroy();
1300
+ result.gradWeight.buffer.destroy();
1301
+ return { gradInput, gradWeight };
1302
+ },
1303
+
1304
+ async runDequantQ4K(dev, quantized, numBlocks) {
1305
+ if (!dequantize) {
1306
+ throw new Error('dequantize kernel not available');
1307
+ }
1308
+
1309
+ const qBuf = makeBuffer(quantized, GPUBufferUsage.STORAGE);
1310
+ const outTensor = await dequantize(qBuf, numBlocks, { outputDtype: 'f32', useVec4: false });
1311
+ const out = new Float32Array(await readBufferData(outTensor.buffer, numBlocks * 256 * 4));
1312
+
1313
+ qBuf.destroy();
1314
+ outTensor.buffer.destroy();
1315
+
1316
+ return out;
1317
+ },
1318
+
1319
+
1320
+ async runDequantQ4K_Vec4(dev, quantized, numBlocks) {
1321
+ if (!dequantize) {
1322
+ throw new Error('dequantize kernel not available');
1323
+ }
1324
+
1325
+ const qBuf = makeBuffer(quantized, GPUBufferUsage.STORAGE);
1326
+ const outTensor = await dequantize(qBuf, numBlocks, { outputDtype: 'f32', useVec4: true });
1327
+ const out = new Float32Array(await readBufferData(outTensor.buffer, numBlocks * 256 * 4));
1328
+
1329
+ qBuf.destroy();
1330
+ outTensor.buffer.destroy();
1331
+
1332
+ return out;
1333
+ },
1334
+
1335
+ async runDequantQ4K_F16(dev, quantized, numBlocks) {
1336
+ if (!dequantize) {
1337
+ throw new Error('dequantize kernel not available');
1338
+ }
1339
+
1340
+ const qBuf = makeBuffer(quantized, GPUBufferUsage.STORAGE);
1341
+ // Use F16 output and vec4=true (default) to match production loader path
1342
+ const outTensor = await dequantize(qBuf, numBlocks, { outputDtype: 'f16', useVec4: true });
1343
+
1344
+ // Read back F16 data and convert to F32 for comparison
1345
+ const f16Bytes = numBlocks * 256 * 2; // F16 = 2 bytes per element
1346
+ const rawData = await readBufferData(outTensor.buffer, f16Bytes);
1347
+ const u16 = new Uint16Array(rawData);
1348
+ const out = new Float32Array(numBlocks * 256);
1349
+
1350
+ // Convert F16 to F32
1351
+ for (let i = 0; i < u16.length; i++) {
1352
+ const h = u16[i];
1353
+ const sign = (h >> 15) & 1;
1354
+ const exp = (h >> 10) & 0x1F;
1355
+ const mant = h & 0x3FF;
1356
+ let f;
1357
+ if (exp === 0) {
1358
+ f = mant === 0 ? 0 : Math.pow(2, -14) * (mant / 1024);
1359
+ } else if (exp === 31) {
1360
+ f = mant === 0 ? Infinity : NaN;
1361
+ } else {
1362
+ f = Math.pow(2, exp - 15) * (1 + mant / 1024);
1363
+ }
1364
+ out[i] = sign ? -f : f;
1365
+ }
1366
+
1367
+ qBuf.destroy();
1368
+ outTensor.buffer.destroy();
1369
+
1370
+ return out;
1371
+ },
1372
+
1373
+
1374
+ async runAttention(dev, Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim, mask = null) {
1375
+ if (!runAttention) {
1376
+ // Fallback to reference if kernel not available
1377
+ return references.attentionRef(Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim, mask);
1378
+ }
1379
+
1380
+ // Create GPU buffers
1381
+ const qBuf = makeBuffer(Q);
1382
+ const kBuf = makeBuffer(K);
1383
+ const vBuf = makeBuffer(V);
1384
+ const maskBuf = mask ? makeBuffer(mask) : null;
1385
+ const isCausal = !!mask;
1386
+
1387
+ const qTensor = createTensor(qBuf, 'f32', [seqLen, numHeads, headDim], 'attn_q');
1388
+ const kTensor = createTensor(kBuf, 'f32', [kvLen, numKVHeads, headDim], 'attn_k');
1389
+ const vTensor = createTensor(vBuf, 'f32', [kvLen, numKVHeads, headDim], 'attn_v');
1390
+
1391
+ // Run attention via kernel selector (handles tier selection automatically)
1392
+ const resultTensor = await runAttention(qTensor, kTensor, vTensor, maskBuf, numHeads, headDim, {
1393
+ seqLen,
1394
+ kvLen,
1395
+ numKVHeads,
1396
+ scale: 1 / Math.sqrt(headDim),
1397
+ causal: isCausal,
1398
+ });
1399
+
1400
+ // Read back result
1401
+ const out = new Float32Array(await readBufferData(resultTensor.buffer, seqLen * numHeads * headDim * 4));
1402
+ qBuf.destroy();
1403
+ kBuf.destroy();
1404
+ vBuf.destroy();
1405
+ maskBuf?.destroy();
1406
+ resultTensor.buffer.destroy();
1407
+ return out;
1408
+ },
1409
+
1410
+
1411
+
1412
+ async runAttentionTiered(dev, Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim) {
1413
+ if (!runAttentionTiered) {
1414
+ return references.attentionRef(Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim, null);
1415
+ }
1416
+
1417
+ const qBuf = makeBuffer(toF16Array(Q));
1418
+ const kBuf = makeBuffer(toF16Array(K));
1419
+ const vBuf = makeBuffer(toF16Array(V));
1420
+ const qTensor = createTensor(qBuf, 'f16', [seqLen, numHeads, headDim], 'tiered_q');
1421
+ const hotK = createTensor(kBuf, 'f16', [kvLen, numKVHeads, headDim], 'tiered_hot_k');
1422
+ const hotV = createTensor(vBuf, 'f16', [kvLen, numKVHeads, headDim], 'tiered_hot_v');
1423
+ const coldK = createTensor(makeBuffer(new Uint16Array(2)), 'f16', [1], 'tiered_cold_k');
1424
+ const coldV = createTensor(makeBuffer(new Uint16Array(2)), 'f16', [1], 'tiered_cold_v');
1425
+
1426
+ const resultTensor = await runAttentionTiered(
1427
+ qTensor,
1428
+ hotK,
1429
+ hotV,
1430
+ coldK,
1431
+ coldV,
1432
+ numHeads,
1433
+ headDim,
1434
+ {
1435
+ seqLen,
1436
+ hotLen: kvLen,
1437
+ coldLen: 0,
1438
+ numKVHeads,
1439
+ causal: false,
1440
+ }
1441
+ );
1442
+ const result = await readTensorToFloat32(resultTensor, seqLen * numHeads * headDim);
1443
+
1444
+ qBuf.destroy();
1445
+ kBuf.destroy();
1446
+ vBuf.destroy();
1447
+ coldK.buffer.destroy();
1448
+ coldV.buffer.destroy();
1449
+ resultTensor.buffer.destroy();
1450
+
1451
+ return result;
1452
+ },
1453
+
1454
+
1455
+ async runAttentionTieredQuant(dev, Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim) {
1456
+ if (!runAttentionTieredQuant) {
1457
+ return references.attentionRef(Q, K, V, seqLen, kvLen, numHeads, numKVHeads, headDim, null);
1458
+ }
1459
+
1460
+ const qBuf = makeBuffer(Q);
1461
+ const kBuf = makeBuffer(toF16Array(K));
1462
+ const vBuf = makeBuffer(toF16Array(V));
1463
+ const qTensor = createTensor(qBuf, 'f32', [seqLen, numHeads, headDim], 'tieredq_q');
1464
+ const hotK = createTensor(kBuf, 'f16', [kvLen, numKVHeads, headDim], 'tieredq_hot_k');
1465
+ const hotV = createTensor(vBuf, 'f16', [kvLen, numKVHeads, headDim], 'tieredq_hot_v');
1466
+
1467
+ const coldPackedK = makeBuffer(new Uint8Array(4));
1468
+ const coldPackedV = makeBuffer(new Uint8Array(4));
1469
+ const coldScalesK = makeBuffer(new Uint8Array(4));
1470
+ const coldScalesV = makeBuffer(new Uint8Array(4));
1471
+
1472
+ const resultTensor = await runAttentionTieredQuant(
1473
+ qTensor,
1474
+ hotK,
1475
+ hotV,
1476
+ coldPackedK,
1477
+ coldPackedV,
1478
+ coldScalesK,
1479
+ coldScalesV,
1480
+ numHeads,
1481
+ headDim,
1482
+ {
1483
+ seqLen,
1484
+ hotLen: kvLen,
1485
+ coldLen: 0,
1486
+ numKVHeads,
1487
+ packedStride: 1,
1488
+ mode: 'int8',
1489
+ causal: false,
1490
+ }
1491
+ );
1492
+ const result = await readTensorToFloat32(resultTensor, seqLen * numHeads * headDim);
1493
+
1494
+ qBuf.destroy();
1495
+ kBuf.destroy();
1496
+ vBuf.destroy();
1497
+ coldPackedK.destroy();
1498
+ coldPackedV.destroy();
1499
+ coldScalesK.destroy();
1500
+ coldScalesV.destroy();
1501
+ resultTensor.buffer.destroy();
1502
+
1503
+ return result;
1504
+ },
1505
+
1506
+
1507
+ async benchmarkAttentionDecodeVariant(dev, options = {}) {
1508
+ if (!runAttention) {
1509
+ throw new Error('runAttention kernel not available');
1510
+ }
1511
+
1512
+ const {
1513
+ kernel = 'attention_decode_chunked_f16.wgsl',
1514
+ kvLens = [128, 256, 512, 1024, 1536, 2048],
1515
+ headDim = 256,
1516
+ numHeads = 8,
1517
+ numKVHeads = 4,
1518
+ warmupRuns = 5,
1519
+ timedRuns = 20,
1520
+ } = options;
1521
+
1522
+ const { device } = await getGPU();
1523
+ const benchmark = new KernelBenchmark(device);
1524
+ const seqLen = 1;
1525
+
1526
+ const qData = fillDeterministic(new Float32Array(numHeads * headDim));
1527
+ const qBuf = makeBuffer(toF16Array(qData));
1528
+ const qTensor = createTensor(qBuf, 'f16', [seqLen, numHeads, headDim], 'bench_q');
1529
+
1530
+ setActiveKernelPath(buildAttentionKernelPath(`bench-${kernel}`, kernel), 'runtime');
1531
+
1532
+ const results = [];
1533
+ for (const kvLen of kvLens) {
1534
+ const kData = fillDeterministic(new Float32Array(kvLen * numKVHeads * headDim));
1535
+ const vData = fillDeterministic(new Float32Array(kvLen * numKVHeads * headDim), 0.02);
1536
+ const kBuf = makeBuffer(toF16Array(kData));
1537
+ const vBuf = makeBuffer(toF16Array(vData));
1538
+
1539
+ const kTensor = createTensor(kBuf, 'f16', [kvLen, numKVHeads, headDim], 'bench_k');
1540
+ const vTensor = createTensor(vBuf, 'f16', [kvLen, numKVHeads, headDim], 'bench_v');
1541
+
1542
+ const outputSize = seqLen * numHeads * headDim * 2;
1543
+ const outputBuffer = device.createBuffer({
1544
+ label: 'bench_attention_output',
1545
+ size: outputSize,
1546
+ usage: GPUBufferUsage.STORAGE,
1547
+ });
1548
+
1549
+ const stats = await benchmark.runBenchmark(
1550
+ async () => {
1551
+ await runAttention(qTensor, kTensor, vTensor, null, numHeads, headDim, {
1552
+ seqLen,
1553
+ kvLen,
1554
+ numKVHeads,
1555
+ scale: 1 / Math.sqrt(headDim),
1556
+ causal: true,
1557
+ outputBuffer,
1558
+ });
1559
+ },
1560
+ {
1561
+ warmupRuns,
1562
+ timedRuns,
1563
+ label: `kv${kvLen}`,
1564
+ }
1565
+ );
1566
+
1567
+ const metrics = computeMetrics(stats, {
1568
+ operation: 'attention',
1569
+ seqLen,
1570
+ kvLen,
1571
+ numHeads,
1572
+ headDim,
1573
+ elementSize: 2,
1574
+ });
1575
+
1576
+ results.push({
1577
+ kvLen,
1578
+ stats: metrics,
1579
+ });
1580
+
1581
+ kBuf.destroy();
1582
+ vBuf.destroy();
1583
+ outputBuffer.destroy();
1584
+ }
1585
+
1586
+ qBuf.destroy();
1587
+ setActiveKernelPath(null, 'none');
1588
+
1589
+ return {
1590
+ kernel,
1591
+ headDim,
1592
+ numHeads,
1593
+ numKVHeads,
1594
+ warmupRuns,
1595
+ timedRuns,
1596
+ results,
1597
+ };
1598
+ },
1599
+
1600
+
1601
+ async runMoEGather(dev, tokens, expertIndices, numTokens, hiddenSize, numExperts, topK) {
1602
+ if (!runMoEGather) {
1603
+ // Fallback to reference if kernel not available
1604
+ const result = references.moeGatherRef(tokens, expertIndices, numTokens, hiddenSize, numExperts, topK);
1605
+ return {
1606
+ gatheredTokens: result.gatheredTokens,
1607
+ tokenCounts: result.tokenCounts,
1608
+ };
1609
+ }
1610
+
1611
+ // Create GPU buffers
1612
+ const tokensBuf = makeBuffer(tokens);
1613
+ const indicesBuf = makeBuffer(expertIndices);
1614
+
1615
+ // Wrap tokensBuf in Tensor (MoE kernels now use Tensor abstraction)
1616
+ const tokensTensor = createTensor(tokensBuf, 'f32', [numTokens, hiddenSize], 'moe_input');
1617
+
1618
+ // Run MoE gather via kernel selector
1619
+ const result = await runMoEGather(tokensTensor, indicesBuf, numTokens, hiddenSize, numExperts, topK);
1620
+
1621
+ // Read back results (result.gathered is now a Tensor)
1622
+ const maxTokensPerExpert = result.maxTokensPerExpert;
1623
+ const gatheredTokens = new Float32Array(await readBufferData(result.gathered.buffer, numExperts * maxTokensPerExpert * hiddenSize * 4));
1624
+ const tokenCounts = new Uint32Array(await readBufferData(result.tokenCounts, numExperts * 4));
1625
+
1626
+ tokensBuf.destroy();
1627
+ indicesBuf.destroy();
1628
+ result.gathered.buffer.destroy();
1629
+ result.tokenCounts.destroy();
1630
+ result.tokenMap.destroy();
1631
+
1632
+ return {
1633
+ gatheredTokens,
1634
+ tokenCounts,
1635
+ };
1636
+ },
1637
+
1638
+
1639
+ async runArgmax(dev, logits) {
1640
+ const logitsBuf = makeBuffer(logits);
1641
+ const tokenId = await sampleKernel.runArgmax(logitsBuf, logits.length, {
1642
+ logitsDtype: 'f32',
1643
+ padTokenId: null,
1644
+ logitSoftcap: 0,
1645
+ outputIndex: 0,
1646
+ });
1647
+ logitsBuf.destroy();
1648
+ return tokenId;
1649
+ },
1650
+
1651
+
1652
+ async runSampleTopK(dev, logits, temperature, topK, randomValue) {
1653
+ const logitsBuf = makeBuffer(logits);
1654
+ const tokenId = await sampleKernel.runGPUSample(logitsBuf, logits.length, {
1655
+ temperature,
1656
+ topK,
1657
+ randomSeed: randomValue * 10000, // Convert to seed
1658
+ logitsDtype: 'f32',
1659
+ padTokenId: null,
1660
+ logitSoftcap: 0,
1661
+ outputIndex: 0,
1662
+ greedyThreshold: 0.01,
1663
+ });
1664
+ logitsBuf.destroy();
1665
+ return tokenId;
1666
+ },
1667
+
1668
+
1669
+ async runSwiGLU(dev, gate, up, gateBias, upBias) {
1670
+ // For testing, we pre-add bias to gate and up, then use SiLU with gating
1671
+ const size = gate.length;
1672
+
1673
+ // Add biases
1674
+ const gateWithBias = new Float32Array(size);
1675
+ const upWithBias = new Float32Array(size);
1676
+ for (let i = 0; i < size; i++) {
1677
+ gateWithBias[i] = gate[i] + gateBias[i];
1678
+ upWithBias[i] = up[i] + upBias[i];
1679
+ }
1680
+
1681
+ if (!runSiLU) {
1682
+ // Fallback to reference implementation
1683
+ const result = new Float32Array(size);
1684
+ for (let i = 0; i < size; i++) {
1685
+ const silu = gateWithBias[i] / (1 + Math.exp(-gateWithBias[i]));
1686
+ result[i] = silu * upWithBias[i];
1687
+ }
1688
+ return result;
1689
+ }
1690
+
1691
+ const gateBuf = makeBuffer(gateWithBias);
1692
+ const upBuf = makeBuffer(upWithBias);
1693
+
1694
+ const gateTensor = createTensor(gateBuf, 'f32', [size], 'swiglu_gate');
1695
+ const upTensor = createTensor(upBuf, 'f32', [size], 'swiglu_up');
1696
+
1697
+ // runSiLU with gate option: output = silu(gate) * up
1698
+ const resultTensor = await runSiLU(upTensor, {
1699
+ size,
1700
+ gate: gateTensor,
1701
+ swigluLimit: null,
1702
+ inputActivation: 'identity',
1703
+ });
1704
+
1705
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, size * 4));
1706
+
1707
+ gateBuf.destroy();
1708
+ upBuf.destroy();
1709
+ resultTensor.buffer.destroy();
1710
+
1711
+ return result;
1712
+ },
1713
+
1714
+
1715
+ async runScale(dev, input, scale) {
1716
+ if (!runScale) {
1717
+ // Fallback to reference implementation
1718
+ const result = new Float32Array(input.length);
1719
+ for (let i = 0; i < input.length; i++) {
1720
+ result[i] = input[i] * scale;
1721
+ }
1722
+ return result;
1723
+ }
1724
+
1725
+ const inputBuf = makeBuffer(input);
1726
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'scale_input');
1727
+
1728
+ const resultTensor = await runScale(inputTensor, scale, { count: input.length });
1729
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, input.length * 4));
1730
+
1731
+ inputBuf.destroy();
1732
+ resultTensor.buffer.destroy();
1733
+
1734
+ return result;
1735
+ },
1736
+
1737
+
1738
+ async runGeLU(dev, input) {
1739
+ if (!runGeLU) {
1740
+ return references.geluRef(input);
1741
+ }
1742
+
1743
+ const inputBuf = makeBuffer(input);
1744
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'gelu_input');
1745
+
1746
+ const resultTensor = await runGeLU(inputTensor, { size: input.length });
1747
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, input.length * 4));
1748
+
1749
+ inputBuf.destroy();
1750
+ resultTensor.buffer.destroy();
1751
+
1752
+ return result;
1753
+ },
1754
+
1755
+
1756
+ async runGeGLU(dev, gate, up) {
1757
+ if (!runGeLU) {
1758
+ return references.gegluRef(gate, up);
1759
+ }
1760
+
1761
+ const gateBuf = makeBuffer(gate);
1762
+ const upBuf = makeBuffer(up);
1763
+
1764
+ const gateTensor = createTensor(gateBuf, 'f32', [gate.length], 'geglu_gate');
1765
+ const upTensor = createTensor(upBuf, 'f32', [up.length], 'geglu_up');
1766
+
1767
+ const resultTensor = await runGeLU(upTensor, { size: up.length, gate: gateTensor });
1768
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, up.length * 4));
1769
+
1770
+ gateBuf.destroy();
1771
+ upBuf.destroy();
1772
+ resultTensor.buffer.destroy();
1773
+
1774
+ return result;
1775
+ },
1776
+
1777
+
1778
+ async runSplitQKV(dev, qkv, numTokens, qSize, kSize, vSize) {
1779
+ if (!runSplitQKV) {
1780
+ return references.splitQkvRef(qkv, numTokens, qSize, kSize, vSize);
1781
+ }
1782
+
1783
+ const qkvBuf = makeBuffer(qkv);
1784
+ const qkvTensor = createTensor(qkvBuf, 'f32', [numTokens, qSize + kSize + vSize], 'split_qkv_input');
1785
+
1786
+ const { Q: qTensor, K: kTensor, V: vTensor } = await runSplitQKV(qkvTensor, {
1787
+ numTokens,
1788
+ qSize,
1789
+ kSize,
1790
+ vSize,
1791
+ });
1792
+
1793
+ const Q = new Float32Array(await readBufferData(qTensor.buffer, numTokens * qSize * 4));
1794
+ const K = new Float32Array(await readBufferData(kTensor.buffer, numTokens * kSize * 4));
1795
+ const V = new Float32Array(await readBufferData(vTensor.buffer, numTokens * vSize * 4));
1796
+
1797
+ qkvBuf.destroy();
1798
+ qTensor.buffer.destroy();
1799
+ kTensor.buffer.destroy();
1800
+ vTensor.buffer.destroy();
1801
+
1802
+ return { Q, K, V };
1803
+ },
1804
+
1805
+
1806
+ async runF16ToF32(dev, input) {
1807
+ if (!castF16ToF32) {
1808
+ const out = new Float32Array(input.length);
1809
+ for (let i = 0; i < input.length; i++) {
1810
+ out[i] = f16ToF32(input[i]);
1811
+ }
1812
+ return out;
1813
+ }
1814
+
1815
+ const inputBuf = makeBuffer(input);
1816
+ const inputTensor = createTensor(inputBuf, 'f16', [input.length], 'f16_to_f32_input');
1817
+ const outTensor = await castF16ToF32(inputTensor);
1818
+ const out = new Float32Array(await readBufferData(outTensor.buffer, input.length * 4));
1819
+
1820
+ inputBuf.destroy();
1821
+ outTensor.buffer.destroy();
1822
+
1823
+ return out;
1824
+ },
1825
+
1826
+
1827
+ async runSoftmaxBackward(dev, softmax, gradOutput, rows, cols) {
1828
+ if (!runSoftmaxBackward) {
1829
+ const output = new Float32Array(rows * cols);
1830
+ for (let r = 0; r < rows; r++) {
1831
+ const base = r * cols;
1832
+ let sum = 0;
1833
+ for (let c = 0; c < cols; c++) {
1834
+ sum += softmax[base + c] * gradOutput[base + c];
1835
+ }
1836
+ for (let c = 0; c < cols; c++) {
1837
+ const idx = base + c;
1838
+ output[idx] = softmax[idx] * (gradOutput[idx] - sum);
1839
+ }
1840
+ }
1841
+ return output;
1842
+ }
1843
+
1844
+ const softmaxBuf = makeBuffer(softmax);
1845
+ const gradBuf = makeBuffer(gradOutput);
1846
+ const softmaxTensor = createTensor(softmaxBuf, 'f32', [rows, cols], 'softmax_backward_input');
1847
+ const gradTensor = createTensor(gradBuf, 'f32', [rows, cols], 'softmax_backward_grad');
1848
+ const resultTensor = await runSoftmaxBackward(softmaxTensor, gradTensor, { rows, cols });
1849
+ const result = await readTensorToFloat32(resultTensor, rows * cols);
1850
+
1851
+ softmaxBuf.destroy();
1852
+ gradBuf.destroy();
1853
+ resultTensor.buffer.destroy();
1854
+
1855
+ return result;
1856
+ },
1857
+
1858
+
1859
+ async runSiluBackward(dev, input, gradOutput) {
1860
+ if (!runSiluBackward) {
1861
+ const output = new Float32Array(input.length);
1862
+ for (let i = 0; i < input.length; i++) {
1863
+ const x = input[i];
1864
+ const sigmoid = 1 / (1 + Math.exp(-x));
1865
+ const deriv = sigmoid * (1 + x * (1 - sigmoid));
1866
+ output[i] = gradOutput[i] * deriv;
1867
+ }
1868
+ return output;
1869
+ }
1870
+
1871
+ const inputBuf = makeBuffer(input);
1872
+ const gradBuf = makeBuffer(gradOutput);
1873
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'silu_backward_input');
1874
+ const gradTensor = createTensor(gradBuf, 'f32', [input.length], 'silu_backward_grad');
1875
+ const resultTensor = await runSiluBackward(inputTensor, gradTensor, { count: input.length });
1876
+ const result = await readTensorToFloat32(resultTensor, input.length);
1877
+
1878
+ inputBuf.destroy();
1879
+ gradBuf.destroy();
1880
+ resultTensor.buffer.destroy();
1881
+
1882
+ return result;
1883
+ },
1884
+
1885
+
1886
+ async runGeluBackward(dev, input, gradOutput) {
1887
+ if (!runGeluBackward) {
1888
+ const output = new Float32Array(input.length);
1889
+ for (let i = 0; i < input.length; i++) {
1890
+ const x = input[i];
1891
+ const sqrt2pi = 0.7978845608;
1892
+ const c = 0.044715;
1893
+ const x3 = x * x * x;
1894
+ const inner = sqrt2pi * (x + c * x3);
1895
+ const innerClamped = Math.max(-15, Math.min(15, inner));
1896
+ const tanhInner = Math.tanh(innerClamped);
1897
+ const sech2 = 1 - tanhInner * tanhInner;
1898
+ const innerDeriv = sqrt2pi * (1 + 3 * c * x * x);
1899
+ const deriv = 0.5 * (1 + tanhInner) + 0.5 * x * sech2 * innerDeriv;
1900
+ output[i] = gradOutput[i] * deriv;
1901
+ }
1902
+ return output;
1903
+ }
1904
+
1905
+ const inputBuf = makeBuffer(input);
1906
+ const gradBuf = makeBuffer(gradOutput);
1907
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'gelu_backward_input');
1908
+ const gradTensor = createTensor(gradBuf, 'f32', [input.length], 'gelu_backward_grad');
1909
+ const resultTensor = await runGeluBackward(inputTensor, gradTensor, { count: input.length });
1910
+ const result = await readTensorToFloat32(resultTensor, input.length);
1911
+
1912
+ inputBuf.destroy();
1913
+ gradBuf.destroy();
1914
+ resultTensor.buffer.destroy();
1915
+
1916
+ return result;
1917
+ },
1918
+
1919
+
1920
+ async runScaleBackward(dev, input, gradOutput, scale) {
1921
+ if (!runScaleBackward) {
1922
+ const output = new Float32Array(input.length);
1923
+ for (let i = 0; i < input.length; i++) {
1924
+ output[i] = gradOutput[i] * scale;
1925
+ }
1926
+ return output;
1927
+ }
1928
+
1929
+ const gradBuf = makeBuffer(gradOutput);
1930
+ const gradTensor = createTensor(gradBuf, 'f32', [input.length], 'scale_backward_grad');
1931
+ const resultTensor = await runScaleBackward(gradTensor, gradTensor, { count: input.length, scale });
1932
+ const result = await readTensorToFloat32(resultTensor, input.length);
1933
+
1934
+ gradBuf.destroy();
1935
+ resultTensor.buffer.destroy();
1936
+
1937
+ return result;
1938
+ },
1939
+
1940
+
1941
+ async runRoPEBackward(dev, gradOutput, cos, sin, seqLen, numHeads, headDim, startPos = 0) {
1942
+ if (!runRoPEBackward) {
1943
+ const output = new Float32Array(gradOutput.length);
1944
+ const halfDim = headDim / 2;
1945
+ for (let pos = 0; pos < seqLen; pos++) {
1946
+ for (let h = 0; h < numHeads; h++) {
1947
+ const base = (pos * numHeads + h) * headDim;
1948
+ const freqBase = (startPos + pos) * halfDim;
1949
+ for (let i = 0; i < halfDim; i++) {
1950
+ const dy0 = gradOutput[base + i];
1951
+ const dy1 = gradOutput[base + i + halfDim];
1952
+ const c = cos[freqBase + i];
1953
+ const s = sin[freqBase + i];
1954
+ output[base + i] = dy0 * c + dy1 * s;
1955
+ output[base + i + halfDim] = -dy0 * s + dy1 * c;
1956
+ }
1957
+ }
1958
+ }
1959
+ return output;
1960
+ }
1961
+
1962
+ const gradBuf = makeBuffer(gradOutput);
1963
+ const cosBuf = makeBuffer(cos);
1964
+ const sinBuf = makeBuffer(sin);
1965
+ const gradTensor = createTensor(gradBuf, 'f32', [seqLen, numHeads, headDim], 'rope_backward_grad');
1966
+ const cosTensor = createTensor(cosBuf, 'f32', [cos.length], 'rope_cos');
1967
+ const sinTensor = createTensor(sinBuf, 'f32', [sin.length], 'rope_sin');
1968
+ const resultTensor = await runRoPEBackward(gradTensor, cosTensor, sinTensor, {
1969
+ seqLen,
1970
+ numHeads,
1971
+ headDim,
1972
+ startPos,
1973
+ });
1974
+ const result = await readTensorToFloat32(resultTensor, seqLen * numHeads * headDim);
1975
+
1976
+ gradBuf.destroy();
1977
+ cosBuf.destroy();
1978
+ sinBuf.destroy();
1979
+ resultTensor.buffer.destroy();
1980
+
1981
+ return result;
1982
+ },
1983
+
1984
+
1985
+ async runRmsNormBackward(dev, input, weight, gradOutput, numTokens, hiddenSize, eps = 1e-6) {
1986
+ if (!runRmsNormBackward) {
1987
+ const output = new Float32Array(numTokens * hiddenSize);
1988
+ for (let t = 0; t < numTokens; t++) {
1989
+ const base = t * hiddenSize;
1990
+ let sumSq = 0;
1991
+ let sumGX = 0;
1992
+ for (let i = 0; i < hiddenSize; i++) {
1993
+ const x = input[base + i];
1994
+ const g = gradOutput[base + i] * weight[i];
1995
+ sumSq += x * x;
1996
+ sumGX += g * x;
1997
+ }
1998
+ const invRms = 1 / Math.sqrt(sumSq / hiddenSize + eps);
1999
+ const invRms3 = invRms * invRms * invRms;
2000
+ const coeff = (sumGX / hiddenSize) * invRms3;
2001
+ for (let i = 0; i < hiddenSize; i++) {
2002
+ const x = input[base + i];
2003
+ const g = gradOutput[base + i] * weight[i];
2004
+ output[base + i] = g * invRms - x * coeff;
2005
+ }
2006
+ }
2007
+ return output;
2008
+ }
2009
+
2010
+ const inputBuf = makeBuffer(input);
2011
+ const weightBuf = makeBuffer(weight);
2012
+ const gradBuf = makeBuffer(gradOutput);
2013
+ const inputTensor = createTensor(inputBuf, 'f32', [numTokens, hiddenSize], 'rmsnorm_backward_input');
2014
+ const weightTensor = createTensor(weightBuf, 'f32', [hiddenSize], 'rmsnorm_backward_weight');
2015
+ const gradTensor = createTensor(gradBuf, 'f32', [numTokens, hiddenSize], 'rmsnorm_backward_grad');
2016
+ const resultTensor = await runRmsNormBackward(inputTensor, weightTensor, gradTensor, { numTokens, hiddenSize, eps });
2017
+ const result = await readTensorToFloat32(resultTensor, numTokens * hiddenSize);
2018
+
2019
+ inputBuf.destroy();
2020
+ weightBuf.destroy();
2021
+ gradBuf.destroy();
2022
+ resultTensor.buffer.destroy();
2023
+
2024
+ return result;
2025
+ },
2026
+
2027
+
2028
+ async runEmbedBackward(dev, input, gradOutput, vocabSize = null) {
2029
+ if (!runEmbedBackward) {
2030
+ const fallbackVocabSize = Number.isFinite(vocabSize) && vocabSize > 0
2031
+ ? Math.max(1, Math.floor(vocabSize))
2032
+ : Math.max(1, Math.max(...input, 0) + 1);
2033
+ const numTokens = input.length;
2034
+ const hiddenSize = gradOutput.length / numTokens;
2035
+ const output = new Float32Array(fallbackVocabSize * hiddenSize);
2036
+ for (let i = 0; i < numTokens; i += 1) {
2037
+ const token = input[i];
2038
+ const outputBase = token * hiddenSize;
2039
+ const gradBase = i * hiddenSize;
2040
+ for (let d = 0; d < hiddenSize; d += 1) {
2041
+ output[outputBase + d] += gradOutput[gradBase + d];
2042
+ }
2043
+ }
2044
+ return output;
2045
+ }
2046
+
2047
+ const inputBuf = makeBuffer(input);
2048
+ const gradBuf = makeBuffer(gradOutput);
2049
+ const numTokens = input.length;
2050
+ const inputTensor = createTensor(inputBuf, 'u32', [input.length], 'embed_backward_input');
2051
+ const gradTensor = createTensor(gradBuf, 'f32', [gradOutput.length], 'embed_backward_grad');
2052
+ const maxInput = input.length > 0 ? Math.max(...input, 0) : 0;
2053
+ const hiddenSize = Math.max(1, gradOutput.length / numTokens);
2054
+ const resolvedVocabSize = Math.max(1, Math.floor(vocabSize ?? Math.ceil(maxInput + 1)));
2055
+ const resultTensor = await runEmbedBackward(inputTensor, gradTensor, {
2056
+ numTokens,
2057
+ hiddenSize,
2058
+ vocabSize: resolvedVocabSize,
2059
+ transpose: false,
2060
+ indexOffset: 0,
2061
+ });
2062
+ const result = await readTensorToFloat32(resultTensor, resolvedVocabSize * hiddenSize);
2063
+
2064
+ inputBuf.destroy();
2065
+ gradBuf.destroy();
2066
+ resultTensor.buffer.destroy();
2067
+
2068
+ return result;
2069
+ },
2070
+
2071
+
2072
+ async runAttentionBackward(dev, q, k, v, softmax, gradOutput, seqLen, numHeads, headDim, scale = 1.0) {
2073
+ if (!runAttentionBackward) {
2074
+ return null;
2075
+ }
2076
+
2077
+ const elementsPerHead = seqLen * headDim;
2078
+ const totalElements = seqLen * numHeads * headDim;
2079
+
2080
+ const packByHead = (source) => {
2081
+ const packed = new Float32Array(totalElements);
2082
+ for (let h = 0; h < numHeads; h++) {
2083
+ const headBase = h * elementsPerHead;
2084
+ for (let t = 0; t < seqLen; t++) {
2085
+ const sourceBase = (t * numHeads + h) * headDim;
2086
+ const packedBase = headBase + t * headDim;
2087
+ for (let d = 0; d < headDim; d++) {
2088
+ packed[packedBase + d] = source[sourceBase + d];
2089
+ }
2090
+ }
2091
+ }
2092
+ return packed;
2093
+ };
2094
+
2095
+ const unpackByHead = (source) => {
2096
+ const unpacked = new Float32Array(totalElements);
2097
+ for (let h = 0; h < numHeads; h++) {
2098
+ const headBase = h * elementsPerHead;
2099
+ for (let t = 0; t < seqLen; t++) {
2100
+ const targetBase = (t * numHeads + h) * headDim;
2101
+ const sourceBase = headBase + t * headDim;
2102
+ for (let d = 0; d < headDim; d++) {
2103
+ unpacked[targetBase + d] = source[sourceBase + d];
2104
+ }
2105
+ }
2106
+ }
2107
+ return unpacked;
2108
+ };
2109
+
2110
+ const qPacked = packByHead(q);
2111
+ const kPacked = packByHead(k);
2112
+ const vPacked = packByHead(v);
2113
+ const gPacked = packByHead(gradOutput);
2114
+
2115
+ const qBuf = makeBuffer(qPacked);
2116
+ const kBuf = makeBuffer(kPacked);
2117
+ const vBuf = makeBuffer(vPacked);
2118
+ const sBuf = makeBuffer(softmax);
2119
+ const gBuf = makeBuffer(gPacked);
2120
+ const qTensor = createTensor(qBuf, 'f32', [numHeads, seqLen, headDim], 'attn_bw_q');
2121
+ const kTensor = createTensor(kBuf, 'f32', [numHeads, seqLen, headDim], 'attn_bw_k');
2122
+ const vTensor = createTensor(vBuf, 'f32', [numHeads, seqLen, headDim], 'attn_bw_v');
2123
+ const sTensor = createTensor(sBuf, 'f32', [numHeads, seqLen, seqLen], 'attn_bw_softmax');
2124
+ const gTensor = createTensor(gBuf, 'f32', [numHeads, seqLen, headDim], 'attn_bw_grad');
2125
+
2126
+ const result = await runAttentionBackward(qTensor, kTensor, vTensor, sTensor, gTensor, {
2127
+ seqLen,
2128
+ numHeads,
2129
+ headDim,
2130
+ scale,
2131
+ causal: false,
2132
+ });
2133
+
2134
+ const gradQPacked = await readTensorToFloat32(result.gradQ, seqLen * numHeads * headDim);
2135
+ const gradKPacked = await readTensorToFloat32(result.gradK, seqLen * numHeads * headDim);
2136
+ const gradVPacked = await readTensorToFloat32(result.gradV, seqLen * numHeads * headDim);
2137
+ const gradQ = unpackByHead(gradQPacked);
2138
+ const gradK = unpackByHead(gradKPacked);
2139
+ const gradV = unpackByHead(gradVPacked);
2140
+
2141
+ qBuf.destroy();
2142
+ kBuf.destroy();
2143
+ vBuf.destroy();
2144
+ sBuf.destroy();
2145
+ gBuf.destroy();
2146
+ result.gradQ.buffer.destroy();
2147
+ result.gradK.buffer.destroy();
2148
+ result.gradV.buffer.destroy();
2149
+
2150
+ return { gradQ, gradK, gradV };
2151
+ },
2152
+
2153
+
2154
+ async runAdam(dev, params, grads, moment1, moment2, options) {
2155
+ if (!runAdam) {
2156
+ const outParams = new Float32Array(params);
2157
+ const outM1 = new Float32Array(moment1);
2158
+ const outM2 = new Float32Array(moment2);
2159
+ const { count, step, lr, beta1, beta2, eps } = options;
2160
+ for (let i = 0; i < count; i++) {
2161
+ const g = grads[i];
2162
+ outM1[i] = beta1 * outM1[i] + (1 - beta1) * g;
2163
+ outM2[i] = beta2 * outM2[i] + (1 - beta2) * g * g;
2164
+ const mHat = outM1[i] / (1 - Math.pow(beta1, step));
2165
+ const vHat = outM2[i] / (1 - Math.pow(beta2, step));
2166
+ outParams[i] = outParams[i] - lr * mHat / (Math.sqrt(vHat) + eps);
2167
+ }
2168
+ return { params: outParams, moment1: outM1, moment2: outM2 };
2169
+ }
2170
+
2171
+ const paramBuf = makeBuffer(params);
2172
+ const gradBuf = makeBuffer(grads);
2173
+ const m1Buf = makeBuffer(moment1);
2174
+ const m2Buf = makeBuffer(moment2);
2175
+ const paramTensor = createTensor(paramBuf, 'f32', [params.length], 'adam_params');
2176
+ const gradTensor = createTensor(gradBuf, 'f32', [grads.length], 'adam_grads');
2177
+ const m1Tensor = createTensor(m1Buf, 'f32', [moment1.length], 'adam_m1');
2178
+ const m2Tensor = createTensor(m2Buf, 'f32', [moment2.length], 'adam_m2');
2179
+
2180
+ const resultTensor = await runAdam(paramTensor, gradTensor, m1Tensor, m2Tensor, options);
2181
+ const outParams = await readTensorToFloat32(resultTensor, params.length);
2182
+ const outM1 = new Float32Array(await readBufferData(m1Buf, moment1.length * 4));
2183
+ const outM2 = new Float32Array(await readBufferData(m2Buf, moment2.length * 4));
2184
+
2185
+ paramBuf.destroy();
2186
+ gradBuf.destroy();
2187
+ m1Buf.destroy();
2188
+ m2Buf.destroy();
2189
+ resultTensor.buffer.destroy();
2190
+
2191
+ return { params: outParams, moment1: outM1, moment2: outM2 };
2192
+ },
2193
+
2194
+
2195
+ async runBF16ToF32(dev, input) {
2196
+ if (!runBF16ToF32) {
2197
+ const out = new Float32Array(input.length);
2198
+ for (let i = 0; i < input.length; i++) {
2199
+ const view = new DataView(new ArrayBuffer(4));
2200
+ view.setUint32(0, input[i] << 16, true);
2201
+ out[i] = view.getFloat32(0, true);
2202
+ }
2203
+ return out;
2204
+ }
2205
+
2206
+ const inputBuf = makeBuffer(input, GPUBufferUsage.STORAGE);
2207
+ const outTensor = await runBF16ToF32(inputBuf, [input.length], 'bf16_to_f32_test');
2208
+ const out = new Float32Array(await readBufferData(outTensor.buffer, input.length * 4));
2209
+
2210
+ inputBuf.destroy();
2211
+ outTensor.buffer.destroy();
2212
+
2213
+ return out;
2214
+ },
2215
+
2216
+
2217
+ async runF32ToF16(dev, input) {
2218
+ if (!castF32ToF16) {
2219
+ const out = new Uint16Array(input.length);
2220
+ for (let i = 0; i < input.length; i++) {
2221
+ const view = new DataView(new ArrayBuffer(4));
2222
+ view.setFloat32(0, input[i], true);
2223
+ const bits = view.getUint32(0, true);
2224
+ const sign = (bits >> 31) & 0x1;
2225
+ const exp = (bits >> 23) & 0xff;
2226
+ const mant = bits & 0x7fffff;
2227
+
2228
+ let hExp = 0;
2229
+ let hMant = 0;
2230
+ if (exp === 0xff) {
2231
+ hExp = 0x1f;
2232
+ hMant = mant ? 0x200 : 0;
2233
+ } else if (exp !== 0) {
2234
+ const newExp = exp - 127 + 15;
2235
+ if (newExp >= 0x1f) {
2236
+ hExp = 0x1f;
2237
+ } else if (newExp > 0) {
2238
+ hExp = newExp;
2239
+ hMant = mant >> 13;
2240
+ }
2241
+ }
2242
+ out[i] = (sign << 15) | (hExp << 10) | hMant;
2243
+ }
2244
+ return out;
2245
+ }
2246
+
2247
+ const inputBuf = makeBuffer(input);
2248
+ const inputTensor = createTensor(inputBuf, 'f32', [input.length], 'f32_to_f16_input');
2249
+ const outTensor = await castF32ToF16(inputTensor);
2250
+ const out = new Uint16Array(await readBufferData(outTensor.buffer, input.length * 2));
2251
+
2252
+ inputBuf.destroy();
2253
+ outTensor.buffer.destroy();
2254
+
2255
+ return out;
2256
+ },
2257
+
2258
+
2259
+ async runBF16ToF16(dev, input) {
2260
+ if (!runBF16ToF16) {
2261
+ const out = new Uint16Array(input.length);
2262
+ for (let i = 0; i < input.length; i++) {
2263
+ const view = new DataView(new ArrayBuffer(4));
2264
+ view.setUint32(0, input[i] << 16, true);
2265
+ const bits = view.getUint32(0, true);
2266
+ const sign = (bits >> 31) & 0x1;
2267
+ const exp = (bits >> 23) & 0xff;
2268
+ const mant = bits & 0x7fffff;
2269
+
2270
+ let hExp = 0;
2271
+ let hMant = 0;
2272
+ if (exp === 0xff) {
2273
+ hExp = 0x1f;
2274
+ hMant = mant ? 0x200 : 0;
2275
+ } else if (exp !== 0) {
2276
+ const newExp = exp - 127 + 15;
2277
+ if (newExp >= 0x1f) {
2278
+ hExp = 0x1f;
2279
+ } else if (newExp > 0) {
2280
+ hExp = newExp;
2281
+ hMant = mant >> 13;
2282
+ }
2283
+ }
2284
+ out[i] = (sign << 15) | (hExp << 10) | hMant;
2285
+ }
2286
+ return out;
2287
+ }
2288
+
2289
+ const inputBuf = makeBuffer(input, GPUBufferUsage.STORAGE);
2290
+ const outTensor = await runBF16ToF16(inputBuf, [input.length], 'bf16_to_f16_test');
2291
+ const out = new Uint16Array(await readBufferData(outTensor.buffer, input.length * 2));
2292
+
2293
+ inputBuf.destroy();
2294
+ outTensor.buffer.destroy();
2295
+
2296
+ return out;
2297
+ },
2298
+
2299
+
2300
+ async runDequantQ6K(dev, quantized, numBlocks) {
2301
+ if (!dequantizeQ6K) {
2302
+ throw new Error('dequantizeQ6K kernel not available');
2303
+ }
2304
+
2305
+ const blockSize = 256; // Q6_K: 256 elements per block
2306
+ const quantizedBytes = quantized instanceof Uint8Array
2307
+ ? quantized
2308
+ : new Uint8Array(quantized.buffer ?? quantized);
2309
+ const alignedBytes = quantizedBytes.byteLength % 4 === 0
2310
+ ? quantizedBytes
2311
+ : (() => {
2312
+ const padded = new Uint8Array(Math.ceil(quantizedBytes.byteLength / 4) * 4);
2313
+ padded.set(quantizedBytes);
2314
+ return padded;
2315
+ })();
2316
+ const qBuf = makeBuffer(alignedBytes, GPUBufferUsage.STORAGE);
2317
+ const outTensor = await dequantizeQ6K(qBuf, numBlocks, { outputOffset: 0, outputDtype: 'f16' });
2318
+
2319
+ const outBuf = outTensor.buffer;
2320
+ // Q6K outputs f16 - read raw bytes and convert
2321
+ const rawData = await readBufferData(outBuf, numBlocks * blockSize * 2); // f16 = 2 bytes
2322
+ const u16View = new Uint16Array(rawData);
2323
+ const out = new Float32Array(u16View.length);
2324
+
2325
+ // Convert f16 to f32
2326
+ for (let i = 0; i < u16View.length; i++) {
2327
+ out[i] = f16ToF32(u16View[i]);
2328
+ }
2329
+
2330
+ qBuf.destroy();
2331
+ outBuf.destroy();
2332
+
2333
+ return out;
2334
+ },
2335
+
2336
+
2337
+ async runMatmulF16W(dev, A, B_f16, M, N, K) {
2338
+ if (!runMatmul) {
2339
+ throw new Error('runMatmul kernel not available');
2340
+ }
2341
+
2342
+ // Create A buffer (F32 activations)
2343
+ const bufA = makeBuffer(A);
2344
+ const tensorA = createTensor(bufA, 'f32', [M, K], 'matmul_f16w_a');
2345
+
2346
+ // Create B buffer (F16 weights) - pass raw Uint16Array
2347
+ const bufB = makeBuffer(B_f16, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC);
2348
+
2349
+ // Run matmul with bDtype='f16' and preferF16=true to trigger f16w_f32a kernel
2350
+ // transposeB=true because B is [N, K] format (weight matrix layout)
2351
+ const resultTensor = await runMatmul(tensorA, bufB, M, N, K, {
2352
+ bDtype: 'f16',
2353
+ preferF16: true,
2354
+ transposeB: true,
2355
+ outputDtype: 'f32',
2356
+ });
2357
+
2358
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
2359
+
2360
+ bufA.destroy();
2361
+ bufB.destroy();
2362
+ resultTensor.buffer.destroy();
2363
+
2364
+ return result;
2365
+ },
2366
+
2367
+
2368
+ async runDequantAndMatmulF16W(dev, A, B_q4k, M, N, K, numBlocks) {
2369
+ if (!runMatmul || !dequantize) {
2370
+ throw new Error('runMatmul or dequantize kernel not available');
2371
+ }
2372
+
2373
+ // Dequant Q4K -> F16 on GPU
2374
+ const qBuf = makeBuffer(B_q4k, GPUBufferUsage.STORAGE);
2375
+ const dequantTensor = await dequantize(qBuf, numBlocks, { outputDtype: 'f16', useVec4: true });
2376
+
2377
+ // Create A buffer (F32 activations)
2378
+ const bufA = makeBuffer(A);
2379
+ const tensorA = createTensor(bufA, 'f32', [M, K], 'dequant_matmul_a');
2380
+
2381
+ // Run matmul with F16 weights (stays on GPU, no CPU round-trip)
2382
+ // transposeB=true because dequanted weights are [N, K] format
2383
+ const resultTensor = await runMatmul(tensorA, dequantTensor.buffer, M, N, K, {
2384
+ bDtype: 'f16',
2385
+ preferF16: true,
2386
+ transposeB: true,
2387
+ outputDtype: 'f32',
2388
+ });
2389
+
2390
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, M * N * 4));
2391
+
2392
+ qBuf.destroy();
2393
+ bufA.destroy();
2394
+ dequantTensor.buffer.destroy();
2395
+ resultTensor.buffer.destroy();
2396
+
2397
+ return result;
2398
+ },
2399
+
2400
+
2401
+ async runCheckStop(dev, sampledToken, eosTokenId, maxTokens, currentPos) {
2402
+ // Create buffer for sampled token
2403
+ const tokenBuffer = makeBuffer(new Uint32Array([sampledToken]));
2404
+
2405
+ const shouldStop = await checkStop({
2406
+ sampledTokenBuffer: tokenBuffer,
2407
+ eosTokenId,
2408
+ maxTokens,
2409
+ currentPos,
2410
+ });
2411
+
2412
+ tokenBuffer.destroy();
2413
+
2414
+ return shouldStop;
2415
+ },
2416
+
2417
+
2418
+ checkStopRef(sampledToken, eosTokenId, maxTokens, currentPos) {
2419
+ const isEOS = sampledToken === eosTokenId;
2420
+ const reachedMax = currentPos >= maxTokens;
2421
+ return isEOS || reachedMax;
2422
+ },
2423
+
2424
+
2425
+ async runFusedMatmulResidual(dev, input, weight, residual, N, K, alpha = 1.0) {
2426
+ const inputBuf = makeBuffer(input);
2427
+ const weightF16 = toF16Array(weight);
2428
+ const weightBuf = makeBuffer(weightF16);
2429
+ const residualBuf = makeBuffer(residual);
2430
+
2431
+ const inputTensor = createTensor(inputBuf, 'f32', [1, K], 'fused_matmul_res_input');
2432
+ const residualTensor = createTensor(residualBuf, 'f32', [1, N], 'fused_matmul_res_residual');
2433
+
2434
+ const resultTensor = await runMatmulResidualFused(inputTensor, weightBuf, residualTensor, {
2435
+ N,
2436
+ K,
2437
+ alpha,
2438
+ });
2439
+
2440
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, N * 4));
2441
+
2442
+ inputBuf.destroy();
2443
+ weightBuf.destroy();
2444
+ residualBuf.destroy();
2445
+ resultTensor.buffer.destroy();
2446
+
2447
+ return result;
2448
+ },
2449
+
2450
+
2451
+ fusedMatmulResidualRef(input, weight, residual, N, K, alpha = 1.0) {
2452
+ // input: [1, K], weight: [N, K] (row-major, transposeB=true), residual: [1, N]
2453
+ const weightF16 = toF16RoundedFloat32(weight);
2454
+ const output = new Float32Array(N);
2455
+ for (let n = 0; n < N; n++) {
2456
+ let sum = 0;
2457
+ for (let k = 0; k < K; k++) {
2458
+ sum += input[k] * weightF16[n * K + k];
2459
+ }
2460
+ output[n] = sum * alpha + residual[n];
2461
+ }
2462
+ return output;
2463
+ },
2464
+
2465
+
2466
+ async runFusedMatmulRMSNorm(dev, input, weight, normWeight, N, K, eps = 1e-5, residual = null) {
2467
+ const inputBuf = makeBuffer(input);
2468
+ const weightBuf = makeBuffer(weight);
2469
+ const normWeightBuf = makeBuffer(normWeight);
2470
+ const residualBuf = residual ? makeBuffer(residual) : null;
2471
+
2472
+ const inputTensor = createTensor(inputBuf, 'f32', [1, K], 'fused_matmul_rmsnorm_input');
2473
+
2474
+ const resultTensor = await runMatmulRMSNormFused(inputTensor, weightBuf, normWeightBuf, {
2475
+ N,
2476
+ K,
2477
+ eps,
2478
+ residual: residualBuf,
2479
+ transposeB: true,
2480
+ });
2481
+
2482
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, N * 4));
2483
+
2484
+ inputBuf.destroy();
2485
+ weightBuf.destroy();
2486
+ normWeightBuf.destroy();
2487
+ if (residualBuf) residualBuf.destroy();
2488
+ resultTensor.buffer.destroy();
2489
+
2490
+ return result;
2491
+ },
2492
+
2493
+
2494
+ fusedMatmulRMSNormRef(input, weight, normWeight, N, K, eps = 1e-5, residual = null) {
2495
+ // Step 1: matmul input[1,K] @ weight[N,K]^T -> intermediate[1,N]
2496
+ const intermediate = new Float32Array(N);
2497
+ for (let n = 0; n < N; n++) {
2498
+ let sum = 0;
2499
+ for (let k = 0; k < K; k++) {
2500
+ sum += input[k] * weight[n * K + k];
2501
+ }
2502
+ intermediate[n] = sum;
2503
+ }
2504
+
2505
+ // Step 2: RMSNorm
2506
+ let sumSq = 0;
2507
+ for (let i = 0; i < N; i++) {
2508
+ sumSq += intermediate[i] * intermediate[i];
2509
+ }
2510
+ const rms = Math.sqrt(sumSq / N + eps);
2511
+
2512
+ const output = new Float32Array(N);
2513
+ for (let i = 0; i < N; i++) {
2514
+ const normalized = (intermediate[i] / rms) * normWeight[i];
2515
+ output[i] = residual ? normalized + residual[i] : normalized;
2516
+ }
2517
+
2518
+ return output;
2519
+ },
2520
+
2521
+
2522
+ async runFusedFFN(dev, input, W_gate, W_up, hiddenSize, intermediateSize, activation = 'silu') {
2523
+ const inputBuf = makeBuffer(input);
2524
+ const gateBuf = makeBuffer(W_gate);
2525
+ const upBuf = makeBuffer(W_up);
2526
+ const gateWeight = createWeightBuffer(gateBuf, 'f32', 'row', [intermediateSize, hiddenSize], 'fused_ffn_gate');
2527
+ const upWeight = createWeightBuffer(upBuf, 'f32', 'row', [intermediateSize, hiddenSize], 'fused_ffn_up');
2528
+
2529
+ const inputTensor = createTensor(inputBuf, 'f32', [1, hiddenSize], 'fused_ffn_input');
2530
+
2531
+ const resultTensor = await runFusedFFN(inputTensor, gateWeight, upWeight, hiddenSize, intermediateSize, {
2532
+ batchSize: 1,
2533
+ activation,
2534
+ alpha: 1.0,
2535
+ swigluLimit: null,
2536
+ });
2537
+
2538
+ const result = new Float32Array(await readBufferData(resultTensor.buffer, intermediateSize * 4));
2539
+
2540
+ inputBuf.destroy();
2541
+ gateBuf.destroy();
2542
+ upBuf.destroy();
2543
+ resultTensor.buffer.destroy();
2544
+
2545
+ return result;
2546
+ },
2547
+
2548
+
2549
+ fusedFFNRef(input, W_gate, W_up, hiddenSize, intermediateSize, activation = 'silu') {
2550
+ // Step 1: gate = input @ W_gate^T (input[1,K] @ W_gate[N,K]^T -> gate[1,N])
2551
+ const gate = new Float32Array(intermediateSize);
2552
+ for (let n = 0; n < intermediateSize; n++) {
2553
+ let sum = 0;
2554
+ for (let k = 0; k < hiddenSize; k++) {
2555
+ sum += input[k] * W_gate[n * hiddenSize + k];
2556
+ }
2557
+ gate[n] = sum;
2558
+ }
2559
+
2560
+ // Step 2: up = input @ W_up^T
2561
+ const up = new Float32Array(intermediateSize);
2562
+ for (let n = 0; n < intermediateSize; n++) {
2563
+ let sum = 0;
2564
+ for (let k = 0; k < hiddenSize; k++) {
2565
+ sum += input[k] * W_up[n * hiddenSize + k];
2566
+ }
2567
+ up[n] = sum;
2568
+ }
2569
+
2570
+ // Step 3: Apply activation to gate and multiply by up
2571
+ const output = new Float32Array(intermediateSize);
2572
+ for (let i = 0; i < intermediateSize; i++) {
2573
+ let activated;
2574
+ if (activation === 'silu') {
2575
+ // SiLU(x) = x * sigmoid(x) = x / (1 + exp(-x))
2576
+ activated = gate[i] / (1 + Math.exp(-gate[i]));
2577
+ } else {
2578
+ // GELU(x) = x * 0.5 * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))
2579
+ const x = gate[i];
2580
+ const sqrt2pi = Math.sqrt(2 / Math.PI);
2581
+ const cdf = 0.5 * (1 + Math.tanh(sqrt2pi * (x + 0.044715 * x * x * x)));
2582
+ activated = x * cdf;
2583
+ }
2584
+ output[i] = activated * up[i];
2585
+ }
2586
+
2587
+ return output;
2588
+ },
2589
+ };
2590
+
2591
+ // Expose to window for browser automation
2592
+ window.testHarness = testHarness;
2593
+ window.gpuReady = false;
2594
+ window.gpuError = undefined;
2595
+
2596
+ // Auto-initialize on load
2597
+ window.addEventListener('DOMContentLoaded', async () => {
2598
+ try {
2599
+ await initGPU();
2600
+ console.log('WebGPU initialized successfully');
2601
+ window.gpuReady = true;
2602
+
2603
+ // Display status
2604
+ const status = document.getElementById('status');
2605
+ if (status) {
2606
+ const caps = getKernelCapabilities();
2607
+ status.innerHTML = `
2608
+ <strong>WebGPU Ready</strong><br>
2609
+ Adapter: ${caps?.adapterInfo || 'Unknown'}<br>
2610
+ F16 Support: ${caps?.hasF16 ? 'Yes' : 'No'}<br>
2611
+ Subgroups: ${caps?.hasSubgroups ? 'Yes' : 'No'}
2612
+ `;
2613
+ status.style.color = 'green';
2614
+ }
2615
+ } catch (e) {
2616
+ console.error('Failed to initialize WebGPU:', e);
2617
+ window.gpuReady = false;
2618
+ window.gpuError = e.message;
2619
+
2620
+ const status = document.getElementById('status');
2621
+ if (status) {
2622
+ status.innerHTML = `<strong>WebGPU Error:</strong> ${e.message}`;
2623
+ status.style.color = 'red';
2624
+ }
2625
+ }
2626
+ });
2627
+
2628
+ // Export for module usage
2629
+ export { testHarness, initGPU, getGPU };