doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,445 +0,0 @@
1
- // Attention Decode Kernel - Highly Optimized (Tier 2 P0)
2
- //
3
- // Optimized for seqLen=1 (decode) with 3.6-7.2x speedup over baseline.
4
- //
5
- // Key optimizations:
6
- // 1. Vectorized loads (vec4 for Q, K, V) - 4x memory bandwidth efficiency
7
- // 2. Subgroup operations for reductions - eliminates barriers
8
- // 3. Online softmax (FlashAttention-style) - fuses softmax with attention
9
- // 4. Register-based accumulation - minimizes shared memory traffic
10
- // 5. Warp-level primitives - leverages GPU parallelism
11
- //
12
- // Architecture:
13
- // - One workgroup per head
14
- // - 256 threads per workgroup
15
- // - Processes KV cache in chunks of 256 positions
16
- // - Uses online softmax to avoid storing full score vector
17
-
18
- enable subgroups;
19
-
20
- struct Uniforms {
21
- seq_len: u32, // Always 1 for decode
22
- kv_len: u32, // Current KV cache length
23
- num_heads: u32, // Number of query heads
24
- num_kv_heads: u32, // Number of KV heads (GQA support)
25
- head_dim: u32, // Head dimension (typically 64, 128, or 256)
26
- kv_len_source: u32, // 0 = use uniform kv_len, 1 = use buffer
27
- _pad1: u32,
28
- _pad2: u32,
29
- }
30
-
31
- @group(0) @binding(0) var<uniform> u: Uniforms;
32
- @group(0) @binding(1) var<storage, read> Q: array<f32>;
33
- @group(0) @binding(2) var<storage, read> K_cache: array<u32>;
34
- @group(0) @binding(3) var<storage, read> V_cache: array<u32>;
35
- @group(0) @binding(4) var<storage, read_write> output: array<f32>;
36
- @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
37
-
38
- // Workgroup size for decode: 256 threads
39
- override WORKGROUP_SIZE: u32 = 256u;
40
-
41
- // Chunk size for KV cache processing (matches workgroup size)
42
- override CHUNK_SIZE: u32 = 256u;
43
- const MAX_CHUNK_SIZE: u32 = 256u;
44
- const MAX_HEAD_DIM: u32 = 256u;
45
- const MAX_SUBGROUPS: u32 = 256u;
46
-
47
- // Shared memory for Q vector and partial results
48
- var<workgroup> shared_q: array<f32, MAX_HEAD_DIM>; // Q values for this head
49
- var<workgroup> shared_scores: array<f32, MAX_CHUNK_SIZE>;
50
-
51
- // For cross-subgroup reductions
52
- var<workgroup> sg_max: array<f32, MAX_SUBGROUPS>; // Subgroup maxes
53
- var<workgroup> sg_sum: array<f32, MAX_SUBGROUPS>; // Subgroup sums
54
- var<workgroup> global_max: f32;
55
- var<workgroup> global_sum: f32;
56
-
57
- fn get_kv_len() -> u32 {
58
- if (u.kv_len_source == 0u) {
59
- return u.kv_len;
60
- }
61
- return kv_len_buffer[0];
62
- }
63
-
64
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
65
- fn main(
66
- @builtin(local_invocation_id) local_id: vec3<u32>,
67
- @builtin(workgroup_id) workgroup_id: vec3<u32>,
68
- @builtin(subgroup_size) subgroup_size: u32,
69
- @builtin(subgroup_invocation_id) sg_tid: u32,
70
- ) {
71
- let head_idx = workgroup_id.x;
72
- let tid = local_id.x;
73
- let head_dim = u.head_dim;
74
- let kv_len = get_kv_len();
75
- if (CHUNK_SIZE > MAX_CHUNK_SIZE || head_dim > MAX_HEAD_DIM) {
76
- return;
77
- }
78
-
79
- // GQA: map query head to KV head
80
- let heads_per_kv = u.num_heads / u.num_kv_heads;
81
- let kv_head_idx = head_idx / heads_per_kv;
82
-
83
- let subgroup_id = tid / subgroup_size;
84
- let num_subgroups = (WORKGROUP_SIZE + subgroup_size - 1u) / subgroup_size;
85
-
86
- // Scale factor for attention
87
- let scale = 1.0 / sqrt(f32(head_dim));
88
-
89
- // Phase 1: Load Q vector into shared memory
90
- if (tid < head_dim) {
91
- let q_offset = head_idx * head_dim + tid;
92
- shared_q[tid] = Q[q_offset];
93
- }
94
- workgroupBarrier();
95
-
96
- // Initialize accumulators for online softmax
97
- var running_max: f32 = -1e38;
98
- var running_sum: f32 = 0.0;
99
-
100
- // Initialize output accumulator (one per thread, covering headDim)
101
- var out_accum: f32 = 0.0;
102
-
103
- // Phase 2: Process KV cache in chunks using online softmax
104
- // Each thread processes one K position per chunk iteration
105
- let num_chunks = (kv_len + CHUNK_SIZE - 1u) / CHUNK_SIZE;
106
-
107
- for (var chunk = 0u; chunk < num_chunks; chunk++) {
108
- let k_pos = chunk * CHUNK_SIZE + tid;
109
- let valid_k = k_pos < kv_len;
110
-
111
- // Compute dot product Q @ K^T for this position
112
- var score: f32 = 0.0;
113
- if (valid_k) {
114
- let k_base = k_pos * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
115
-
116
- // Vectorized dot product (4 elements at a time)
117
- for (var d = 0u; d < head_dim; d += 4u) {
118
- if (d + 3u < head_dim) {
119
- let q0 = shared_q[d];
120
- let q1 = shared_q[d + 1u];
121
- let q2 = shared_q[d + 2u];
122
- let q3 = shared_q[d + 3u];
123
-
124
- let k0 = bitcast<f32>(K_cache[k_base + d]);
125
- let k1 = bitcast<f32>(K_cache[k_base + d + 1u]);
126
- let k2 = bitcast<f32>(K_cache[k_base + d + 2u]);
127
- let k3 = bitcast<f32>(K_cache[k_base + d + 3u]);
128
-
129
- score += q0 * k0 + q1 * k1 + q2 * k2 + q3 * k3;
130
- } else {
131
- // Handle remainder
132
- for (var dd = d; dd < head_dim; dd++) {
133
- score += shared_q[dd] * bitcast<f32>(K_cache[k_base + dd]);
134
- }
135
- }
136
- }
137
- score *= scale;
138
- } else {
139
- score = -1e38; // Mask invalid positions
140
- }
141
-
142
- // Store score in shared memory
143
- shared_scores[tid] = score;
144
- workgroupBarrier();
145
-
146
- // Find max in this chunk using subgroup operations
147
- var chunk_max = subgroupMax(score);
148
- if (sg_tid == 0u && subgroup_id < num_subgroups) {
149
- sg_max[subgroup_id] = chunk_max;
150
- }
151
- workgroupBarrier();
152
-
153
- if (tid == 0u) {
154
- var m = -1e38;
155
- for (var s = 0u; s < num_subgroups; s++) {
156
- m = max(m, sg_max[s]);
157
- }
158
- global_max = m;
159
- }
160
- workgroupBarrier();
161
-
162
- let chunk_max_val = global_max;
163
-
164
- // Online softmax update
165
- // new_max = max(running_max, chunk_max)
166
- // rescale = exp(running_max - new_max)
167
- // running_sum = running_sum * rescale + sum(exp(scores - new_max))
168
- let new_max = max(running_max, chunk_max_val);
169
- let rescale = exp(running_max - new_max);
170
-
171
- // Compute exp(score - new_max) and sum
172
- var exp_score: f32 = 0.0;
173
- if (valid_k) {
174
- exp_score = exp(score - new_max);
175
- }
176
- shared_scores[tid] = exp_score;
177
-
178
- // Sum exp_scores using subgroup reduction
179
- var chunk_sum = subgroupAdd(exp_score);
180
- if (sg_tid == 0u && subgroup_id < num_subgroups) {
181
- sg_sum[subgroup_id] = chunk_sum;
182
- }
183
- workgroupBarrier();
184
-
185
- if (tid == 0u) {
186
- var s = 0.0;
187
- for (var i = 0u; i < num_subgroups; i++) {
188
- s += sg_sum[i];
189
- }
190
- global_sum = s;
191
- }
192
- workgroupBarrier();
193
-
194
- let chunk_sum_val = global_sum;
195
-
196
- // Update running sum with rescaling
197
- running_sum = running_sum * rescale + chunk_sum_val;
198
- running_max = new_max;
199
-
200
- // Accumulate weighted V values
201
- // Each thread handles one dimension of output
202
- if (tid < head_dim) {
203
- // Rescale previous accumulator
204
- out_accum *= rescale;
205
-
206
- // Add contribution from this chunk
207
- for (var k = 0u; k < min(CHUNK_SIZE, kv_len - chunk * CHUNK_SIZE); k++) {
208
- let k_pos_inner = chunk * CHUNK_SIZE + k;
209
- let v_offset = k_pos_inner * u.num_kv_heads * head_dim + kv_head_idx * head_dim + tid;
210
- let attn_weight = shared_scores[k];
211
- out_accum += attn_weight * bitcast<f32>(V_cache[v_offset]);
212
- }
213
- }
214
- workgroupBarrier();
215
- }
216
-
217
- // Phase 3: Finalize output (divide by sum)
218
- if (tid < head_dim) {
219
- let out_offset = head_idx * head_dim + tid;
220
- let inv_sum = select(0.0, 1.0 / running_sum, running_sum > 0.0);
221
- output[out_offset] = out_accum * inv_sum;
222
- }
223
- }
224
-
225
- // Alternative: Parallel-head variant for models with many heads
226
- // Processes multiple heads per workgroup for better occupancy
227
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
228
- fn main_multihead(
229
- @builtin(local_invocation_id) local_id: vec3<u32>,
230
- @builtin(workgroup_id) workgroup_id: vec3<u32>,
231
- @builtin(subgroup_size) subgroup_size: u32,
232
- @builtin(subgroup_invocation_id) sg_tid: u32,
233
- ) {
234
- // For models with many heads (32+), process 4 heads per workgroup
235
- let heads_per_wg = 4u;
236
- let threads_per_head = WORKGROUP_SIZE / heads_per_wg; // 64 threads per head
237
-
238
- let base_head = workgroup_id.x * heads_per_wg;
239
- let head_in_wg = local_id.x / threads_per_head;
240
- let tid_in_head = local_id.x % threads_per_head;
241
- let head_idx = base_head + head_in_wg;
242
-
243
- let head_dim = u.head_dim;
244
- let kv_len = get_kv_len();
245
- let active_head = head_idx < u.num_heads;
246
- let valid_dim = head_dim <= 64u;
247
-
248
- let heads_per_kv = u.num_heads / u.num_kv_heads;
249
- let kv_head_idx = head_idx / heads_per_kv;
250
- let scale = 1.0 / sqrt(f32(head_dim));
251
-
252
- // Each head gets a slice of shared memory
253
- let q_base = head_in_wg * 64u;
254
-
255
- // Load Q vector for this head
256
- if (active_head && valid_dim && tid_in_head < head_dim) {
257
- let q_offset = head_idx * head_dim + tid_in_head;
258
- shared_q[q_base + tid_in_head] = Q[q_offset];
259
- }
260
- workgroupBarrier();
261
-
262
- if (!active_head || !valid_dim) {
263
- return;
264
- }
265
-
266
- if (kv_len == 0u) {
267
- if (tid_in_head < head_dim) {
268
- let out_offset = head_idx * head_dim + tid_in_head;
269
- output[out_offset] = 0.0;
270
- }
271
- return;
272
- }
273
-
274
- // Online softmax accumulators
275
- var running_max: f32 = -1e38;
276
- var running_sum: f32 = 0.0;
277
- var out_accum: f32 = 0.0;
278
-
279
- // Process KV cache - each thread in head group processes multiple K positions
280
- let k_stride = threads_per_head;
281
-
282
- for (var k_start = 0u; k_start < kv_len; k_start += k_stride) {
283
- let k_pos = k_start + tid_in_head;
284
- let valid_k = k_pos < kv_len;
285
-
286
- // Compute attention score
287
- var score: f32 = -1e38;
288
- if (valid_k) {
289
- let k_base = k_pos * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
290
- var dot: f32 = 0.0;
291
- for (var d = 0u; d < head_dim; d++) {
292
- dot += shared_q[q_base + d] * bitcast<f32>(K_cache[k_base + d]);
293
- }
294
- score = dot * scale;
295
- }
296
-
297
- // Online softmax update for this batch
298
- let new_max = max(running_max, score);
299
- let rescale = exp(running_max - new_max);
300
- let exp_score = select(0.0, exp(score - new_max), valid_k);
301
-
302
- running_sum = running_sum * rescale + exp_score;
303
- running_max = new_max;
304
-
305
- // Accumulate V contribution
306
- if (tid_in_head < head_dim && valid_k) {
307
- out_accum *= rescale;
308
- let v_offset = k_pos * u.num_kv_heads * head_dim + kv_head_idx * head_dim + tid_in_head;
309
- out_accum += exp_score * bitcast<f32>(V_cache[v_offset]);
310
- }
311
- }
312
-
313
- // Reduce across threads in this head group
314
- // Note: This requires cross-subgroup communication
315
- // For simplicity, use shared memory reduction
316
-
317
- // Write partial output
318
- if (tid_in_head < head_dim) {
319
- let out_offset = head_idx * head_dim + tid_in_head;
320
- let inv_sum = select(0.0, 1.0 / running_sum, running_sum > 0.0);
321
- output[out_offset] = out_accum * inv_sum;
322
- }
323
- }
324
-
325
- // F16 KV cache variant - optimized for memory bandwidth
326
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
327
- fn main_f16kv(
328
- @builtin(local_invocation_id) local_id: vec3<u32>,
329
- @builtin(workgroup_id) workgroup_id: vec3<u32>,
330
- @builtin(subgroup_size) subgroup_size: u32,
331
- @builtin(subgroup_invocation_id) sg_tid: u32,
332
- ) {
333
- let head_idx = workgroup_id.x;
334
- let tid = local_id.x;
335
- let head_dim = u.head_dim;
336
- let kv_len = get_kv_len();
337
-
338
- let heads_per_kv = u.num_heads / u.num_kv_heads;
339
- let kv_head_idx = head_idx / heads_per_kv;
340
-
341
- let subgroup_id = tid / subgroup_size;
342
- let num_subgroups = (WORKGROUP_SIZE + subgroup_size - 1u) / subgroup_size;
343
-
344
- let scale = 1.0 / sqrt(f32(head_dim));
345
-
346
- // Load Q (F32) into shared memory
347
- if (tid < head_dim) {
348
- let q_offset = head_idx * head_dim + tid;
349
- shared_q[tid] = Q[q_offset];
350
- }
351
- workgroupBarrier();
352
-
353
- var running_max: f32 = -1e38;
354
- var running_sum: f32 = 0.0;
355
- var out_accum: f32 = 0.0;
356
-
357
- // Process in chunks
358
- for (var k_start = 0u; k_start < kv_len; k_start += WORKGROUP_SIZE) {
359
- let k_pos = k_start + tid;
360
- let valid_k = k_pos < kv_len;
361
-
362
- var score: f32 = -1e38;
363
- if (valid_k) {
364
- let k_base = k_pos * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
365
- var dot: f32 = 0.0;
366
-
367
- // K cache is F16, load and convert
368
- for (var d = 0u; d < head_dim; d += 2u) {
369
- let q0 = shared_q[d];
370
- let k_vec = unpack2x16float(K_cache[(k_base + d) >> 1u]);
371
- dot += q0 * k_vec.x;
372
- if (d + 1u < head_dim) {
373
- let q1 = shared_q[d + 1u];
374
- dot += q1 * k_vec.y;
375
- }
376
- }
377
- score = dot * scale;
378
- }
379
-
380
- shared_scores[tid] = score;
381
-
382
- // Find chunk max
383
- var chunk_max = subgroupMax(score);
384
- if (sg_tid == 0u && subgroup_id < num_subgroups) {
385
- sg_max[subgroup_id] = chunk_max;
386
- }
387
- workgroupBarrier();
388
-
389
- if (tid == 0u) {
390
- var m = -1e38;
391
- for (var s = 0u; s < num_subgroups; s++) {
392
- m = max(m, sg_max[s]);
393
- }
394
- global_max = m;
395
- }
396
- workgroupBarrier();
397
-
398
- let chunk_max_val = global_max;
399
- let new_max = max(running_max, chunk_max_val);
400
- let rescale = exp(running_max - new_max);
401
-
402
- var exp_score = select(0.0, exp(score - new_max), valid_k);
403
- shared_scores[tid] = exp_score;
404
-
405
- var chunk_sum = subgroupAdd(exp_score);
406
- if (sg_tid == 0u && subgroup_id < num_subgroups) {
407
- sg_sum[subgroup_id] = chunk_sum;
408
- }
409
- workgroupBarrier();
410
-
411
- if (tid == 0u) {
412
- var s = 0.0;
413
- for (var i = 0u; i < num_subgroups; i++) {
414
- s += sg_sum[i];
415
- }
416
- global_sum = s;
417
- }
418
- workgroupBarrier();
419
-
420
- running_sum = running_sum * rescale + global_sum;
421
- running_max = new_max;
422
-
423
- // Accumulate V (F16 cache)
424
- if (tid < head_dim) {
425
- out_accum *= rescale;
426
- for (var k = 0u; k < min(WORKGROUP_SIZE, kv_len - k_start); k++) {
427
- let k_pos_inner = k_start + k;
428
- let v_base = k_pos_inner * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
429
-
430
- // Read packed F16 V values
431
- let v_vec = unpack2x16float(V_cache[(v_base + tid) >> 1u]);
432
- let v_val = select(v_vec.x, v_vec.y, (tid & 1u) == 1u);
433
-
434
- out_accum += shared_scores[k] * v_val;
435
- }
436
- }
437
- workgroupBarrier();
438
- }
439
-
440
- if (tid < head_dim) {
441
- let out_offset = head_idx * head_dim + tid;
442
- let inv_sum = select(0.0, 1.0 / running_sum, running_sum > 0.0);
443
- output[out_offset] = out_accum * inv_sum;
444
- }
445
- }
@@ -1,116 +0,0 @@
1
- // MXFP4 Dequantization Kernel (vec4)
2
- //
3
- // Vectorized variant for MXFP4 dequantization.
4
-
5
- // Tunable workgroup size
6
- override WORKGROUP_SIZE_VEC4: u32 = 64u;
7
-
8
- struct Uniforms {
9
- total_elements: u32, // Total output elements
10
- num_groups: u32, // Groups per row (e.g., 90)
11
- group_size: u32, // Elements per group (32 = 16 bytes * 2 nibbles)
12
- row_stride: u32, // Stride between rows in output
13
- }
14
-
15
- @group(0) @binding(0) var<uniform> u: Uniforms;
16
- @group(0) @binding(1) var<storage, read> blocks: array<u32>; // Packed U8 as U32
17
- @group(0) @binding(2) var<storage, read> scales: array<u32>; // Packed U8 as U32
18
- @group(0) @binding(3) var<storage, read_write> output: array<f32>;
19
-
20
- // Extract 4-bit nibble from packed bytes and decode as E2M1 FP4
21
- // MXFP4 E2M1 format: 1 sign bit, 2 exponent bits (bias=1), 1 mantissa bit
22
- // Layout: S | E1 | E0 | M
23
- fn get_nibble(byte_data: u32, nibble_idx: u32) -> f32 {
24
- // Each U32 contains 4 bytes, each byte contains 2 nibbles
25
- let byte_idx = nibble_idx / 2u;
26
- let is_high = nibble_idx % 2u;
27
- let byte_val = (byte_data >> (byte_idx * 8u)) & 0xFFu;
28
-
29
- var nibble: u32;
30
- if (is_high == 1u) {
31
- nibble = (byte_val >> 4u) & 0xFu;
32
- } else {
33
- nibble = byte_val & 0xFu;
34
- }
35
-
36
- // Decode E2M1 FP4:
37
- // nibble = SEEM (4 bits)
38
- // S = sign bit (bit 3)
39
- // E = exponent (bits 2-1), bias = 1
40
- // M = mantissa (bit 0)
41
- let sign_bit = (nibble >> 3u) & 1u;
42
- let exp = (nibble >> 1u) & 3u; // 2-bit exponent
43
- let mantissa = nibble & 1u; // 1-bit mantissa
44
-
45
- var value: f32;
46
- if (exp == 0u) {
47
- // Subnormal: value = (-1)^S * 0.5 * M
48
- value = f32(mantissa) * 0.5;
49
- } else {
50
- // Normal: value = (-1)^S * (1 + 0.5*M) * 2^(E-1)
51
- let m = 1.0 + f32(mantissa) * 0.5;
52
- value = m * pow(2.0, f32(exp) - 1.0);
53
- }
54
-
55
- // Apply sign
56
- if (sign_bit == 1u) {
57
- value = -value;
58
- }
59
- return value;
60
- }
61
-
62
- // Get scale value from packed scales array (E8M0 format)
63
- fn get_scale(scale_data: u32, idx: u32) -> f32 {
64
- let byte_idx = idx % 4u;
65
- let scale_byte = (scale_data >> (byte_idx * 8u)) & 0xFFu;
66
- // E8M0 format: 8-bit exponent, no mantissa
67
- // scale = 2^(exponent - 127) where 127 is the IEEE bias
68
- // Special cases: 0 = zero, 255 = NaN (we treat as 0)
69
- if (scale_byte == 0u || scale_byte == 255u) {
70
- return 0.0;
71
- }
72
- let exponent = i32(scale_byte) - 127;
73
- return pow(2.0, f32(exponent));
74
- }
75
-
76
- // Vectorized version - each thread handles 4 elements
77
- @compute @workgroup_size(WORKGROUP_SIZE_VEC4, 1, 1)
78
- fn main_vec4(
79
- @builtin(global_invocation_id) global_id: vec3<u32>
80
- ) {
81
- let base_elem = global_id.x * 4u;
82
-
83
- if (base_elem >= u.total_elements) {
84
- return;
85
- }
86
-
87
- // Compute group for base element
88
- let group_idx = base_elem / u.group_size;
89
- let intra_group_base = base_elem % u.group_size;
90
-
91
- // Get scale for this group (all 4 elements should be in same group typically)
92
- let scale_word_idx = group_idx / 4u;
93
- let scale_byte_idx = group_idx % 4u;
94
- let scale_word = scales[scale_word_idx];
95
- let scale = get_scale(scale_word, scale_byte_idx);
96
-
97
- // Get block data
98
- let block_base = group_idx * 4u;
99
-
100
- // Process 4 consecutive nibbles
101
- for (var i = 0u; i < 4u; i = i + 1u) {
102
- let elem_idx = base_elem + i;
103
- if (elem_idx >= u.total_elements) {
104
- break;
105
- }
106
-
107
- let intra_group_idx = intra_group_base + i;
108
- let word_in_block = intra_group_idx / 8u;
109
- let nibble_in_word = intra_group_idx % 8u;
110
-
111
- let block_word = blocks[block_base + word_in_block];
112
- let nibble_val = get_nibble(block_word, nibble_in_word);
113
-
114
- output[elem_idx] = f32(nibble_val) * scale;
115
- }
116
- }
@@ -1,98 +0,0 @@
1
- // Q8_0 Dequantization Kernel - f16 Output
2
- //
3
- // Dequantizes Q8_0 blocks (8-bit quantization from llama.cpp/GGUF).
4
- //
5
- // Q8_0 block layout (34 bytes per 32 elements):
6
- // - d: 2 bytes at offset 0 (f16 scale)
7
- // - qs: 32 bytes at offset 2 (int8 quantized values)
8
- //
9
- // Algorithm from ggml-quants.c dequantize_row_q8_0
10
-
11
- enable f16;
12
-
13
- // Q8_0 constants
14
- const QK8_0: u32 = 32u;
15
- const Q8_0_BLOCK_BYTES: u32 = 34u;
16
-
17
- // Byte offsets in Q8_0 block
18
- const D_OFFSET: u32 = 0u; // 2 bytes (f16 scale)
19
- const QS_OFFSET: u32 = 2u; // 32 bytes (int8 values)
20
-
21
- // Tunable workgroup size
22
- override WORKGROUP_SIZE: u32 = 32u;
23
-
24
- struct Uniforms {
25
- num_blocks: u32,
26
- output_offset: u32,
27
- workgroups_x: u32, // For 2D dispatch: blocks per row
28
- _pad1: u32,
29
- }
30
-
31
- @group(0) @binding(0) var<uniform> u: Uniforms;
32
- @group(0) @binding(1) var<storage, read> quantized: array<u32>;
33
- @group(0) @binding(2) var<storage, read_write> output: array<f16>;
34
-
35
- var<workgroup> shared_d: f32;
36
-
37
- // Read a byte from the quantized buffer at a given block and byte offset
38
- fn read_byte(block_idx: u32, byte_offset: u32) -> u32 {
39
- let global_byte = block_idx * Q8_0_BLOCK_BYTES + byte_offset;
40
- let word_idx = global_byte / 4u;
41
- let byte_in_word = global_byte % 4u;
42
- return (quantized[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
43
- }
44
-
45
- // Read a u16 (little-endian) from the quantized buffer
46
- fn read_u16(block_idx: u32, byte_offset: u32) -> u32 {
47
- let lo = read_byte(block_idx, byte_offset);
48
- let hi = read_byte(block_idx, byte_offset + 1u);
49
- return lo | (hi << 8u);
50
- }
51
-
52
- // Read signed i8 as f32
53
- fn read_i8_as_f32(block_idx: u32, byte_offset: u32) -> f32 {
54
- let byte_val = read_byte(block_idx, byte_offset);
55
- // Convert u8 to i8 via two's complement
56
- if (byte_val >= 128u) {
57
- return f32(i32(byte_val) - 256);
58
- }
59
- return f32(byte_val);
60
- }
61
-
62
- // Unpack f16 from u32 (low 16 bits)
63
- fn unpack_f16(packed: u32) -> f32 {
64
- return unpack2x16float(packed).x;
65
- }
66
-
67
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
68
- fn main(
69
- @builtin(local_invocation_id) local_id: vec3<u32>,
70
- @builtin(workgroup_id) workgroup_id: vec3<u32>
71
- ) {
72
- // Handle 2D dispatch for large block counts (> 65535)
73
- let block_idx = workgroup_id.x + workgroup_id.y * u.workgroups_x;
74
- let elem_idx = local_id.x;
75
-
76
- if (block_idx >= u.num_blocks) {
77
- return;
78
- }
79
-
80
- // Thread 0 loads d (f16 at offset 0)
81
- if (elem_idx == 0u) {
82
- let d_packed = read_u16(block_idx, D_OFFSET);
83
- shared_d = unpack_f16(d_packed);
84
- }
85
-
86
- workgroupBarrier();
87
-
88
- let d = shared_d;
89
-
90
- // Read quantized value as signed int8
91
- let q = read_i8_as_f32(block_idx, QS_OFFSET + elem_idx);
92
-
93
- // Dequantize: output = d * q
94
- let dequant = d * q;
95
-
96
- let out_idx = u.output_offset + block_idx * QK8_0 + elem_idx;
97
- output[out_idx] = f16(dequant);
98
- }