doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,35 +1,33 @@
1
- --- /Users/xyz/deco/doppler/src/gpu/kernels/rmsnorm.wgsl 2026-03-23 16:49:01
2
- +++ /Users/xyz/deco/doppler/src/gpu/kernels/rmsnorm_f16.wgsl 2026-03-23 16:49:01
3
- @@ -1,45 +1,27 @@
1
+ --- src/gpu/kernels/rmsnorm.wgsl
2
+ +++ src/gpu/kernels/rmsnorm_f16.wgsl
3
+ @@ -1,46 +1,24 @@
4
4
  -// RMSNorm Kernel with Fused Residual Add
5
- +// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
6
- +// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
7
- +// RMSNorm Kernel with F16 Input/Output
8
5
  //
9
6
  -// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
10
- +// F16 variant for reduced memory bandwidth when using F16 activations.
11
- +// Intermediate computations (sum of squares, RMS) remain in F32 for precision.
12
- +// Weight buffer may be F16 or F32 (small size, precision matters).
13
- //
7
+ -//
14
8
  -// Optionally fuses residual addition (POST-norm, for Gemma 3 sandwich norm):
15
9
  -// output = residual + RMSNorm(x) * weight
16
10
  -//
17
11
  -// Uses workgroup reduction for efficient mean calculation.
18
12
  -// Subgroup variants use subgroupAdd() for 3-5x faster reductions.
19
- -//
13
+ +// F16 variant for reduced memory bandwidth when using F16 activations.
14
+ +// The all-f16 lane keeps normalization arithmetic in f16.
15
+ +// Weight buffer may be F16 or F32.
16
+ //
20
17
  -// Uses override constants for compile-time feature selection:
21
18
  -// - RMS_NORM_OFFSET: Use (1 + weight) pattern (Gemma models)
22
- -// - HAS_RESIDUAL: Enable post-norm residual addition
19
+ -// - HAS_RESIDUAL: Compile-time override (reserved for codegen patches, not used by JS dispatch)
23
20
  -// - OUTPUT_PRENORM: Output pre-normalized values (before weight multiplication)
24
- +// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
25
-
21
+ -
26
22
  -enable subgroups;
27
- +enable f16;
28
-
23
+ -
29
24
  -// =============================================================================
30
25
  -// Override Constants (compile-time configuration)
31
26
  -// =============================================================================
32
- -
27
+ +// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
28
+ +
29
+ +enable f16;
30
+
33
31
  override WORKGROUP_SIZE: u32 = 256u;
34
32
  const MAX_WORKGROUP_SIZE: u32 = 256u;
35
33
  -
@@ -39,8 +37,9 @@
39
37
  override PRE_RESIDUAL: bool = false; // Add residual BEFORE normalization: rmsnorm(input + residual)
40
38
  override OUTPUT_PRENORM: bool = false; // Write pre-norm sum to residual_sum_output binding
41
39
  override WEIGHT_IS_F16: bool = false; // Weight buffer packed as f16 pairs
42
-
40
+
43
41
  -const MAX_SUBGROUPS: u32 = 32u; // Support up to 32 subgroups per workgroup
42
+ -const MAX_CACHE_SIZE: u32 = 4608u; // Supports hiddenSize up to 1152 with 4 elements/thread
44
43
  -
45
44
  -// =============================================================================
46
45
  -// Uniforms (per-dispatch)
@@ -56,77 +55,81 @@
56
55
  token_stride: u32, // Workgroup rows per dispatch row
57
56
  _pad0: u32,
58
57
  _pad1: u32,
59
- @@ -47,29 +29,20 @@
58
+ @@ -48,47 +26,32 @@
60
59
  }
61
-
60
+
62
61
  @group(0) @binding(0) var<uniform> u: Uniforms;
63
62
  -@group(0) @binding(1) var<storage, read> input: array<f32>;
64
63
  -@group(0) @binding(2) var<storage, read> weight: array<u32>; // [size] as f32 or packed f16
65
64
  -@group(0) @binding(3) var<storage, read_write> output: array<f32>;
66
65
  -@group(0) @binding(4) var<storage, read> residual: array<f32>; // Optional residual input
67
66
  -@group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f32>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
68
- +@group(0) @binding(1) var<storage, read> input: array<f16>;
69
- +@group(0) @binding(2) var<storage, read> weight: array<u32>; // F32 or packed F16
70
- +@group(0) @binding(3) var<storage, read_write> output: array<f16>;
71
- +@group(0) @binding(4) var<storage, read> residual: array<f16>; // Optional residual
72
- +@group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f16>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
73
-
67
+ -
74
68
  -// =============================================================================
75
69
  -// Shared Memory
76
70
  -// =============================================================================
77
71
  -
78
- +// Shared memory for reduction (F32 for precision)
79
- var<workgroup> shared_sum: array<f32, MAX_WORKGROUP_SIZE>;
80
-
72
+ -var<workgroup> shared_sum: array<f32, MAX_WORKGROUP_SIZE>;
73
+ -
81
74
  -// =============================================================================
82
75
  -// Helper Functions
83
76
  -// =============================================================================
84
- -
77
+ +@group(0) @binding(1) var<storage, read> input: array<f16>;
78
+ +@group(0) @binding(2) var<storage, read> weight: array<u32>; // F32 or packed F16
79
+ +@group(0) @binding(3) var<storage, read_write> output: array<f16>;
80
+ +@group(0) @binding(4) var<storage, read> residual: array<f16>; // Optional residual
81
+ +@group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f16>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
82
+
85
83
  -// Apply weight with optional offset (Gemma uses 1 + weight pattern)
86
- fn apply_weight(w: f32) -> f32 {
84
+ -fn apply_weight(w: f32) -> f32 {
85
+ +var<workgroup> shared_sum: array<f16, MAX_WORKGROUP_SIZE>;
86
+ +
87
+ +fn apply_weight(w: f16) -> f16 {
87
88
  if (RMS_NORM_OFFSET) {
88
- return 1.0 + w;
89
+ - return 1.0 + w;
89
90
  - } else {
90
91
  - return w;
92
+ + return f16(1.0) + w;
91
93
  }
92
94
  + return w;
93
95
  }
94
-
95
- fn load_weight(idx: u32) -> f32 {
96
- @@ -81,18 +54,11 @@
97
- return bitcast<f32>(weight[idx]);
98
- }
99
-
100
- -// Check if residual should be added (compile-time OR runtime flag)
101
- -fn should_add_residual() -> bool {
102
- - // HAS_RESIDUAL is compile-time override (for optimized paths)
103
- - // u.has_residual is runtime flag (for flexibility)
104
- - return HAS_RESIDUAL || (u.has_residual != 0u);
96
+
97
+ -fn load_weight(idx: u32) -> f32 {
98
+ +fn load_weight(idx: u32) -> f16 {
99
+ if (WEIGHT_IS_F16) {
100
+ let packed = weight[idx >> 1u];
101
+ let pair = unpack2x16float(packed);
102
+ - return select(pair.x, pair.y, (idx & 1u) == 1u);
103
+ + return f16(select(pair.x, pair.y, (idx & 1u) == 1u));
104
+ }
105
+ - return bitcast<f32>(weight[idx]);
105
106
  -}
106
107
  -
108
+ -// Check if residual should be added (runtime uniform only — JS dispatch controls this)
109
+ -fn should_add_residual() -> bool {
110
+ - return u.has_residual != 0u;
111
+ + return f16(bitcast<f32>(weight[idx]));
112
+ }
113
+
107
114
  // Load input value, fusing residual add when PRE_RESIDUAL is active
108
- fn load_input(base_offset: u32, idx: u32) -> f32 {
109
- - let x = input[base_offset + idx];
110
- + let x = f32(input[base_offset + idx]);
115
+ -fn load_input(base_offset: u32, idx: u32) -> f32 {
116
+ +fn load_input(base_offset: u32, idx: u32) -> f16 {
117
+ let x = input[base_offset + idx];
111
118
  if (PRE_RESIDUAL) {
112
- - return x + residual[base_offset + idx];
113
- + return x + f32(residual[base_offset + idx]);
114
- }
115
- return x;
119
+ return x + residual[base_offset + idx];
120
+ @@ -97,7 +60,7 @@
116
121
  }
117
- @@ -100,7 +66,7 @@
122
+
118
123
  // Write pre-norm sum when OUTPUT_PRENORM is active (for downstream residual reuse)
119
- fn write_prenorm(base_offset: u32, idx: u32, val: f32) {
124
+ -fn write_prenorm(base_offset: u32, idx: u32, val: f32) {
125
+ +fn write_prenorm(base_offset: u32, idx: u32, val: f16) {
120
126
  if (OUTPUT_PRENORM && PRE_RESIDUAL) {
121
- - residual_sum_output[base_offset + idx] = val;
122
- + residual_sum_output[base_offset + idx] = f16(val);
127
+ residual_sum_output[base_offset + idx] = val;
123
128
  }
124
- }
125
-
126
- @@ -108,10 +74,6 @@
129
+ @@ -107,10 +70,6 @@
127
130
  return wg_id.y * max(u.token_stride, 1u) + wg_id.x;
128
131
  }
129
-
132
+
130
133
  -// =============================================================================
131
134
  -// Main Entry Point
132
135
  -// =============================================================================
@@ -134,74 +137,111 @@
134
137
  // Main RMSNorm kernel - one workgroup per token
135
138
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
136
139
  fn main(
137
- @@ -129,7 +91,7 @@
138
-
140
+ @@ -128,151 +87,44 @@
141
+
139
142
  let base_offset = token_idx * size;
140
-
143
+
141
144
  - // Each thread computes partial sum of squares
142
- + // Each thread computes partial sum of squares (promote to F32 for precision)
143
- // When PRE_RESIDUAL is active, sum is computed on (input + residual)
144
- var local_sum_sq: f32 = 0.0;
145
+ - // When PRE_RESIDUAL is active, sum is computed on (input + residual)
146
+ - var local_sum_sq: f32 = 0.0;
145
147
  let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
146
- @@ -154,7 +116,7 @@
148
+
149
+ + var local_max: f16 = f16(0.0);
150
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
151
+ let idx = thread_idx * elements_per_thread + i;
152
+ if (idx < size) {
153
+ - let x = load_input(base_offset, idx);
154
+ - local_sum_sq = local_sum_sq + x * x;
155
+ + local_max = max(local_max, abs(load_input(base_offset, idx)));
156
+ }
157
+ }
158
+
159
+ - // Store local sum for reduction
160
+ - shared_sum[thread_idx] = local_sum_sq;
161
+ + shared_sum[thread_idx] = local_max;
162
+ workgroupBarrier();
163
+
164
+ - // Parallel reduction to compute total sum of squares
165
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
166
+ if (thread_idx < stride) {
167
+ - shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
168
+ + shared_sum[thread_idx] = max(shared_sum[thread_idx], shared_sum[thread_idx + stride]);
169
+ }
147
170
  workgroupBarrier();
148
171
  }
149
-
172
+
150
173
  - // Compute RMS
151
- + // Compute RMS (F32 for precision)
152
- let mean_sq = shared_sum[0] / f32(size);
153
- let rms = sqrt(mean_sq + u.eps);
154
- let inv_rms = 1.0 / rms;
155
- @@ -167,29 +129,26 @@
156
- if (idx < size) {
157
- let x = load_input(base_offset, idx);
158
-
174
+ - let mean_sq = shared_sum[0] / f32(size);
175
+ - let rms = sqrt(mean_sq + u.eps);
176
+ - let inv_rms = 1.0 / rms;
177
+ -
178
+ + let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
179
+ workgroupBarrier();
180
+
181
+ - // Apply normalization and weight
182
+ - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
183
+ - let idx = thread_idx * elements_per_thread + i;
184
+ - if (idx < size) {
185
+ - let x = load_input(base_offset, idx);
186
+ -
159
187
  - // Normalize and scale (with optional weight offset for Gemma)
160
- + // Normalize and scale (compute in F32)
161
- var result = x * inv_rms * apply_weight(load_weight(idx));
162
-
188
+ - var result = x * inv_rms * apply_weight(load_weight(idx));
189
+ -
163
190
  - // Add residual AFTER normalization (Gemma 3 sandwich norm pattern)
164
191
  - if (should_add_residual()) {
165
192
  - result = result + residual[base_offset + idx];
166
- + // Add residual AFTER normalization
167
- + if (u.has_residual == 1u) {
168
- + result = result + f32(residual[base_offset + idx]);
169
- }
170
-
171
- // Write pre-norm sum for downstream residual reuse
172
- write_prenorm(base_offset, idx, x);
173
-
193
+ - }
194
+ -
195
+ - // Write pre-norm sum for downstream residual reuse
196
+ - write_prenorm(base_offset, idx, x);
197
+ -
174
198
  - output[base_offset + idx] = result;
175
- + // Convert back to F16 for output
176
- + output[base_offset + idx] = f16(result);
177
- }
178
- }
179
- }
180
-
199
+ - }
200
+ - }
201
+ -}
202
+ -
181
203
  -// =============================================================================
182
204
  -// Small Hidden Size Entry Point
183
205
  -// =============================================================================
184
206
  -
185
- // Optimized version for hidden size <= WORKGROUP_SIZE (single pass)
186
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
207
+ -// Optimized version for hidden size <= WORKGROUP_SIZE (single pass)
208
+ -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
187
209
  -fn main_small(
188
- +fn rmsnorm_small_f16(
189
- @builtin(local_invocation_id) local_id: vec3<u32>,
190
- @builtin(workgroup_id) wg_id: vec3<u32>
191
- ) {
192
- @@ -209,7 +168,7 @@
193
- x = load_input(base_offset, thread_idx);
194
- }
195
-
210
+ - @builtin(local_invocation_id) local_id: vec3<u32>,
211
+ - @builtin(workgroup_id) wg_id: vec3<u32>
212
+ -) {
213
+ - let token_idx = token_index(wg_id);
214
+ - let thread_idx = local_id.x;
215
+ - let size = u.size;
216
+ -
217
+ - if (token_idx >= u.num_tokens) {
218
+ - return;
219
+ - }
220
+ -
221
+ - let base_offset = token_idx * size;
222
+ -
223
+ - // Each thread handles one element (for size <= 256)
224
+ - var x: f32 = 0.0;
225
+ - if (thread_idx < size) {
226
+ - x = load_input(base_offset, thread_idx);
227
+ - }
228
+ -
196
229
  - // Sum of squares
197
- + // Sum of squares (F32 for precision)
198
- shared_sum[thread_idx] = x * x;
199
- workgroupBarrier();
200
-
201
- @@ -225,225 +184,13 @@
202
- let mean_sq = shared_sum[0] / f32(size);
203
- let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
204
-
230
+ - shared_sum[thread_idx] = x * x;
231
+ - workgroupBarrier();
232
+ -
233
+ - // Parallel reduction
234
+ - for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
235
+ - if (thread_idx < stride && thread_idx + stride < size) {
236
+ - shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
237
+ - }
238
+ - workgroupBarrier();
239
+ - }
240
+ -
241
+ - // Compute inverse RMS
242
+ - let mean_sq = shared_sum[0] / f32(size);
243
+ - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
244
+ -
205
245
  - // Apply normalization
206
246
  - if (thread_idx < size) {
207
247
  - var result = x * inv_rms * apply_weight(load_weight(thread_idx));
@@ -219,7 +259,7 @@
219
259
  -
220
260
  -// OPTIMIZED: Caches input to avoid double loads
221
261
  -// Uses shared memory to store input values between passes
222
- -var<workgroup> shared_cache: array<f32, 4608>; // Max for hiddenSize=1152 × 4 elements/thread
262
+ -var<workgroup> shared_cache: array<f32, MAX_CACHE_SIZE>;
223
263
  -
224
264
  -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
225
265
  -fn main_cached(
@@ -239,30 +279,39 @@
239
279
  -
240
280
  - // First pass: cache input and compute sum of squares
241
281
  - var local_sum_sq: f32 = 0.0;
242
- - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
243
- - let idx = thread_idx * elements_per_thread + i;
244
- - if (idx < size) {
282
+ + var local_sum_sq: f16 = f16(0.0);
283
+ + let inv_size = f16(1.0) / f16(size);
284
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
285
+ let idx = thread_idx * elements_per_thread + i;
286
+ if (idx < size) {
245
287
  - let x = load_input(base_offset, idx);
246
288
  - shared_cache[idx] = x; // Cache for second pass (avoids reloading residual)
247
289
  - local_sum_sq = local_sum_sq + x * x;
248
- - }
249
- - }
250
- -
251
- - shared_sum[thread_idx] = local_sum_sq;
252
- - workgroupBarrier();
253
- -
254
- - for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
255
- - if (thread_idx < stride) {
256
- - shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
257
- - }
258
- - workgroupBarrier();
259
- - }
260
- -
290
+ + let x = load_input(base_offset, idx) / row_scale;
291
+ + local_sum_sq = local_sum_sq + (x * x) * inv_size;
292
+ }
293
+ }
294
+
295
+ + // Store local sum for reduction
296
+ shared_sum[thread_idx] = local_sum_sq;
297
+ workgroupBarrier();
298
+
299
+ + // Parallel reduction to compute total sum of squares
300
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
301
+ if (thread_idx < stride) {
302
+ shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
303
+ @@ -280,117 +132,38 @@
304
+ workgroupBarrier();
305
+ }
306
+
261
307
  - let mean_sq = shared_sum[0] / f32(size);
262
308
  - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
263
- -
264
- - workgroupBarrier();
265
- -
309
+ + let mean_sq = shared_sum[0];
310
+ + let eps_scaled = f16(u.eps) / (row_scale * row_scale);
311
+ + let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
312
+
313
+ workgroupBarrier();
314
+
266
315
  - // Second pass: normalize cached input
267
316
  - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
268
317
  - let idx = thread_idx * elements_per_thread + i;
@@ -344,87 +393,108 @@
344
393
  - let mean_sq = total_sum / f32(size);
345
394
  - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
346
395
  -
347
- - // Apply normalization and weight
348
- - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
349
- - let idx = thread_idx * elements_per_thread + i;
350
- - if (idx < size) {
351
- - let x = load_input(base_offset, idx);
352
- - var result = x * inv_rms * apply_weight(load_weight(idx));
353
- -
396
+ // Apply normalization and weight
397
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
398
+ let idx = thread_idx * elements_per_thread + i;
399
+ if (idx < size) {
400
+ let x = load_input(base_offset, idx);
401
+ +
402
+ var result = x * inv_rms * apply_weight(load_weight(idx));
403
+
354
404
  - if (should_add_residual()) {
355
- - result = result + residual[base_offset + idx];
356
- - }
357
- -
358
- - write_prenorm(base_offset, idx, x);
359
- - output[base_offset + idx] = result;
360
- - }
361
- - }
362
- -}
363
- -
405
+ + // Add residual AFTER normalization
406
+ + if (u.has_residual == 1u) {
407
+ result = result + residual[base_offset + idx];
408
+ }
409
+
410
+ + // Write pre-norm sum for downstream residual reuse
411
+ write_prenorm(base_offset, idx, x);
412
+ +
413
+ output[base_offset + idx] = result;
414
+ }
415
+ }
416
+ }
417
+
364
418
  -// Subgroup-accelerated RMSNorm for small hidden sizes (size <= 256)
365
419
  -// Each thread handles one element, uses subgroup reduction
366
- -@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
420
+ +// Optimized version for hidden size <= WORKGROUP_SIZE (single pass)
421
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
367
422
  -fn main_small_subgroup(
368
- - @builtin(local_invocation_id) local_id: vec3<u32>,
423
+ +fn rmsnorm_small_f16(
424
+ @builtin(local_invocation_id) local_id: vec3<u32>,
369
425
  - @builtin(workgroup_id) wg_id: vec3<u32>,
370
426
  - @builtin(subgroup_invocation_id) sg_lane: u32,
371
427
  - @builtin(subgroup_size) sg_size: u32,
372
- -) {
373
- - let token_idx = token_index(wg_id);
374
- - let thread_idx = local_id.x;
375
- - let size = u.size;
376
- -
377
- - if (token_idx >= u.num_tokens) {
378
- - return;
379
- - }
380
- -
381
- - let base_offset = token_idx * size;
428
+ + @builtin(workgroup_id) wg_id: vec3<u32>
429
+ ) {
430
+ let token_idx = token_index(wg_id);
431
+ let thread_idx = local_id.x;
432
+ @@ -401,45 +174,46 @@
433
+ }
434
+
435
+ let base_offset = token_idx * size;
382
436
  - let subgroup_id = thread_idx / sg_size;
383
437
  - let num_subgroups = (WORKGROUP_SIZE + sg_size - 1u) / sg_size;
384
- -
438
+
385
439
  - // Each thread handles one element
386
440
  - var x: f32 = 0.0;
387
441
  - var x_sq: f32 = 0.0;
388
- + // Apply normalization (compute in F32, output F16)
442
+ + // Each thread handles one element (for size <= 256)
443
+ + var x: f16 = f16(0.0);
389
444
  if (thread_idx < size) {
390
- - x = load_input(base_offset, thread_idx);
445
+ x = load_input(base_offset, thread_idx);
391
446
  - x_sq = x * x;
392
- - }
393
- -
447
+ }
448
+
394
449
  - // Phase 1: Subgroup reduction
395
450
  - let sg_sum = subgroupAdd(x_sq);
396
- -
451
+ + shared_sum[thread_idx] = abs(x);
452
+ + workgroupBarrier();
453
+
397
454
  - // Phase 2: Store to shared memory
398
455
  - if (sg_lane == 0u && subgroup_id < num_subgroups) {
399
456
  - sg_partial_sums[subgroup_id] = sg_sum;
400
- - }
401
- - workgroupBarrier();
402
- -
457
+ + for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
458
+ + if (thread_idx < stride) {
459
+ + shared_sum[thread_idx] = max(shared_sum[thread_idx], shared_sum[thread_idx + stride]);
460
+ + }
461
+ + workgroupBarrier();
462
+ }
463
+ +
464
+ + let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
465
+ + let x_scaled = x / row_scale;
466
+ + let inv_size = f16(1.0) / f16(size);
467
+ + shared_sum[thread_idx] = (x_scaled * x_scaled) * inv_size;
468
+ workgroupBarrier();
469
+
403
470
  - // Phase 3: Final reduction - thread 0 sums, then broadcast via shared memory
404
471
  - if (thread_idx == 0u) {
405
472
  - var sum: f32 = 0.0;
406
473
  - for (var s = 0u; s < num_subgroups; s++) {
407
474
  - sum += sg_partial_sums[s];
408
- - }
475
+ + // Parallel reduction
476
+ + for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
477
+ + if (thread_idx < stride && thread_idx + stride < size) {
478
+ + shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
479
+ }
409
480
  - sg_partial_sums[0] = sum;
410
- - }
481
+ + workgroupBarrier();
482
+ }
411
483
  - workgroupBarrier();
412
484
  - let total_sum = sg_partial_sums[0];
413
- -
414
- - // Compute inverse RMS
485
+
486
+ // Compute inverse RMS
415
487
  - let mean_sq = total_sum / f32(size);
416
488
  - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
417
- -
418
- - // Apply normalization
419
- - if (thread_idx < size) {
489
+ + let mean_sq = shared_sum[0];
490
+ + let eps_scaled = f16(u.eps) / (row_scale * row_scale);
491
+ + let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
492
+
493
+ // Apply normalization
494
+ if (thread_idx < size) {
420
495
  var result = x * inv_rms * apply_weight(load_weight(thread_idx));
421
496
  - if (should_add_residual()) {
422
- - result = result + residual[base_offset + thread_idx];
423
497
  + if (u.has_residual == 1u) {
424
- + result = result + f32(residual[base_offset + thread_idx]);
498
+ result = result + residual[base_offset + thread_idx];
425
499
  }
426
500
  write_prenorm(base_offset, thread_idx, x);
427
- - output[base_offset + thread_idx] = result;
428
- + output[base_offset + thread_idx] = f16(result);
429
- }
430
- }