doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,9 +1,8 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/gelu.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // GeLU Activation Kernel with F16 Input/Output
4
4
  //
5
5
  // F16 variant for reduced memory bandwidth when using F16 activations.
6
- // Intermediate computations use F32 for precision.
7
6
 
8
7
  enable f16;
9
8
 
@@ -23,12 +22,12 @@ struct Uniforms {
23
22
  @group(0) @binding(2) var<storage, read_write> output: array<f16>;
24
23
  @group(0) @binding(3) var<storage, read> gate: array<f16>;
25
24
 
26
- fn gelu(x: f32) -> f32 {
27
- let sqrt_2_over_pi: f32 = 0.7978845608;
28
- let c: f32 = 0.044715;
25
+ fn gelu(x: f16) -> f16 {
26
+ let sqrt_2_over_pi = f16(0.7978845608);
27
+ let c = f16(0.044715);
29
28
  let inner = sqrt_2_over_pi * (x + c * x * x * x);
30
- let inner_clamped = clamp(inner, -15.0, 15.0);
31
- return 0.5 * x * (1.0 + tanh(inner_clamped));
29
+ let inner_clamped = clamp(inner, f16(-15.0), f16(15.0));
30
+ return f16(0.5) * x * (f16(1.0) + tanh(inner_clamped));
32
31
  }
33
32
 
34
33
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
@@ -48,19 +47,19 @@ fn main(
48
47
  let token_idx = idx / dim;
49
48
  let dim_idx = idx % dim;
50
49
  let row_base = token_idx * dim * 2u;
51
- let g = f32(input[row_base + dim_idx]);
52
- let up = f32(input[row_base + dim + dim_idx]);
53
- output[idx] = f16(gelu(g) * up);
50
+ let g = input[row_base + dim_idx];
51
+ let up = input[row_base + dim + dim_idx];
52
+ output[idx] = gelu(g) * up;
54
53
  return;
55
54
  }
56
55
 
57
56
  if (HAS_GATE) {
58
- let up = f32(input[idx]);
59
- let g = f32(gate[idx]);
60
- output[idx] = f16(gelu(g) * up);
57
+ let up = input[idx];
58
+ let g = gate[idx];
59
+ output[idx] = gelu(g) * up;
61
60
  return;
62
61
  }
63
62
 
64
- let x = f32(input[idx]);
65
- output[idx] = f16(gelu(x));
63
+ let x = input[idx];
64
+ output[idx] = gelu(x);
66
65
  }
@@ -11,10 +11,6 @@ export interface GroupNormOptions extends OutputBufferOptions {
11
11
  eps: number;
12
12
  }
13
13
 
14
- export interface GroupNormSelectionContext {
15
- isF16: boolean;
16
- }
17
-
18
14
  export declare function runGroupNorm(
19
15
  input: Tensor,
20
16
  weight: GPUBuffer | WeightBuffer,
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/groupnorm_apply.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // GroupNorm Apply Kernel (NCHW, f16)
4
4
 
5
5
  enable f16;
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/groupnorm_stats.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // GroupNorm Stats Kernel (NCHW, f16 input)
4
4
 
5
5
  enable f16;
@@ -49,7 +49,6 @@ export {
49
49
  dequantize,
50
50
  dequantizeRowwise,
51
51
  dequantizeQ6K,
52
- dequantizeQ8_0,
53
52
  dequantizeMXFP4,
54
53
  dequantizeMXFP4Expert,
55
54
  recordDequantize,
@@ -149,6 +148,13 @@ export {
149
148
  type ClampOptions,
150
149
  } from './clamp.js';
151
150
 
151
+ // Static activation quantize/dequantize
152
+ export {
153
+ runActivationStaticQdq,
154
+ recordActivationStaticQdq,
155
+ type ActivationStaticQdqOptions,
156
+ } from './activation-static-qdq.js';
157
+
152
158
  // Energy (EBM helpers)
153
159
  export {
154
160
  runEnergyEval,
@@ -270,10 +276,10 @@ export {
270
276
  } from './sample.js';
271
277
 
272
278
  export {
273
- runSanaLinearAttention,
274
- recordSanaLinearAttention,
275
- type SanaLinearAttentionOptions,
276
- } from './sana_linear_attention.js';
279
+ runLinearAttention,
280
+ recordLinearAttention,
281
+ type LinearAttentionOptions,
282
+ } from './linear_attention.js';
277
283
 
278
284
  export {
279
285
  runRepeatChannels,
@@ -362,6 +368,8 @@ export {
362
368
  recordSoftmaxBackward,
363
369
  runRmsNormBackward,
364
370
  recordRmsNormBackward,
371
+ runLayerNormBackward,
372
+ recordLayerNormBackward,
365
373
  runAttentionBackward,
366
374
  recordAttentionBackward,
367
375
  runRoPEBackward,
@@ -35,7 +35,6 @@ export {
35
35
  dequantize,
36
36
  dequantizeRowwise,
37
37
  dequantizeQ6K,
38
- dequantizeQ8_0,
39
38
  dequantizeMXFP4,
40
39
  dequantizeMXFP4Expert,
41
40
  recordDequantize,
@@ -120,6 +119,12 @@ export {
120
119
  recordClamp,
121
120
  } from './clamp.js';
122
121
 
122
+ // Static activation quantize/dequantize
123
+ export {
124
+ runActivationStaticQdq,
125
+ recordActivationStaticQdq,
126
+ } from './activation-static-qdq.js';
127
+
123
128
  // Energy (EBM helpers)
124
129
  export {
125
130
  runEnergyEval,
@@ -222,9 +227,9 @@ export {
222
227
  } from './sample.js';
223
228
 
224
229
  export {
225
- runSanaLinearAttention,
226
- recordSanaLinearAttention,
227
- } from './sana_linear_attention.js';
230
+ runLinearAttention,
231
+ recordLinearAttention,
232
+ } from './linear_attention.js';
228
233
 
229
234
  export {
230
235
  runRepeatChannels,
@@ -5,13 +5,11 @@
5
5
 
6
6
  enable f16;
7
7
 
8
- const MAX_WORKGROUP_SIZE: u32 = 256u;
9
8
  const MAX_HEAD_DIM: u32 = 256u;
10
9
 
11
10
  override WORKGROUP_SIZE: u32 = 256u;
12
11
  override BIT_WIDTH: u32 = 4u;
13
12
  override PACK_FACTOR: u32 = 8u;
14
- override NUM_CENTROIDS: u32 = 16u;
15
13
  override NUM_BOUNDARIES: u32 = 15u;
16
14
 
17
15
  struct Uniforms {
@@ -7,13 +7,11 @@
7
7
 
8
8
  enable f16;
9
9
 
10
- const MAX_WORKGROUP_SIZE: u32 = 256u;
11
10
  const MAX_HEAD_DIM: u32 = 256u;
12
11
 
13
12
  override WORKGROUP_SIZE: u32 = 256u;
14
13
  override BIT_WIDTH_MSE: u32 = 3u;
15
14
  override PACK_FACTOR_MSE: u32 = 10u;
16
- override NUM_CENTROIDS_MSE: u32 = 8u;
17
15
  override NUM_BOUNDARIES_MSE: u32 = 7u;
18
16
 
19
17
  struct Uniforms {
@@ -13,10 +13,6 @@ export interface LayerNormOptions extends OutputBufferOptions {
13
13
  normWeightDtype?: string;
14
14
  }
15
15
 
16
- export interface LayerNormSelectionContext {
17
- isF16: boolean;
18
- }
19
-
20
16
  export declare function selectLayerNormKernel(options?: LayerNormOptions, isF16?: boolean): string;
21
17
 
22
18
  export declare function runLayerNorm(
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/layernorm.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // LayerNorm Kernel (F16 Input/Output)
4
4
  //
5
5
  // LayerNorm(x) = (x - mean) / sqrt(var + eps) * weight + bias
@@ -2,7 +2,7 @@ import type { Tensor } from '../tensor.js';
2
2
  import type { CommandRecorder } from '../command-recorder.js';
3
3
  import type { OutputBufferOptions } from './types.js';
4
4
 
5
- export interface SanaLinearAttentionOptions extends OutputBufferOptions {
5
+ export interface LinearAttentionOptions extends OutputBufferOptions {
6
6
  numHeads: number;
7
7
  headDim: number;
8
8
  numTokens?: number;
@@ -11,17 +11,17 @@ export interface SanaLinearAttentionOptions extends OutputBufferOptions {
11
11
  summaryBuffer?: GPUBuffer | null;
12
12
  }
13
13
 
14
- export declare function runSanaLinearAttention(
14
+ export declare function runLinearAttention(
15
15
  query: Tensor,
16
16
  key: Tensor,
17
17
  value: Tensor,
18
- options: SanaLinearAttentionOptions
18
+ options: LinearAttentionOptions
19
19
  ): Promise<Tensor>;
20
20
 
21
- export declare function recordSanaLinearAttention(
21
+ export declare function recordLinearAttention(
22
22
  recorder: CommandRecorder,
23
23
  query: Tensor,
24
24
  key: Tensor,
25
25
  value: Tensor,
26
- options: SanaLinearAttentionOptions
26
+ options: LinearAttentionOptions
27
27
  ): Promise<Tensor>;
@@ -5,14 +5,14 @@ import { unifiedKernelWrapper } from './utils.js';
5
5
  import { selectRuleValue } from './rule-registry.js';
6
6
  import { WORKGROUP_SIZES } from './constants.js';
7
7
 
8
- function selectSanaLinearAttentionVariant(isF16) {
9
- return selectRuleValue('sanaLinearAttention', 'variant', { isF16 });
8
+ function selectLinearAttentionVariant(isF16) {
9
+ return selectRuleValue('linearAttention', 'variant', { isF16 });
10
10
  }
11
11
 
12
12
  async function runSummary(target, query, key, value, summaryBuffer, uniforms, variant) {
13
13
  const summarySize = uniforms.num_heads * (uniforms.head_dim + 1) * uniforms.head_dim;
14
14
  await unifiedKernelWrapper(
15
- 'sana_linear_attention_summary',
15
+ 'linear_attention_summary',
16
16
  target,
17
17
  variant,
18
18
  [query, key, value, summaryBuffer],
@@ -30,7 +30,7 @@ async function runSummary(target, query, key, value, summaryBuffer, uniforms, va
30
30
 
31
31
  async function runApply(target, query, summaryBuffer, outputBuffer, uniforms, variant) {
32
32
  await unifiedKernelWrapper(
33
- 'sana_linear_attention_apply',
33
+ 'linear_attention_apply',
34
34
  target,
35
35
  variant,
36
36
  [query, summaryBuffer, outputBuffer],
@@ -48,11 +48,11 @@ async function runApply(target, query, summaryBuffer, outputBuffer, uniforms, va
48
48
  );
49
49
  }
50
50
 
51
- async function _sanaLinearAttention(target, query, key, value, options = {}) {
51
+ async function _linearAttention(target, query, key, value, options = {}) {
52
52
  const recorder = target && typeof target.beginComputePass === 'function' ? target : null;
53
53
  const device = target?.device || getDevice();
54
54
  if (!device) {
55
- throw new Error('SanaLinearAttention requires a WebGPU device.');
55
+ throw new Error('LinearAttention requires a WebGPU device.');
56
56
  }
57
57
 
58
58
  const {
@@ -73,23 +73,23 @@ async function _sanaLinearAttention(target, query, key, value, options = {}) {
73
73
  !Number.isFinite(numTokens) ||
74
74
  !Number.isFinite(hiddenSize)
75
75
  ) {
76
- throw new Error('SanaLinearAttention requires numHeads, headDim, numTokens, and hiddenSize.');
76
+ throw new Error('LinearAttention requires numHeads, headDim, numTokens, and hiddenSize.');
77
77
  }
78
78
  if (hiddenSize !== numHeads * headDim) {
79
- throw new Error(`SanaLinearAttention hiddenSize mismatch: ${hiddenSize} != ${numHeads} * ${headDim}`);
79
+ throw new Error(`LinearAttention hiddenSize mismatch: ${hiddenSize} != ${numHeads} * ${headDim}`);
80
80
  }
81
81
 
82
82
  const isF16 = query.dtype === 'f16';
83
- const variant = selectSanaLinearAttentionVariant(isF16);
83
+ const variant = selectLinearAttentionVariant(isF16);
84
84
  const temporarySummary = summaryBuffer || acquireBuffer(
85
85
  numHeads * (headDim + 1) * headDim * Float32Array.BYTES_PER_ELEMENT,
86
86
  undefined,
87
- 'sana_linear_attention_summary'
87
+ 'linear_attention_summary'
88
88
  );
89
89
  const output = outputBuffer || acquireBuffer(
90
90
  numTokens * hiddenSize * dtypeBytes(query.dtype),
91
91
  undefined,
92
- 'sana_linear_attention_output'
92
+ 'linear_attention_output'
93
93
  );
94
94
 
95
95
  const uniforms = {
@@ -103,7 +103,7 @@ async function _sanaLinearAttention(target, query, key, value, options = {}) {
103
103
  try {
104
104
  await runSummary(target, query, key, value, temporarySummary, uniforms, variant);
105
105
  await runApply(target, query, temporarySummary, output, uniforms, variant);
106
- return createTensor(output, query.dtype, [numTokens, hiddenSize], 'sana_linear_attention_output');
106
+ return createTensor(output, query.dtype, [numTokens, hiddenSize], 'linear_attention_output');
107
107
  } catch (error) {
108
108
  if (ownsOutput) {
109
109
  releaseBuffer(output);
@@ -120,10 +120,10 @@ async function _sanaLinearAttention(target, query, key, value, options = {}) {
120
120
  }
121
121
  }
122
122
 
123
- export async function runSanaLinearAttention(query, key, value, options = {}) {
124
- return _sanaLinearAttention(null, query, key, value, options);
123
+ export async function runLinearAttention(query, key, value, options = {}) {
124
+ return _linearAttention(null, query, key, value, options);
125
125
  }
126
126
 
127
- export async function recordSanaLinearAttention(recorder, query, key, value, options = {}) {
128
- return _sanaLinearAttention(recorder, query, key, value, options);
127
+ export async function recordLinearAttention(recorder, query, key, value, options = {}) {
128
+ return _linearAttention(recorder, query, key, value, options);
129
129
  }
@@ -68,30 +68,6 @@ export declare class LogitMergeKernel {
68
68
  ): Promise<GPUBuffer>;
69
69
  }
70
70
 
71
- /**
72
- * Get the logit merge kernel instance.
73
- */
74
- export declare function getLogitMergeKernel(): LogitMergeKernel;
75
-
76
- /**
77
- * Merge two logit buffers using weighted averaging.
78
- * Convenience function that uses the singleton kernel.
79
- *
80
- * @param logitsA - First logit buffer
81
- * @param logitsB - Second logit buffer
82
- * @param vocabSize - Vocabulary size
83
- * @param weights - Merge weights
84
- * @param temperature - Temperature scaling
85
- * @returns Merged logit buffer
86
- */
87
- export declare function mergeLogits(
88
- logitsA: GPUBuffer,
89
- logitsB: GPUBuffer,
90
- vocabSize: number,
91
- weights: number[],
92
- temperature: number
93
- ): Promise<GPUBuffer>;
94
-
95
71
  /**
96
72
  * Merge multiple logit buffers.
97
73
  * Convenience function that uses the singleton kernel.
@@ -354,7 +354,7 @@ function normalizeWeights(weights, expectedCount) {
354
354
  let _instance = null;
355
355
 
356
356
 
357
- export function getLogitMergeKernel() {
357
+ function getLogitMergeKernel() {
358
358
  if (!_instance) {
359
359
  _instance = new LogitMergeKernel();
360
360
  }
@@ -362,22 +362,6 @@ export function getLogitMergeKernel() {
362
362
  }
363
363
 
364
364
 
365
- export async function mergeLogits(logitsA, logitsB, vocabSize, weights, temperature) {
366
- if (!weights) {
367
- throw new Error('LogitMerge weights are required.');
368
- }
369
- if (temperature == null) {
370
- throw new Error('LogitMerge temperature is required.');
371
- }
372
- const kernel = getLogitMergeKernel();
373
- return kernel.merge(logitsA, logitsB, vocabSize, {
374
- strategy: 'weighted',
375
- weights,
376
- temperature,
377
- });
378
- }
379
-
380
-
381
365
  export async function mergeMultipleLogits(logitBuffers, vocabSize, weights, temperature) {
382
366
  if (!weights) {
383
367
  throw new Error('LogitMerge weights are required.');
@@ -96,7 +96,7 @@ export function calculateMatmulDispatch(variant, useQ4KFused, useGemv, M, N, con
96
96
  }
97
97
 
98
98
 
99
- export function createMatmulUniformBuffer(label, M, N, K, alpha, useQ4KFused, transposeB, uniformWorkgroupsX, recorder, device) {
99
+ export function createMatmulUniformBuffer(label, M, N, K, alpha, useQ4KFused, transposeB, uniformWorkgroupsX, recorder, device, extras = null) {
100
100
  // Shader struct is 32 bytes: M, N, K, alpha, transpose_b/num_blocks, workgroups_x/_pad0, _pad1, _pad2
101
101
  const uniformSize = 32;
102
102
 
@@ -117,6 +117,12 @@ export function createMatmulUniformBuffer(label, M, N, K, alpha, useQ4KFused, tr
117
117
  // workgroups_x (or _pad0 if not needed)
118
118
  view.setUint32(20, uniformWorkgroupsX ?? 0, true);
119
119
  // _pad1, _pad2 - leave as zeros (already zero-initialized)
120
+ // Extras for fused-rmsnorm WideTile variant: eps f32 overwrites slot 20
121
+ // (workgroupsX is unused for those variants). rmsNormOffset is a kernel
122
+ // override constant, not a uniform field.
123
+ if (extras && Number.isFinite(extras.eps)) {
124
+ view.setFloat32(20, extras.eps, true);
125
+ }
120
126
  },
121
127
  recorder,
122
128
  device
@@ -441,9 +441,76 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
441
441
  const fusedAllowed = !fusedQ4KDisabled;
442
442
  const isQ4K = bDtype === 'q4k';
443
443
  const wantF16Output = requestedOutputDtype === 'f16' && capabilities.hasF16;
444
- const q4kVariant = isQ4K && capabilities.hasSubgroups && fusedAllowed
444
+ let q4kVariant = isQ4K && capabilities.hasSubgroups && fusedAllowed
445
445
  ? selectQ4KFusedVariant(M === 1, wantF16Output, aDtype, phase)
446
446
  : null;
447
+ // Opt-in override: when useTiledQ4KPrefill is set on options, replace the
448
+ // default batched f16-output variant with the register-tiled variant
449
+ // whenever the shape (Q4_K weights, f16 output, prefill M>=16) and
450
+ // capability (hasF16) match. Matches both batched_f16 (f32 activations)
451
+ // and batched_f16a (f16 activations); the tiled kernel takes f32 A and
452
+ // f16 C per the q4_fused_batched_f16 binding contract. Decoupled from
453
+ // subgroups because the tiled kernel does not use subgroup reduction.
454
+ if (
455
+ q4kVariant === 'q4_fused_batched_f16'
456
+ && options.useTiledQ4KPrefill === true
457
+ && phase === 'prefill'
458
+ && M >= 16
459
+ && aDtype === 'f32'
460
+ && wantF16Output
461
+ && capabilities.hasF16 === true
462
+ ) {
463
+ q4kVariant = 'q4_fused_prefill_tiled_f16';
464
+ }
465
+ // WideTile override (ported from ORT MatMulNBitsWideTile): register-tiled,
466
+ // 1 thread per output column × TILE_M rows accumulated in registers.
467
+ // Orthogonal to useTiledQ4KPrefill — if both flags are set, WideTile wins
468
+ // because it has materially fewer workgroups at Gemma-4-scale prefill.
469
+ // Routes to the f16-output or f32-output WideTile variant depending on
470
+ // what the caller asked for (Gemma 4's FFN picks q4_fused_batched with
471
+ // f32 output; attention projections may want f16).
472
+ if (
473
+ options.useWideTileQ4KPrefill === true
474
+ && phase === 'prefill'
475
+ && M >= 4
476
+ && aDtype === 'f32'
477
+ && capabilities.hasF16 === true
478
+ ) {
479
+ if (
480
+ wantF16Output
481
+ && (q4kVariant === 'q4_fused_batched_f16' || q4kVariant === 'q4_fused_prefill_tiled_f16')
482
+ ) {
483
+ q4kVariant = 'q4_fused_widetile_f16';
484
+ } else if (
485
+ !wantF16Output
486
+ && q4kVariant === 'q4_fused_batched'
487
+ ) {
488
+ q4kVariant = 'q4_fused_widetile';
489
+ }
490
+ }
491
+ // Residual-fused WideTile override: when the caller passes a residualTensor
492
+ // and opts into the fusion via useWideTileResidualFusion, route the
493
+ // f32-output WideTile variant to its residual-epilogue twin. Saves one
494
+ // downstream doResidualAdd dispatch per fusion point.
495
+ if (
496
+ options.useWideTileResidualFusion === true
497
+ && options.residualTensor != null
498
+ && q4kVariant === 'q4_fused_widetile'
499
+ ) {
500
+ q4kVariant = 'q4_fused_widetile_residual';
501
+ }
502
+ // RMSNorm-fused WideTile override: when the caller passes normWeight and
503
+ // opts into the fusion via useFusedRmsnormWideTile, route the f32-output
504
+ // WideTile variant to its rmsnorm-prologue twin. Each q/k/v or gate/up
505
+ // call runs the norm internally; upstream standalone rmsnorm is skipped
506
+ // by the caller when all downstream matmuls at that site use this variant.
507
+ if (
508
+ options.useFusedRmsnormWideTile === true
509
+ && options.normWeight != null
510
+ && q4kVariant === 'q4_fused_widetile'
511
+ ) {
512
+ q4kVariant = 'q4_fused_rmsnorm_widetile';
513
+ }
447
514
 
448
515
  const effectiveBDtype = bDtype === 'q4k' ? 'f32' : bDtype;
449
516
  const matmulVariant = selectMatmulKernel({
@@ -48,34 +48,6 @@ export interface MatmulOptions extends OutputBufferOptions, OutputDtypeOptions,
48
48
  executionPolicies?: ExecutionV1PoliciesSchema | null;
49
49
  }
50
50
 
51
- /** Context for base matmul kernel selection rules. */
52
- export interface MatmulKernelSelectionContext {
53
- useF16Matmul: boolean;
54
- useF16wF32a: boolean;
55
- useVec4: boolean;
56
- }
57
-
58
- /** Context for fused Q4K variant selection rules. */
59
- export interface MatmulQ4KFusedContext {
60
- useF16A: boolean;
61
- useF16Out: boolean;
62
- isM1: boolean;
63
- }
64
-
65
- /** Context for GEMV variant selection rules. */
66
- export interface GemvSelectionContext {
67
- hasSubgroups: boolean;
68
- useF16Gemv: boolean;
69
- useF32Gemv: boolean;
70
- useMulticol: boolean;
71
- }
72
-
73
- /** Context for top-level matmul variant selection rules. */
74
- export interface MatmulVariantSelectionContext {
75
- canFused: boolean;
76
- useGemv: boolean;
77
- }
78
-
79
51
  /**
80
52
  * Check if fused Q4K kernels are disabled.
81
53
  */