doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -6,7 +6,12 @@ import { createTensor } from '../tensor.js';
6
6
  import { KernelBase } from './kernel-base.js';
7
7
  import { TILE_SIZES } from './constants.js';
8
8
  import { getKernelThresholds, padToQ4KBlock } from '../../config/schema/index.js';
9
- import { createUniformBufferWithView, getKernelConfig, hasRequiredFeatures } from './utils.js';
9
+ import {
10
+ createUniformBufferWithView,
11
+ getKernelConfig,
12
+ getPipelineBindGroupLayout,
13
+ hasRequiredFeatures,
14
+ } from './utils.js';
10
15
  import { dispatchIndirect, recordDispatchIndirect } from './dispatch.js';
11
16
  import { releaseUniformBuffer } from '../uniform-cache.js';
12
17
  import { log, trace } from '../../debug/index.js';
@@ -216,7 +221,6 @@ class AttentionBDPAKernel extends KernelBase {
216
221
  }
217
222
  }
218
223
 
219
-
220
224
  function selectAttentionTier(
221
225
  headDim,
222
226
  seqLen,
@@ -472,10 +476,14 @@ function validateAttentionVariant(
472
476
  }
473
477
 
474
478
  if (normalized.startsWith('decode_online')) {
479
+ const exactHeadDim = config.variantMetadata?.exactHeadDim;
475
480
  const maxHeadDim = config.variantMetadata?.maxHeadDim ?? thresholds.subgroupMaxHeadDim;
476
481
  if (!caps.hasSubgroups) {
477
482
  throw new Error(`Attention kernel "${variant}" requires subgroup support.`);
478
483
  }
484
+ if (Number.isFinite(exactHeadDim) && headDim !== exactHeadDim) {
485
+ throw new Error(`Attention kernel "${variant}" requires headDim == ${exactHeadDim} but got ${headDim}.`);
486
+ }
479
487
  if (headDim > maxHeadDim) {
480
488
  throw new Error(`Attention kernel "${variant}" requires headDim <= ${maxHeadDim} but got ${headDim}.`);
481
489
  }
@@ -986,7 +994,7 @@ async function executeAttentionBDPA(
986
994
 
987
995
  const bindGroup = execution.device.createBindGroup({
988
996
  label: 'attention_bdpa_bind_group',
989
- layout: pipeline.getBindGroupLayout(0),
997
+ layout: getPipelineBindGroupLayout(pipeline, 0),
990
998
  entries: [
991
999
  { binding: 0, resource: { buffer: uniformBuffer } },
992
1000
  { binding: 1, resource: { buffer: Q.buffer } },
@@ -1208,7 +1216,7 @@ async function executeAttention(
1208
1216
  ]);
1209
1217
  const bindGroup = execution.device.createBindGroup({
1210
1218
  label: 'attention_bind_group',
1211
- layout: pipeline.getBindGroupLayout(0),
1219
+ layout: getPipelineBindGroupLayout(pipeline, 0),
1212
1220
  entries: [
1213
1221
  { binding: 0, resource: { buffer: uniformBuffer } },
1214
1222
  { binding: 1, resource: { buffer: Q.buffer } },
@@ -1562,9 +1570,9 @@ async function executeOrtFlashAttentionPrefill(recorder, Q, K, V, numHeads, head
1562
1570
  const kvLenBinding = kvLenBuffer || getKvLenFallbackBuffer(device);
1563
1571
  const pageTableBinding = kvPageTable || getPageTableFallbackBuffer(device);
1564
1572
 
1565
- const bindGroup = device.createBindGroup({
1573
+ const bindGroup = execution.device.createBindGroup({
1566
1574
  label: 'attention_ort_flash_prefill_bg',
1567
- layout: pipeline.getBindGroupLayout(0),
1575
+ layout: getPipelineBindGroupLayout(pipeline, 0),
1568
1576
  entries: [
1569
1577
  { binding: 0, resource: { buffer: uniform } },
1570
1578
  { binding: 1, resource: { buffer: Q.buffer } },
@@ -1697,7 +1705,7 @@ async function executeAttentionTiered(
1697
1705
  ]);
1698
1706
  const bindGroup = execution.device.createBindGroup({
1699
1707
  label: 'attention_tiered_bind_group',
1700
- layout: pipeline.getBindGroupLayout(0),
1708
+ layout: getPipelineBindGroupLayout(pipeline, 0),
1701
1709
  entries: [
1702
1710
  { binding: 0, resource: { buffer: uniformBuffer } },
1703
1711
  { binding: 1, resource: { buffer: Q.buffer } },
@@ -1898,7 +1906,7 @@ async function executeAttentionTieredQuant(
1898
1906
 
1899
1907
  const bindGroup = execution.device.createBindGroup({
1900
1908
  label: 'attention_tiered_quant_bind_group',
1901
- layout: pipeline.getBindGroupLayout(0),
1909
+ layout: getPipelineBindGroupLayout(pipeline, 0),
1902
1910
  entries,
1903
1911
  });
1904
1912
 
@@ -2174,7 +2182,7 @@ async function executeAttentionContiguousQuant(
2174
2182
 
2175
2183
  const bindGroup = execution.device.createBindGroup({
2176
2184
  label: 'attention_contiguous_quant_bind_group',
2177
- layout: pipeline.getBindGroupLayout(0),
2185
+ layout: getPipelineBindGroupLayout(pipeline, 0),
2178
2186
  entries,
2179
2187
  });
2180
2188
 
@@ -0,0 +1,236 @@
1
+ // Online Decode Attention Kernel (fixed head_dim 256, f32 Q + f16 KV + f32 output)
2
+ //
3
+ // Same bindings and online softmax as attention_decode_online_f16kv.wgsl, but
4
+ // specialized for medium Gemma/Qwen-style 256-dim heads.
5
+
6
+ enable f16;
7
+ enable subgroups;
8
+
9
+ const WORKGROUP_SIZE: u32 = 256u;
10
+ const MAX_SUBGROUPS: u32 = 256u;
11
+ const HEAD_DIM: u32 = 256u;
12
+ const HEAD_DIM_VECS: u32 = 64u;
13
+
14
+ struct Uniforms {
15
+ num_heads: u32,
16
+ num_kv_heads: u32,
17
+ head_dim: u32,
18
+ kv_len: u32,
19
+ seq_len: u32,
20
+ scale: f32,
21
+ is_causal: u32,
22
+ start_pos: u32,
23
+ attn_softcap: f32,
24
+ sliding_window: u32,
25
+ kv_len_source: u32,
26
+ kv_start: u32,
27
+ page_size: u32,
28
+ kv_layout: u32,
29
+ _pad: u32,
30
+ }
31
+
32
+ @group(0) @binding(0) var<uniform> u: Uniforms;
33
+ @group(0) @binding(1) var<storage, read> Q: array<f32>;
34
+ @group(0) @binding(2) var<storage, read> K: array<f16>;
35
+ @group(0) @binding(3) var<storage, read> V: array<f16>;
36
+ @group(0) @binding(4) var<storage, read_write> output: array<f32>;
37
+ @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
38
+ @group(0) @binding(6) var<storage, read> page_table: array<u32>;
39
+
40
+ var<workgroup> shared_q: array<vec4<f32>, HEAD_DIM_VECS>;
41
+ var<workgroup> shared_scores: array<f32, WORKGROUP_SIZE>;
42
+ var<workgroup> shared_v_bases: array<u32, WORKGROUP_SIZE>;
43
+ var<workgroup> sg_max: array<f32, MAX_SUBGROUPS>;
44
+ var<workgroup> sg_sum: array<f32, MAX_SUBGROUPS>;
45
+ var<workgroup> global_max: f32;
46
+ var<workgroup> global_sum: f32;
47
+
48
+ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
49
+ let heads_per_kv = u.num_heads / u.num_kv_heads;
50
+ return query_head_idx / heads_per_kv;
51
+ }
52
+
53
+ fn get_kv_len() -> u32 {
54
+ if (u.kv_len_source == 0u) {
55
+ return u.kv_len;
56
+ }
57
+ return kv_len_buffer[0];
58
+ }
59
+
60
+ fn is_masked(abs_key: u32) -> bool {
61
+ let abs_query = u.start_pos;
62
+ if (u.is_causal != 0u && abs_key > abs_query) { return true; }
63
+ if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
64
+ if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
65
+ }
66
+ return false;
67
+ }
68
+
69
+ fn get_kv_pos(key_pos: u32) -> u32 {
70
+ let abs_key = u.kv_start + key_pos;
71
+ if (u.kv_layout == 1u && u.sliding_window > 0u) {
72
+ return abs_key % u.sliding_window;
73
+ }
74
+ if (u.kv_layout == 2u) {
75
+ let page_idx = abs_key / u.page_size;
76
+ let in_page = abs_key - (page_idx * u.page_size);
77
+ let phys_page = page_table[page_idx];
78
+ return phys_page * u.page_size + in_page;
79
+ }
80
+ return abs_key;
81
+ }
82
+
83
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
84
+ fn main(
85
+ @builtin(local_invocation_id) local_id: vec3<u32>,
86
+ @builtin(workgroup_id) workgroup_id: vec3<u32>,
87
+ @builtin(subgroup_size) subgroup_size: u32,
88
+ @builtin(subgroup_invocation_id) sg_tid: u32,
89
+ ) {
90
+ if (u.head_dim != HEAD_DIM) {
91
+ return;
92
+ }
93
+
94
+ let head_idx = workgroup_id.x;
95
+ let tid = local_id.x;
96
+ let kv_len = get_kv_len();
97
+ let q_offset = head_idx * HEAD_DIM;
98
+
99
+ if (kv_len == 0u) {
100
+ output[q_offset + tid] = 0.0;
101
+ return;
102
+ }
103
+
104
+ let subgroup_id = tid / subgroup_size;
105
+ let num_subgroups = (WORKGROUP_SIZE + subgroup_size - 1u) / subgroup_size;
106
+ if (num_subgroups > MAX_SUBGROUPS) {
107
+ return;
108
+ }
109
+
110
+ let kv_head_idx = get_kv_head_idx(head_idx);
111
+
112
+ if (tid < HEAD_DIM_VECS) {
113
+ let d = tid * 4u;
114
+ shared_q[tid] = vec4<f32>(
115
+ Q[q_offset + d],
116
+ Q[q_offset + d + 1u],
117
+ Q[q_offset + d + 2u],
118
+ Q[q_offset + d + 3u]
119
+ );
120
+ }
121
+ workgroupBarrier();
122
+
123
+ var running_max: f32 = -3.402823e+38;
124
+ var running_sum: f32 = 0.0;
125
+ var out_accum: f32 = 0.0;
126
+
127
+ var start_k: u32 = 0u;
128
+ if (u.sliding_window > 0u && kv_len > u.sliding_window) {
129
+ start_k = kv_len - u.sliding_window;
130
+ start_k = (start_k / WORKGROUP_SIZE) * WORKGROUP_SIZE;
131
+ }
132
+
133
+ for (var k_start: u32 = start_k; k_start < kv_len; k_start = k_start + WORKGROUP_SIZE) {
134
+ let k_pos = k_start + tid;
135
+ let valid_k = k_pos < kv_len;
136
+ var masked = false;
137
+ var score: f32 = -3.402823e+38;
138
+
139
+ if (valid_k) {
140
+ let abs_key = u.kv_start + k_pos;
141
+ masked = is_masked(abs_key);
142
+ let k_idx = get_kv_pos(k_pos);
143
+ let k_offset = k_idx * u.num_kv_heads * HEAD_DIM + kv_head_idx * HEAD_DIM;
144
+ shared_v_bases[tid] = k_offset;
145
+ if (!masked) {
146
+ var dot_accum: f32 = 0.0;
147
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 2u) {
148
+ let d0 = d4 * 4u;
149
+ let d1 = d0 + 4u;
150
+ let k0 = vec4<f32>(
151
+ f32(K[k_offset + d0]),
152
+ f32(K[k_offset + d0 + 1u]),
153
+ f32(K[k_offset + d0 + 2u]),
154
+ f32(K[k_offset + d0 + 3u])
155
+ );
156
+ let k1 = vec4<f32>(
157
+ f32(K[k_offset + d1]),
158
+ f32(K[k_offset + d1 + 1u]),
159
+ f32(K[k_offset + d1 + 2u]),
160
+ f32(K[k_offset + d1 + 3u])
161
+ );
162
+ dot_accum = dot_accum + dot(shared_q[d4], k0) + dot(shared_q[d4 + 1u], k1);
163
+ }
164
+ score = dot_accum * u.scale;
165
+ if (u.attn_softcap > 0.0) {
166
+ score = tanh(score / u.attn_softcap) * u.attn_softcap;
167
+ }
168
+ }
169
+ }
170
+
171
+ let chunk_max = subgroupMax(score);
172
+ if (sg_tid == 0u && subgroup_id < num_subgroups) {
173
+ sg_max[subgroup_id] = chunk_max;
174
+ }
175
+ workgroupBarrier();
176
+
177
+ if (tid == 0u) {
178
+ var m: f32 = -3.402823e+38;
179
+ for (var s: u32 = 0u; s < num_subgroups; s++) {
180
+ m = max(m, sg_max[s]);
181
+ }
182
+ global_max = m;
183
+ }
184
+ workgroupBarrier();
185
+
186
+ let chunk_max_val = global_max;
187
+ let new_max = max(running_max, chunk_max_val);
188
+ let rescale = exp(running_max - new_max);
189
+
190
+ var exp_score: f32 = 0.0;
191
+ if (valid_k && !masked) {
192
+ exp_score = exp(score - new_max);
193
+ }
194
+ shared_scores[tid] = exp_score;
195
+
196
+ let chunk_sum = subgroupAdd(exp_score);
197
+ if (sg_tid == 0u && subgroup_id < num_subgroups) {
198
+ sg_sum[subgroup_id] = chunk_sum;
199
+ }
200
+ workgroupBarrier();
201
+
202
+ if (tid == 0u) {
203
+ var s: f32 = 0.0;
204
+ for (var i: u32 = 0u; i < num_subgroups; i++) {
205
+ s = s + sg_sum[i];
206
+ }
207
+ global_sum = s;
208
+ }
209
+ workgroupBarrier();
210
+
211
+ running_sum = running_sum * rescale + global_sum;
212
+ running_max = new_max;
213
+
214
+ out_accum = out_accum * rescale;
215
+ let chunk_len = min(WORKGROUP_SIZE, kv_len - k_start);
216
+ let chunk_len4 = (chunk_len / 4u) * 4u;
217
+ for (var k: u32 = 0u; k < chunk_len4; k = k + 4u) {
218
+ let v_base0 = shared_v_bases[k];
219
+ let v_base1 = shared_v_bases[k + 1u];
220
+ let v_base2 = shared_v_bases[k + 2u];
221
+ let v_base3 = shared_v_bases[k + 3u];
222
+ out_accum = out_accum + shared_scores[k] * f32(V[v_base0 + tid]);
223
+ out_accum = out_accum + shared_scores[k + 1u] * f32(V[v_base1 + tid]);
224
+ out_accum = out_accum + shared_scores[k + 2u] * f32(V[v_base2 + tid]);
225
+ out_accum = out_accum + shared_scores[k + 3u] * f32(V[v_base3 + tid]);
226
+ }
227
+ for (var k: u32 = chunk_len4; k < chunk_len; k = k + 1u) {
228
+ let v_base = shared_v_bases[k];
229
+ out_accum = out_accum + shared_scores[k] * f32(V[v_base + tid]);
230
+ }
231
+ workgroupBarrier();
232
+ }
233
+
234
+ let inv_sum = select(0.0, 1.0 / running_sum, running_sum > 0.0);
235
+ output[q_offset + tid] = out_accum * inv_sum;
236
+ }
@@ -6,6 +6,8 @@
6
6
  // models with head_dim beyond tiled support. Uses two-pass softmax and
7
7
  // reads K/V directly from storage. Slower but compatible.
8
8
 
9
+ enable f16;
10
+
9
11
  override WORKGROUP_SIZE: u32 = 1u;
10
12
  const MAX_HEAD_DIM: u32 = 512u;
11
13
 
@@ -120,7 +122,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>) {
120
122
  let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
121
123
  var dot: f32 = 0.0;
122
124
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
123
- dot = dot + q_local[d] * K[k_offset + d];
125
+ dot = dot + q_local[d] * f32(K[k_offset + d]);
124
126
  }
125
127
  dot = dot * scale;
126
128
  // Gemma 2 attention softcapping
@@ -143,7 +145,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>) {
143
145
  let v_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
144
146
  var dot: f32 = 0.0;
145
147
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
146
- dot = dot + q_local[d] * K[k_offset + d];
148
+ dot = dot + q_local[d] * f32(K[k_offset + d]);
147
149
  }
148
150
  dot = dot * scale;
149
151
  // Gemma 2 attention softcapping
@@ -153,18 +155,18 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>) {
153
155
  let w = exp(dot - max_score);
154
156
  sum_exp = sum_exp + w;
155
157
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
156
- acc[d] = acc[d] + w * V[v_offset + d];
158
+ acc[d] = acc[d] + w * f32(V[v_offset + d]);
157
159
  }
158
160
  }
159
161
 
160
162
  let out_offset = query_pos * num_heads * head_dim + head_idx * head_dim;
161
163
  if (sum_exp <= 0.0) {
162
164
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
163
- output[out_offset + d] = 0.0;
165
+ output[out_offset + d] = f16(0.0);
164
166
  }
165
167
  return;
166
168
  }
167
169
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
168
- output[out_offset + d] = acc[d] / sum_exp;
170
+ output[out_offset + d] = f16(acc[d] / sum_exp);
169
171
  }
170
172
  }
@@ -1,29 +1,98 @@
1
1
  import { dtypeBytes } from '../../tensor.js';
2
+ import { getDevice } from '../../device.js';
2
3
  import { WORKGROUP_SIZES } from '../constants.js';
3
4
  import { createBackwardKernel } from './utils.js';
4
5
 
6
+ function resolveMaxWorkgroupsPerDimension(device) {
7
+ const limit = device?.limits?.maxComputeWorkgroupsPerDimension;
8
+ return Number.isFinite(limit) && limit > 0 ? limit : 65535;
9
+ }
10
+
11
+ function planDispatch(numTokens, vocabSize, maxWorkgroupsPerDimension) {
12
+ const total = numTokens * vocabSize;
13
+ const dispatchStride = Math.min(
14
+ total,
15
+ maxWorkgroupsPerDimension * WORKGROUP_SIZES.DEFAULT
16
+ );
17
+ return {
18
+ dispatchStride,
19
+ workgroups: [
20
+ Math.ceil(dispatchStride / WORKGROUP_SIZES.DEFAULT),
21
+ Math.ceil(total / dispatchStride),
22
+ 1,
23
+ ],
24
+ };
25
+ }
26
+
5
27
  const { run, record } = createBackwardKernel('cross_entropy_backward', {
6
28
  uniformSize: 16,
7
29
  writeUniforms: (view, opts) => {
8
30
  view.setUint32(0, opts.numTokens, true);
9
31
  view.setUint32(4, opts.vocabSize, true);
32
+ view.setUint32(8, opts._dispatchStride, true);
10
33
  },
11
- calcWorkgroups: (opts) => Math.ceil((opts.numTokens * opts.vocabSize) / WORKGROUP_SIZES.DEFAULT),
34
+ calcWorkgroups: (opts) => planDispatch(
35
+ opts.numTokens,
36
+ opts.vocabSize,
37
+ opts._maxComputeWorkgroupsPerDimension
38
+ ).workgroups,
12
39
  outputBytes: (opts) => opts.numTokens * opts.vocabSize * opts._bytesPerElement,
13
40
  outputShape: (opts) => [opts.numTokens, opts.vocabSize],
14
41
  dtype: (opts, inputs) => inputs[0].dtype,
15
42
  getDevice: true,
16
43
  validate: (opts) => {
17
44
  if (!opts.numTokens || !opts.vocabSize) throw new Error('cross entropy backward requires numTokens and vocabSize');
45
+ const dispatchPlan = planDispatch(
46
+ opts.numTokens,
47
+ opts.vocabSize,
48
+ opts._maxComputeWorkgroupsPerDimension
49
+ );
50
+ if (dispatchPlan.workgroups[1] > opts._maxComputeWorkgroupsPerDimension) {
51
+ throw new Error(
52
+ `cross entropy backward dispatch requires ${dispatchPlan.workgroups[1]} y workgroups ` +
53
+ `but device limit is ${opts._maxComputeWorkgroupsPerDimension}`
54
+ );
55
+ }
18
56
  },
19
57
  });
20
58
 
59
+ function withDispatchPlan(options, device) {
60
+ const maxComputeWorkgroupsPerDimension = resolveMaxWorkgroupsPerDimension(device);
61
+ const dispatchPlan = planDispatch(
62
+ options.numTokens,
63
+ options.vocabSize,
64
+ maxComputeWorkgroupsPerDimension
65
+ );
66
+ return {
67
+ ...options,
68
+ _maxComputeWorkgroupsPerDimension: maxComputeWorkgroupsPerDimension,
69
+ _dispatchStride: dispatchPlan.dispatchStride,
70
+ };
71
+ }
72
+
21
73
  export async function runCrossEntropyBackward(softmax, targets, gradOutput, options = {}) {
22
74
  const bytesPerElement = dtypeBytes(softmax.dtype);
23
- return run(softmax, targets, gradOutput, { ...options, _bytesPerElement: bytesPerElement });
75
+ return run(
76
+ softmax,
77
+ targets,
78
+ gradOutput,
79
+ {
80
+ ...withDispatchPlan(options, getDevice()),
81
+ _bytesPerElement: bytesPerElement,
82
+ }
83
+ );
24
84
  }
25
85
 
26
86
  export async function recordCrossEntropyBackward(recorder, softmax, targets, gradOutput, options = {}) {
27
87
  const bytesPerElement = dtypeBytes(softmax.dtype);
28
- return record(recorder, softmax, targets, gradOutput, { ...options, _bytesPerElement: bytesPerElement });
88
+ return record(
89
+ recorder,
90
+ softmax,
91
+ targets,
92
+ gradOutput,
93
+ {
94
+ ...withDispatchPlan(options, recorder.device),
95
+ _bytesPerElement: bytesPerElement,
96
+ }
97
+ );
29
98
  }
@@ -9,7 +9,7 @@ override WORKGROUP_SIZE: u32 = 256u;
9
9
  struct Uniforms {
10
10
  num_tokens: u32,
11
11
  vocab_size: u32,
12
- _pad0: u32,
12
+ dispatch_stride: u32,
13
13
  _pad1: u32,
14
14
  }
15
15
 
@@ -21,7 +21,7 @@ struct Uniforms {
21
21
 
22
22
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
23
23
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
24
- let idx = gid.x;
24
+ let idx = gid.y * max(u.dispatch_stride, 1u) + gid.x;
25
25
  let total = u.num_tokens * u.vocab_size;
26
26
  if (idx >= total) {
27
27
  return;
@@ -148,45 +148,49 @@ export async function runBF16ToF32(
148
148
  trace.kernels('BF16ToF32: Pipeline created');
149
149
 
150
150
  const output = acquireBuffer(outputSize, undefined, name);
151
- trace.kernels(`BF16ToF32: Output buffer acquired, size=${output.size}`);
152
-
153
- const uniformBuffer = createUniformBufferWithView(
154
- 'bf16_to_f32_uniforms',
155
- 16,
156
- (view) => {
157
- view.setUint32(0, numElements, true);
158
- },
159
- null,
160
- device
161
- );
162
- trace.kernels(`BF16ToF32: Uniform numElements=${numElements}`);
163
-
164
- const bindGroup = device.createBindGroup({
165
- label: 'bf16_to_f32_bind_group',
166
- layout: pipeline.getBindGroupLayout(0),
167
- entries: [
168
- { binding: 0, resource: { buffer: uniformBuffer } },
169
- { binding: 1, resource: { buffer: input } },
170
- { binding: 2, resource: { buffer: output } },
171
- ],
172
- });
173
- trace.kernels('BF16ToF32: BindGroup created');
174
-
175
- // Each thread processes 2 BF16 values (1 u32), so divide by 2 for thread count
176
- // Then divide by 256 for workgroup count
177
- const numPairs = Math.ceil(numElements / 2);
178
- const workgroups = Math.ceil(numPairs / WORKGROUP_SIZES.DEFAULT);
179
- const dispatchSize = calculate2DDispatch(workgroups);
180
-
181
- trace.kernels(`BF16ToF32: Dispatching ${dispatchSize[0]}x${dispatchSize[1]} workgroups for ${numPairs} pairs (${numElements} elements)`);
151
+ let uniformBuffer = null;
152
+ let completed = false;
182
153
  try {
154
+ trace.kernels(`BF16ToF32: Output buffer acquired, size=${output.size}`);
155
+
156
+ uniformBuffer = createUniformBufferWithView(
157
+ 'bf16_to_f32_uniforms',
158
+ 16,
159
+ (view) => {
160
+ view.setUint32(0, numElements, true);
161
+ },
162
+ null,
163
+ device
164
+ );
165
+ trace.kernels(`BF16ToF32: Uniform numElements=${numElements}`);
166
+
167
+ const bindGroup = device.createBindGroup({
168
+ label: 'bf16_to_f32_bind_group',
169
+ layout: pipeline.getBindGroupLayout(0),
170
+ entries: [
171
+ { binding: 0, resource: { buffer: uniformBuffer } },
172
+ { binding: 1, resource: { buffer: input } },
173
+ { binding: 2, resource: { buffer: output } },
174
+ ],
175
+ });
176
+ trace.kernels('BF16ToF32: BindGroup created');
177
+
178
+ // Each thread processes 2 BF16 values (1 u32), so divide by 2 for thread count
179
+ // Then divide by 256 for workgroup count
180
+ const numPairs = Math.ceil(numElements / 2);
181
+ const workgroups = Math.ceil(numPairs / WORKGROUP_SIZES.DEFAULT);
182
+ const dispatchSize = calculate2DDispatch(workgroups);
183
+
184
+ trace.kernels(`BF16ToF32: Dispatching ${dispatchSize[0]}x${dispatchSize[1]} workgroups for ${numPairs} pairs (${numElements} elements)`);
183
185
  dispatch(device, pipeline, bindGroup, dispatchSize, 'bf16_to_f32');
184
- return createTensor(output, 'f32', [...shape], name);
185
- } catch (error) {
186
- releaseBuffer(output);
187
- throw error;
186
+ const tensor = createTensor(output, 'f32', [...shape], name);
187
+ completed = true;
188
+ return tensor;
188
189
  } finally {
189
- uniformBuffer.destroy();
190
+ uniformBuffer?.destroy();
191
+ if (!completed) {
192
+ releaseBuffer(output);
193
+ }
190
194
  }
191
195
  }
192
196
 
@@ -217,41 +221,44 @@ export async function runBF16ToF16(
217
221
  }
218
222
 
219
223
  const output = acquireBuffer(outputSize, undefined, name);
224
+ let uniformBuffer = null;
225
+ let completed = false;
226
+ try {
227
+ uniformBuffer = createUniformBufferWithView(
228
+ 'bf16_to_f16_uniforms',
229
+ 16,
230
+ (view) => {
231
+ view.setUint32(0, numElements, true);
232
+ view.setUint32(4, 0, true);
233
+ view.setUint32(8, 0, true);
234
+ },
235
+ null,
236
+ device
237
+ );
238
+
239
+ const bindGroup = device.createBindGroup({
240
+ label: 'bf16_to_f16_bind_group',
241
+ layout: pipeline.getBindGroupLayout(0),
242
+ entries: [
243
+ { binding: 0, resource: { buffer: uniformBuffer } },
244
+ { binding: 1, resource: { buffer: input } },
245
+ { binding: 2, resource: { buffer: output } },
246
+ ],
247
+ });
220
248
 
221
- const uniformBuffer = createUniformBufferWithView(
222
- 'bf16_to_f16_uniforms',
223
- 16,
224
- (view) => {
225
- view.setUint32(0, numElements, true);
226
- view.setUint32(4, 0, true);
227
- view.setUint32(8, 0, true);
228
- },
229
- null,
230
- device
231
- );
232
-
233
- const bindGroup = device.createBindGroup({
234
- label: 'bf16_to_f16_bind_group',
235
- layout: pipeline.getBindGroupLayout(0),
236
- entries: [
237
- { binding: 0, resource: { buffer: uniformBuffer } },
238
- { binding: 1, resource: { buffer: input } },
239
- { binding: 2, resource: { buffer: output } },
240
- ],
241
- });
242
-
243
- const numPairs = Math.ceil(numElements / 2);
244
- const workgroups = Math.ceil(numPairs / WORKGROUP_SIZES.DEFAULT);
245
- const dispatchSize = calculate2DDispatch(workgroups);
249
+ const numPairs = Math.ceil(numElements / 2);
250
+ const workgroups = Math.ceil(numPairs / WORKGROUP_SIZES.DEFAULT);
251
+ const dispatchSize = calculate2DDispatch(workgroups);
246
252
 
247
- try {
248
253
  dispatch(device, pipeline, bindGroup, dispatchSize, 'bf16_to_f16');
249
- return createTensor(output, 'f16', [...shape], name);
250
- } catch (error) {
251
- releaseBuffer(output);
252
- throw error;
254
+ const tensor = createTensor(output, 'f16', [...shape], name);
255
+ completed = true;
256
+ return tensor;
253
257
  } finally {
254
- uniformBuffer.destroy();
258
+ uniformBuffer?.destroy();
259
+ if (!completed) {
260
+ releaseBuffer(output);
261
+ }
255
262
  }
256
263
  }
257
264