doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -1,5 +1,12 @@
1
1
  --- src/gpu/kernels/attention_streaming.wgsl
2
2
  +++ src/gpu/kernels/attention_streaming_f16.wgsl
3
+ @@ -6,6 +6,8 @@
4
+ // reads K/V directly from storage. Slower but compatible.
5
+
6
+ +enable f16;
7
+ +
8
+ override WORKGROUP_SIZE: u32 = 1u;
9
+ const MAX_HEAD_DIM: u32 = 512u;
3
10
  @@ -30,10 +30,10 @@
4
11
  }
5
12
 
@@ -14,7 +21,6 @@
14
21
  +@group(0) @binding(4) var<storage, read_write> output: array<f16>;
15
22
  @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
16
23
  @group(0) @binding(6) var<storage, read> page_table: array<u32>;
17
-
18
24
  @@ -108,7 +108,7 @@
19
25
  var q_local: array<f32, MAX_HEAD_DIM>;
20
26
  let q_offset = query_pos * num_heads * head_dim + head_idx * head_dim;
@@ -24,3 +30,42 @@
24
30
  }
25
31
 
26
32
  var max_score: f32 = -3.402823e+38;
33
+ @@ -121,7 +121,7 @@
34
+ let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
35
+ var dot: f32 = 0.0;
36
+ for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
37
+ - dot = dot + q_local[d] * K[k_offset + d];
38
+ + dot = dot + q_local[d] * f32(K[k_offset + d]);
39
+ }
40
+ dot = dot * scale;
41
+ // Gemma 2 attention softcapping
42
+ @@ -144,7 +144,7 @@
43
+ let v_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
44
+ var dot: f32 = 0.0;
45
+ for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
46
+ - dot = dot + q_local[d] * K[k_offset + d];
47
+ + dot = dot + q_local[d] * f32(K[k_offset + d]);
48
+ }
49
+ dot = dot * scale;
50
+ // Gemma 2 attention softcapping
51
+ @@ -157,7 +157,7 @@
52
+ let w = exp(dot - max_score);
53
+ sum_exp = sum_exp + w;
54
+ for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
55
+ - acc[d] = acc[d] + w * V[v_offset + d];
56
+ + acc[d] = acc[d] + w * f32(V[v_offset + d]);
57
+ }
58
+ }
59
+ @@ -165,12 +165,12 @@
60
+ if (sum_exp <= 0.0) {
61
+ for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
62
+ - output[out_offset + d] = 0.0;
63
+ + output[out_offset + d] = f16(0.0);
64
+ }
65
+ return;
66
+ }
67
+ for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
68
+ - output[out_offset + d] = acc[d] / sum_exp;
69
+ + output[out_offset + d] = f16(acc[d] / sum_exp);
70
+ }
71
+ }
@@ -39,23 +39,3 @@
39
39
  }
40
40
 
41
41
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
42
- @@ -46,19 +46,19 @@
43
- let token_idx = idx / dim;
44
- let dim_idx = idx % dim;
45
- let row_base = token_idx * dim * 2u;
46
- let g = input[row_base + dim_idx];
47
- let up = input[row_base + dim + dim_idx];
48
- output[idx] = gelu(g) * up;
49
- return;
50
- }
51
-
52
- if (HAS_GATE) {
53
- let up = input[idx];
54
- let g = gate[idx];
55
- output[idx] = gelu(g) * up;
56
- return;
57
- }
58
-
59
- let x = input[idx];
60
- output[idx] = gelu(x);
61
- }
@@ -1,7 +1,9 @@
1
- --- src/gpu/kernels/residual.wgsl
2
- +++ src/gpu/kernels/residual_f16.wgsl
3
- @@ -1,11 +1,13 @@
1
+ --- src/gpu/kernels/residual.wgsl 2026-06-08 16:13:27.295360931 -0400
2
+ +++ src/gpu/kernels/residual_f16.wgsl 2026-06-08 16:32:38.676137468 -0400
3
+ @@ -1,12 +1,16 @@
4
4
  -// residual.wgsl
5
+ +// AUTO-GENERATED from src/gpu/kernels/residual.wgsl.
6
+ +// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
5
7
  +// residual_f16.wgsl
6
8
 
7
9
  /**
@@ -9,14 +11,15 @@
9
11
  + * Residual Add Kernel (F16)
10
12
  *
11
13
  * Performs element-wise addition for residual connections.
12
- * output = a + b
14
+ * output = (a + b) * scale
13
15
  */
14
- +
15
- +enable f16;
16
16
 
17
+ +enable f16;
18
+ +
17
19
  struct Uniforms {
18
20
  size: u32, // Total number of elements
19
- @@ -17,9 +19,9 @@
21
+ scale: f32, // Output scale for main/add_vec4; residual scale for add_scaled
22
+ @@ -17,9 +21,9 @@
20
23
  override WORKGROUP_SIZE: u32 = 256u;
21
24
 
22
25
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -29,13 +32,12 @@
29
32
 
30
33
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
31
34
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
32
- @@ -28,29 +30,5 @@
35
+ @@ -28,29 +32,5 @@
33
36
  if (idx >= u.size) {
34
37
  return;
35
38
  }
36
- - output[idx] = a[idx] + b[idx];
37
- + output[idx] = a[idx] + b[idx];
38
- }
39
+ - output[idx] = (a[idx] + b[idx]) * u.scale;
40
+ -}
39
41
  -
40
42
  -// In-place version: output = output + b
41
43
  -// Note: Caller should copy 'a' to 'output' first, then call this kernel
@@ -59,4 +61,5 @@
59
61
  - return;
60
62
  - }
61
63
  - output[idx] = a[idx] + u.scale * b[idx];
62
- -}
64
+ + output[idx] = f16(f32(a[idx] + b[idx]) * u.scale);
65
+ }
@@ -1,7 +1,11 @@
1
- --- src/gpu/kernels/rmsnorm.wgsl
2
- +++ src/gpu/kernels/rmsnorm_f16.wgsl
3
- @@ -1,46 +1,24 @@
1
+ --- src/gpu/kernels/rmsnorm.wgsl 2026-06-08 16:21:47.561276560 -0400
2
+ +++ src/gpu/kernels/rmsnorm_f16.wgsl 2026-06-08 16:32:38.675963250 -0400
3
+ @@ -1,46 +1,28 @@
4
4
  -// RMSNorm Kernel with Fused Residual Add
5
+ +// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
6
+ +// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
7
+ +// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
8
+ +// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
5
9
  //
6
10
  -// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
7
11
  -//
@@ -27,7 +31,7 @@
27
31
  +// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
28
32
  +
29
33
  +enable f16;
30
-
34
+
31
35
  override WORKGROUP_SIZE: u32 = 256u;
32
36
  const MAX_WORKGROUP_SIZE: u32 = 256u;
33
37
  -
@@ -37,7 +41,7 @@
37
41
  override PRE_RESIDUAL: bool = false; // Add residual BEFORE normalization: rmsnorm(input + residual)
38
42
  override OUTPUT_PRENORM: bool = false; // Write pre-norm sum to residual_sum_output binding
39
43
  override WEIGHT_IS_F16: bool = false; // Weight buffer packed as f16 pairs
40
-
44
+
41
45
  -const MAX_SUBGROUPS: u32 = 32u; // Support up to 32 subgroups per workgroup
42
46
  -const MAX_CACHE_SIZE: u32 = 4608u; // Supports hiddenSize up to 1152 with 4 elements/thread
43
47
  -
@@ -53,11 +57,11 @@
53
57
  + eps: f32, // Epsilon for numerical stability
54
58
  + has_residual: u32, // 1 if residual input provided, 0 otherwise
55
59
  token_stride: u32, // Workgroup rows per dispatch row
56
- _pad0: u32,
60
+ output_scale: f32, // Output epilogue scale
57
61
  _pad1: u32,
58
- @@ -48,47 +26,32 @@
62
+ @@ -48,47 +30,32 @@
59
63
  }
60
-
64
+
61
65
  @group(0) @binding(0) var<uniform> u: Uniforms;
62
66
  -@group(0) @binding(1) var<storage, read> input: array<f32>;
63
67
  -@group(0) @binding(2) var<storage, read> weight: array<u32>; // [size] as f32 or packed f16
@@ -79,7 +83,7 @@
79
83
  +@group(0) @binding(3) var<storage, read_write> output: array<f16>;
80
84
  +@group(0) @binding(4) var<storage, read> residual: array<f16>; // Optional residual
81
85
  +@group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f16>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
82
-
86
+
83
87
  -// Apply weight with optional offset (Gemma uses 1 + weight pattern)
84
88
  -fn apply_weight(w: f32) -> f32 {
85
89
  +var<workgroup> shared_sum: array<f16, MAX_WORKGROUP_SIZE>;
@@ -93,7 +97,7 @@
93
97
  }
94
98
  + return w;
95
99
  }
96
-
100
+
97
101
  -fn load_weight(idx: u32) -> f32 {
98
102
  +fn load_weight(idx: u32) -> f16 {
99
103
  if (WEIGHT_IS_F16) {
@@ -110,26 +114,32 @@
110
114
  - return u.has_residual != 0u;
111
115
  + return f16(bitcast<f32>(weight[idx]));
112
116
  }
113
-
117
+
114
118
  // Load input value, fusing residual add when PRE_RESIDUAL is active
115
119
  -fn load_input(base_offset: u32, idx: u32) -> f32 {
116
120
  +fn load_input(base_offset: u32, idx: u32) -> f16 {
117
121
  let x = input[base_offset + idx];
118
122
  if (PRE_RESIDUAL) {
119
123
  return x + residual[base_offset + idx];
120
- @@ -97,7 +60,7 @@
124
+ @@ -97,24 +64,16 @@
121
125
  }
122
-
126
+
123
127
  // Write pre-norm sum when OUTPUT_PRENORM is active (for downstream residual reuse)
124
128
  -fn write_prenorm(base_offset: u32, idx: u32, val: f32) {
125
129
  +fn write_prenorm(base_offset: u32, idx: u32, val: f16) {
126
130
  if (OUTPUT_PRENORM && PRE_RESIDUAL) {
127
131
  residual_sum_output[base_offset + idx] = val;
128
132
  }
129
- @@ -107,10 +70,6 @@
133
+ }
134
+
135
+ -fn apply_output_scale(value: f32) -> f32 {
136
+ - return value * u.output_scale;
137
+ -}
138
+ -
139
+ fn token_index(wg_id: vec3<u32>) -> u32 {
130
140
  return wg_id.y * max(u.token_stride, 1u) + wg_id.x;
131
141
  }
132
-
142
+
133
143
  -// =============================================================================
134
144
  -// Main Entry Point
135
145
  -// =============================================================================
@@ -137,15 +147,15 @@
137
147
  // Main RMSNorm kernel - one workgroup per token
138
148
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
139
149
  fn main(
140
- @@ -128,151 +87,44 @@
141
-
150
+ @@ -132,151 +91,44 @@
151
+
142
152
  let base_offset = token_idx * size;
143
-
153
+
144
154
  - // Each thread computes partial sum of squares
145
155
  - // When PRE_RESIDUAL is active, sum is computed on (input + residual)
146
156
  - var local_sum_sq: f32 = 0.0;
147
157
  let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
148
-
158
+
149
159
  + var local_max: f16 = f16(0.0);
150
160
  for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
151
161
  let idx = thread_idx * elements_per_thread + i;
@@ -155,12 +165,12 @@
155
165
  + local_max = max(local_max, abs(load_input(base_offset, idx)));
156
166
  }
157
167
  }
158
-
168
+
159
169
  - // Store local sum for reduction
160
170
  - shared_sum[thread_idx] = local_sum_sq;
161
171
  + shared_sum[thread_idx] = local_max;
162
172
  workgroupBarrier();
163
-
173
+
164
174
  - // Parallel reduction to compute total sum of squares
165
175
  for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
166
176
  if (thread_idx < stride) {
@@ -169,7 +179,7 @@
169
179
  }
170
180
  workgroupBarrier();
171
181
  }
172
-
182
+
173
183
  - // Compute RMS
174
184
  - let mean_sq = shared_sum[0] / f32(size);
175
185
  - let rms = sqrt(mean_sq + u.eps);
@@ -177,7 +187,7 @@
177
187
  -
178
188
  + let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
179
189
  workgroupBarrier();
180
-
190
+
181
191
  - // Apply normalization and weight
182
192
  - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
183
193
  - let idx = thread_idx * elements_per_thread + i;
@@ -195,7 +205,7 @@
195
205
  - // Write pre-norm sum for downstream residual reuse
196
206
  - write_prenorm(base_offset, idx, x);
197
207
  -
198
- - output[base_offset + idx] = result;
208
+ - output[base_offset + idx] = apply_output_scale(result);
199
209
  - }
200
210
  - }
201
211
  -}
@@ -249,7 +259,7 @@
249
259
  - result = result + residual[base_offset + thread_idx];
250
260
  - }
251
261
  - write_prenorm(base_offset, thread_idx, x);
252
- - output[base_offset + thread_idx] = result;
262
+ - output[base_offset + thread_idx] = apply_output_scale(result);
253
263
  - }
254
264
  -}
255
265
  -
@@ -291,27 +301,27 @@
291
301
  + local_sum_sq = local_sum_sq + (x * x) * inv_size;
292
302
  }
293
303
  }
294
-
304
+
295
305
  + // Store local sum for reduction
296
306
  shared_sum[thread_idx] = local_sum_sq;
297
307
  workgroupBarrier();
298
-
308
+
299
309
  + // Parallel reduction to compute total sum of squares
300
310
  for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
301
311
  if (thread_idx < stride) {
302
312
  shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
303
- @@ -280,117 +132,38 @@
313
+ @@ -284,117 +136,38 @@
304
314
  workgroupBarrier();
305
315
  }
306
-
316
+
307
317
  - let mean_sq = shared_sum[0] / f32(size);
308
318
  - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
309
319
  + let mean_sq = shared_sum[0];
310
320
  + let eps_scaled = f16(u.eps) / (row_scale * row_scale);
311
321
  + let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
312
-
322
+
313
323
  workgroupBarrier();
314
-
324
+
315
325
  - // Second pass: normalize cached input
316
326
  - for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
317
327
  - let idx = thread_idx * elements_per_thread + i;
@@ -322,7 +332,7 @@
322
332
  - result = result + residual[base_offset + idx];
323
333
  - }
324
334
  - write_prenorm(base_offset, idx, x);
325
- - output[base_offset + idx] = result;
335
+ - output[base_offset + idx] = apply_output_scale(result);
326
336
  - }
327
337
  - }
328
338
  -}
@@ -400,21 +410,22 @@
400
410
  let x = load_input(base_offset, idx);
401
411
  +
402
412
  var result = x * inv_rms * apply_weight(load_weight(idx));
403
-
413
+
404
414
  - if (should_add_residual()) {
405
415
  + // Add residual AFTER normalization
406
416
  + if (u.has_residual == 1u) {
407
417
  result = result + residual[base_offset + idx];
408
418
  }
409
-
419
+
410
420
  + // Write pre-norm sum for downstream residual reuse
411
421
  write_prenorm(base_offset, idx, x);
422
+ - output[base_offset + idx] = apply_output_scale(result);
412
423
  +
413
- output[base_offset + idx] = result;
424
+ + output[base_offset + idx] = f16(f32(result) * u.output_scale);
414
425
  }
415
426
  }
416
427
  }
417
-
428
+
418
429
  -// Subgroup-accelerated RMSNorm for small hidden sizes (size <= 256)
419
430
  -// Each thread handles one element, uses subgroup reduction
420
431
  +// Optimized version for hidden size <= WORKGROUP_SIZE (single pass)
@@ -429,13 +440,13 @@
429
440
  ) {
430
441
  let token_idx = token_index(wg_id);
431
442
  let thread_idx = local_id.x;
432
- @@ -401,45 +174,46 @@
443
+ @@ -405,48 +178,49 @@
433
444
  }
434
-
445
+
435
446
  let base_offset = token_idx * size;
436
447
  - let subgroup_id = thread_idx / sg_size;
437
448
  - let num_subgroups = (WORKGROUP_SIZE + sg_size - 1u) / sg_size;
438
-
449
+
439
450
  - // Each thread handles one element
440
451
  - var x: f32 = 0.0;
441
452
  - var x_sq: f32 = 0.0;
@@ -445,12 +456,12 @@
445
456
  x = load_input(base_offset, thread_idx);
446
457
  - x_sq = x * x;
447
458
  }
448
-
459
+
449
460
  - // Phase 1: Subgroup reduction
450
461
  - let sg_sum = subgroupAdd(x_sq);
451
462
  + shared_sum[thread_idx] = abs(x);
452
463
  + workgroupBarrier();
453
-
464
+
454
465
  - // Phase 2: Store to shared memory
455
466
  - if (sg_lane == 0u && subgroup_id < num_subgroups) {
456
467
  - sg_partial_sums[subgroup_id] = sg_sum;
@@ -466,7 +477,7 @@
466
477
  + let inv_size = f16(1.0) / f16(size);
467
478
  + shared_sum[thread_idx] = (x_scaled * x_scaled) * inv_size;
468
479
  workgroupBarrier();
469
-
480
+
470
481
  - // Phase 3: Final reduction - thread 0 sums, then broadcast via shared memory
471
482
  - if (thread_idx == 0u) {
472
483
  - var sum: f32 = 0.0;
@@ -482,14 +493,14 @@
482
493
  }
483
494
  - workgroupBarrier();
484
495
  - let total_sum = sg_partial_sums[0];
485
-
496
+
486
497
  // Compute inverse RMS
487
498
  - let mean_sq = total_sum / f32(size);
488
499
  - let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
489
500
  + let mean_sq = shared_sum[0];
490
501
  + let eps_scaled = f16(u.eps) / (row_scale * row_scale);
491
502
  + let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
492
-
503
+
493
504
  // Apply normalization
494
505
  if (thread_idx < size) {
495
506
  var result = x * inv_rms * apply_weight(load_weight(thread_idx));
@@ -498,3 +509,7 @@
498
509
  result = result + residual[base_offset + thread_idx];
499
510
  }
500
511
  write_prenorm(base_offset, thread_idx, x);
512
+ - output[base_offset + thread_idx] = apply_output_scale(result);
513
+ + output[base_offset + thread_idx] = f16(f32(result) * u.output_scale);
514
+ }
515
+ }
@@ -0,0 +1,19 @@
1
+ export interface DiffusionGemmaCanvasStatsOptions {
2
+ canvasLength: number;
3
+ vocabSize: number;
4
+ temperature: number;
5
+ padTokenId: number | null;
6
+ logitSoftcap: number;
7
+ argmaxBuffer?: GPUBuffer | null;
8
+ entropyBuffer?: GPUBuffer | null;
9
+ }
10
+
11
+ export interface DiffusionGemmaCanvasStatsResult {
12
+ argmaxBuffer: GPUBuffer;
13
+ entropyBuffer: GPUBuffer;
14
+ }
15
+
16
+ export function runDiffusionGemmaCanvasStats(
17
+ logitsBuffer: GPUBuffer,
18
+ options: DiffusionGemmaCanvasStatsOptions
19
+ ): Promise<DiffusionGemmaCanvasStatsResult>;
@@ -0,0 +1,104 @@
1
+ import { getDevice } from '../device.js';
2
+ import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
3
+ import {
4
+ createPipeline,
5
+ createUniformBufferWithView,
6
+ createBindGroupWithValidation,
7
+ } from './utils.js';
8
+ import { dispatchKernel } from './dispatch.js';
9
+
10
+ const UNIFORM_SIZE = 32;
11
+
12
+ function assertPositiveInteger(value, label) {
13
+ if (!Number.isInteger(value) || value <= 0) {
14
+ throw new Error(`[DiffusionGemmaSampling] ${label} must be a positive integer.`);
15
+ }
16
+ }
17
+
18
+ function assertFinitePositiveNumber(value, label) {
19
+ if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
20
+ throw new Error(`[DiffusionGemmaSampling] ${label} must be a positive finite number.`);
21
+ }
22
+ }
23
+
24
+ function assertFiniteNonNegativeNumber(value, label) {
25
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
26
+ throw new Error(`[DiffusionGemmaSampling] ${label} must be a non-negative finite number.`);
27
+ }
28
+ }
29
+
30
+ function createStatsUniformBuffer(device, options) {
31
+ return createUniformBufferWithView(
32
+ 'diffusion_gemma_sampling_uniforms',
33
+ UNIFORM_SIZE,
34
+ (view) => {
35
+ view.setUint32(0, options.vocabSize, true);
36
+ view.setUint32(4, options.canvasLength, true);
37
+ view.setUint32(8, options.padTokenId == null ? 0xffffffff : options.padTokenId, true);
38
+ view.setUint32(12, 0, true);
39
+ view.setFloat32(16, options.temperature, true);
40
+ view.setFloat32(20, options.logitSoftcap, true);
41
+ view.setFloat32(24, 0, true);
42
+ view.setFloat32(28, 0, true);
43
+ },
44
+ null,
45
+ device
46
+ );
47
+ }
48
+
49
+ export async function runDiffusionGemmaCanvasStats(logitsBuffer, options = {}) {
50
+ const device = getDevice();
51
+ if (!device) {
52
+ throw new Error('[DiffusionGemmaSampling] GPU device is not initialized.');
53
+ }
54
+ if (!logitsBuffer) {
55
+ throw new Error('[DiffusionGemmaSampling] logitsBuffer is required.');
56
+ }
57
+ assertPositiveInteger(options.canvasLength, 'canvasLength');
58
+ assertPositiveInteger(options.vocabSize, 'vocabSize');
59
+ assertFinitePositiveNumber(options.temperature, 'temperature');
60
+ assertFiniteNonNegativeNumber(options.logitSoftcap, 'logitSoftcap');
61
+ if (options.padTokenId != null && (!Number.isInteger(options.padTokenId) || options.padTokenId < 0)) {
62
+ throw new Error('[DiffusionGemmaSampling] padTokenId must be null or a non-negative integer.');
63
+ }
64
+
65
+ const tokenBytes = options.canvasLength * Uint32Array.BYTES_PER_ELEMENT;
66
+ const entropyBytes = options.canvasLength * Float32Array.BYTES_PER_ELEMENT;
67
+ const argmaxBuffer = options.argmaxBuffer ?? acquireBuffer(tokenBytes, undefined, 'diffusion_gemma_argmax_canvas');
68
+ const entropyBuffer = options.entropyBuffer ?? acquireBuffer(entropyBytes, undefined, 'diffusion_gemma_entropy_canvas');
69
+ const ownsArgmax = options.argmaxBuffer == null;
70
+ const ownsEntropy = options.entropyBuffer == null;
71
+ let uniformBuffer = null;
72
+ let completed = false;
73
+
74
+ try {
75
+ if (argmaxBuffer.size < tokenBytes) {
76
+ throw new Error('[DiffusionGemmaSampling] argmaxBuffer is too small.');
77
+ }
78
+ if (entropyBuffer.size < entropyBytes) {
79
+ throw new Error('[DiffusionGemmaSampling] entropyBuffer is too small.');
80
+ }
81
+ const pipeline = await createPipeline('diffusion_gemma_sampling', 'entropy_stats');
82
+ uniformBuffer = createStatsUniformBuffer(device, options);
83
+ const bindGroup = await createBindGroupWithValidation(device, {
84
+ label: 'diffusion_gemma_sampling_bind_group',
85
+ layout: pipeline.getBindGroupLayout(0),
86
+ entries: [
87
+ { binding: 0, resource: { buffer: uniformBuffer } },
88
+ { binding: 1, resource: { buffer: logitsBuffer } },
89
+ { binding: 2, resource: { buffer: argmaxBuffer } },
90
+ { binding: 3, resource: { buffer: entropyBuffer } },
91
+ ],
92
+ }, 'diffusion_gemma_sampling/entropy_stats');
93
+
94
+ dispatchKernel(null, pipeline, bindGroup, options.canvasLength, 'diffusion_gemma_entropy_stats');
95
+ completed = true;
96
+ return { argmaxBuffer, entropyBuffer };
97
+ } finally {
98
+ uniformBuffer?.destroy?.();
99
+ if (!completed) {
100
+ if (ownsArgmax) releaseBuffer(argmaxBuffer);
101
+ if (ownsEntropy) releaseBuffer(entropyBuffer);
102
+ }
103
+ }
104
+ }