doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -1,4 +1,4 @@
1
- import { getKernelCapabilities } from '../device.js';
1
+ import { getDevice, getKernelCapabilities } from '../device.js';
2
2
  import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
3
3
  import { WORKGROUP_SIZES, VEC4_ELEMENTS_PER_WG } from './constants.js';
4
4
  import { unifiedKernelWrapper } from './utils.js';
@@ -8,14 +8,233 @@ import { DTYPE_SIZES, padToQ4KBlock } from '../../config/schema/index.js';
8
8
  import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
9
9
  import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
10
10
 
11
- function selectGatherVariant(useF16Input, useF16Output, useVec4) {
11
+ const SPLIT_GATHER_STORAGE_BUFFER_OVERHEAD = 2;
12
+ const SPLIT4_GATHER_SECTION_COUNT = 4;
13
+ const SPLIT8_GATHER_SECTION_COUNT = 8;
14
+
15
+ function selectGatherVariant(useF16Input, useF16Output, useVec4, useLiteRTInt4Input = false) {
12
16
  return selectKernelRuleValue(
13
17
  'gather',
14
18
  'variant',
15
- { useF16Input, useF16Output, useVec4 }
19
+ { useF16Input, useF16Output, useVec4, useLiteRTInt4Input }
16
20
  );
17
21
  }
18
22
 
23
+ function resolveLiteRTInt4StorageEncoding(storageEncoding) {
24
+ const normalized = String(storageEncoding ?? '').toLowerCase();
25
+ if (normalized !== 'signed' && normalized !== 'offset_binary') {
26
+ throw new Error(
27
+ `[Gather] LiteRT INT4 embeddings require storageEncoding "signed" or "offset_binary", ` +
28
+ `got "${normalized || 'missing'}".`
29
+ );
30
+ }
31
+ return normalized;
32
+ }
33
+
34
+ function normalizeSplitGatherSections(splitEmbedding, sectionCount, label) {
35
+ const sections = splitEmbedding?.sections;
36
+ if (!Array.isArray(sections) || sections.length === 0) {
37
+ throw new Error('[Gather] split embeddings require at least one GPU section.');
38
+ }
39
+ if (sections.length > sectionCount) {
40
+ throw new Error(
41
+ `[Gather] split embeddings have ${sections.length} sections; ` +
42
+ `${label} supports at most ${sectionCount}.`
43
+ );
44
+ }
45
+ let nextRowStart = 0;
46
+ for (let i = 0; i < sections.length; i++) {
47
+ const section = sections[i];
48
+ if (!section?.buffer) {
49
+ throw new Error(`[Gather] split section ${i} is missing a GPU buffer.`);
50
+ }
51
+ if (section.rowStart !== nextRowStart) {
52
+ throw new Error(
53
+ `[Gather] split section ${i} rowStart=${section.rowStart} is not contiguous from row ${nextRowStart}.`
54
+ );
55
+ }
56
+ if (!Number.isInteger(section.rowCount) || section.rowCount <= 0) {
57
+ throw new Error(`[Gather] split section ${i} has invalid rowCount.`);
58
+ }
59
+ nextRowStart += section.rowCount;
60
+ }
61
+ const padded = [...sections];
62
+ while (padded.length < sectionCount) {
63
+ padded.push({ buffer: sections[0].buffer, rowStart: nextRowStart, rowCount: 0 });
64
+ }
65
+ return padded;
66
+ }
67
+
68
+ function selectSplitGatherConfig(splitEmbedding, outputDtype, forcedSectionCount = null) {
69
+ const sectionCount = Array.isArray(splitEmbedding?.sections) ? splitEmbedding.sections.length : 0;
70
+ if (sectionCount <= 0) {
71
+ throw new Error('[Gather] split embeddings require at least one GPU section.');
72
+ }
73
+ if (
74
+ forcedSectionCount != null
75
+ && forcedSectionCount !== SPLIT4_GATHER_SECTION_COUNT
76
+ && forcedSectionCount !== SPLIT8_GATHER_SECTION_COUNT
77
+ ) {
78
+ throw new Error(`[Gather] unsupported split gather section count ${forcedSectionCount}.`);
79
+ }
80
+ if (sectionCount > SPLIT8_GATHER_SECTION_COUNT) {
81
+ throw new Error(
82
+ `[Gather] split embeddings have ${sectionCount} sections; ` +
83
+ `gather_split8 supports at most ${SPLIT8_GATHER_SECTION_COUNT}.`
84
+ );
85
+ }
86
+ const config = selectKernelRuleValue('gather', 'splitConfig', {
87
+ forcedSectionCount,
88
+ outputDtype,
89
+ sectionCount,
90
+ });
91
+ if (sectionCount > config.sectionCount) {
92
+ throw new Error(
93
+ `[Gather] split embeddings have ${sectionCount} sections; ` +
94
+ `${config.label} supports at most ${config.sectionCount}.`
95
+ );
96
+ }
97
+ return config;
98
+ }
99
+
100
+ function assertSplitGatherBindingLimit(config) {
101
+ const maxStorageBuffersPerShaderStage = getDevice()?.limits?.maxStorageBuffersPerShaderStage;
102
+ const requiredStorageBuffers = config.sectionCount + SPLIT_GATHER_STORAGE_BUFFER_OVERHEAD;
103
+ if (
104
+ Number.isFinite(maxStorageBuffersPerShaderStage)
105
+ && maxStorageBuffersPerShaderStage < requiredStorageBuffers
106
+ ) {
107
+ throw new Error(
108
+ `[Gather] ${config.label} requires ${requiredStorageBuffers} storage buffers per shader stage; ` +
109
+ `device reports maxStorageBuffersPerShaderStage=${maxStorageBuffersPerShaderStage}.`
110
+ );
111
+ }
112
+ }
113
+
114
+ function createSplitGatherUniforms(base, sections, sectionCount) {
115
+ const uniforms = { ...base };
116
+ for (let index = 0; index < sectionCount; index++) {
117
+ uniforms[`section${index}_rows`] = sections[index].rowCount;
118
+ }
119
+ uniforms._pad0 = 0;
120
+ uniforms._pad1 = 0;
121
+ return uniforms;
122
+ }
123
+
124
+ async function _gatherSplit(
125
+ target,
126
+ indices,
127
+ splitEmbedding,
128
+ numTokens,
129
+ hiddenSize,
130
+ vocabSize,
131
+ options = {}
132
+ ) {
133
+ const {
134
+ outputBuffer = null,
135
+ embeddingDtype,
136
+ outputDtype,
137
+ transpose = false,
138
+ indexOffset = 0,
139
+ inputHiddenSize = hiddenSize,
140
+ hiddenOffset = 0,
141
+ indirectBuffer = null,
142
+ indirectOffset = 0,
143
+ splitGatherSectionCount = null,
144
+ } = options;
145
+
146
+ if (embeddingDtype == null) {
147
+ throw new Error('[Gather] embeddingDtype is required.');
148
+ }
149
+ if (outputDtype == null) {
150
+ throw new Error('[Gather] outputDtype is required.');
151
+ }
152
+ const config = selectSplitGatherConfig(
153
+ splitEmbedding,
154
+ outputDtype,
155
+ splitGatherSectionCount ?? splitEmbedding?.metadata?.splitGatherSectionCount ?? null
156
+ );
157
+ const variant = config.variants[outputDtype];
158
+ if (!variant) {
159
+ throw new Error(`[Gather] ${config.label} does not support outputDtype=${outputDtype}.`);
160
+ }
161
+ const sections = normalizeSplitGatherSections(splitEmbedding, config.sectionCount, config.label);
162
+ const logicalRows = sections.reduce((sum, section) => sum + section.rowCount, 0);
163
+ if (logicalRows < vocabSize) {
164
+ throw new Error(
165
+ `[Gather] split embeddings expose ${logicalRows} rows but vocabSize=${vocabSize}.`
166
+ );
167
+ }
168
+
169
+ const caps = getKernelCapabilities();
170
+ if (!caps.hasF16) {
171
+ throw new Error(`[Gather] ${config.label} requires shader-f16 support.`);
172
+ }
173
+ if (splitEmbedding?.dtype !== 'f16' || embeddingDtype !== 'f16') {
174
+ throw new Error(`[Gather] ${config.label} requires f16 split embeddings.`);
175
+ }
176
+ if (splitEmbedding?.layout !== 'row' || transpose) {
177
+ throw new Error(`[Gather] ${config.label} requires row-major embeddings.`);
178
+ }
179
+ if (hiddenSize % 4 !== 0) {
180
+ throw new Error(`[Gather] ${config.label} requires hiddenSize to be divisible by 4.`);
181
+ }
182
+ if (!Number.isFinite(inputHiddenSize) || inputHiddenSize < hiddenSize) {
183
+ throw new Error('[Gather] inputHiddenSize must be >= hiddenSize.');
184
+ }
185
+ if (!Number.isFinite(hiddenOffset) || hiddenOffset < 0 || (hiddenOffset + hiddenSize) > inputHiddenSize) {
186
+ throw new Error('[Gather] hiddenOffset must select a valid hidden slice inside inputHiddenSize.');
187
+ }
188
+ assertSplitGatherBindingLimit(config);
189
+
190
+ trace.embed(
191
+ `${config.label}: numTokens=${numTokens}, hiddenSize=${hiddenSize}, vocabSize=${vocabSize}, ` +
192
+ `indexOffset=${indexOffset}, inputHiddenSize=${inputHiddenSize}, hiddenOffset=${hiddenOffset}, ` +
193
+ `sections=${splitEmbedding.sections.length}, embeddingDtype=${embeddingDtype}, outputDtype=${outputDtype}`
194
+ );
195
+
196
+ const actualDtype = outputDtype;
197
+ const bytesPerElement = DTYPE_SIZES[actualDtype];
198
+ const paddedHiddenSize = padToQ4KBlock(hiddenSize);
199
+ const outputSize = numTokens * paddedHiddenSize * bytesPerElement;
200
+ const output = outputBuffer || acquireBuffer(outputSize, undefined, `${config.label}_output`);
201
+ const ownedOutput = outputBuffer ? null : output;
202
+
203
+ const uniforms = createSplitGatherUniforms({
204
+ num_tokens: numTokens,
205
+ hidden_size: hiddenSize,
206
+ vocab_size: vocabSize,
207
+ index_offset: indexOffset,
208
+ input_hidden_size: inputHiddenSize,
209
+ hidden_offset: hiddenOffset,
210
+ }, sections, config.sectionCount);
211
+
212
+ const workgroups = indirectBuffer
213
+ ? { indirectBuffer, indirectOffset }
214
+ : Math.ceil((numTokens * hiddenSize) / VEC4_ELEMENTS_PER_WG);
215
+
216
+ try {
217
+ await unifiedKernelWrapper(
218
+ config.op,
219
+ target,
220
+ variant,
221
+ [
222
+ indices,
223
+ ...sections.map((section) => section.buffer),
224
+ output,
225
+ ],
226
+ uniforms,
227
+ workgroups
228
+ );
229
+ return createTensor(output, actualDtype, [numTokens, hiddenSize], `${config.label}_output`);
230
+ } catch (error) {
231
+ if (ownedOutput) {
232
+ releaseBuffer(ownedOutput);
233
+ }
234
+ throw error;
235
+ }
236
+ }
237
+
19
238
  async function _gather(
20
239
  target,
21
240
  indices,
@@ -35,6 +254,7 @@ async function _gather(
35
254
  hiddenOffset = 0,
36
255
  indirectBuffer = null,
37
256
  indirectOffset = 0,
257
+ storageEncoding = null,
38
258
  } = options;
39
259
 
40
260
  const caps = getKernelCapabilities();
@@ -44,6 +264,10 @@ async function _gather(
44
264
  if (outputDtype == null) {
45
265
  throw new Error('[Gather] outputDtype is required.');
46
266
  }
267
+ const useLiteRTInt4Input = embeddingDtype === 'litert_int4';
268
+ if (embeddingDtype !== 'f16' && embeddingDtype !== 'f32' && !useLiteRTInt4Input) {
269
+ throw new Error(`[Gather] unsupported embeddingDtype="${embeddingDtype}".`);
270
+ }
47
271
  if (embeddingDtype === 'f16' && !caps.hasF16) {
48
272
  throw new Error('[Gather] embeddingDtype=f16 requires shader-f16 support.');
49
273
  }
@@ -67,6 +291,21 @@ async function _gather(
67
291
  const useF16Input = embeddingDtype === 'f16';
68
292
  const useF16Output = outputDtype === 'f16';
69
293
  const useVec4 = wantsVec4 && hiddenSize % 4 === 0;
294
+ if (useLiteRTInt4Input) {
295
+ resolveLiteRTInt4StorageEncoding(storageEncoding);
296
+ if (outputDtype !== 'f16' && outputDtype !== 'f32') {
297
+ throw new Error('[Gather] LiteRT INT4 embeddings require outputDtype=f16 or outputDtype=f32.');
298
+ }
299
+ if (transpose) {
300
+ throw new Error('[Gather] LiteRT INT4 embeddings require row-major layout.');
301
+ }
302
+ if (usesHiddenSlice) {
303
+ throw new Error('[Gather] LiteRT INT4 embeddings do not support hidden slicing.');
304
+ }
305
+ if (!useVec4) {
306
+ throw new Error('[Gather] LiteRT INT4 embeddings require vec4 gather; hiddenSize must be divisible by 4.');
307
+ }
308
+ }
70
309
 
71
310
  trace.embed(
72
311
  `Gather: numTokens=${numTokens}, hiddenSize=${hiddenSize}, vocabSize=${vocabSize}, ` +
@@ -76,8 +315,11 @@ async function _gather(
76
315
  `useF16Input=${useF16Input}, useF16Output=${useF16Output}`
77
316
  );
78
317
 
79
- const variant = selectGatherVariant(useF16Input, useF16Output, useVec4);
318
+ const variant = selectGatherVariant(useF16Input, useF16Output, useVec4, useLiteRTInt4Input);
80
319
  trace.embed(`Gather variant: ${variant}`);
320
+ const constants = useLiteRTInt4Input
321
+ ? { STORAGE_OFFSET_BINARY: resolveLiteRTInt4StorageEncoding(storageEncoding) === 'offset_binary' ? 1 : 0 }
322
+ : null;
81
323
 
82
324
  // Pad hiddenSize to Q4K alignment for downstream fused Q4K matmul kernels
83
325
  // that read 256-element blocks. Extra padding elements stay zero.
@@ -113,7 +355,8 @@ async function _gather(
113
355
  variant,
114
356
  [indices, embeddings, output],
115
357
  uniforms,
116
- workgroups
358
+ workgroups,
359
+ constants
117
360
  );
118
361
  return createTensor(output, actualDtype, [numTokens, hiddenSize], 'gather_output');
119
362
  } catch (error) {
@@ -146,3 +389,108 @@ export async function recordGather(
146
389
  ) {
147
390
  return _gather(recorder, indices, embeddings, numTokens, hiddenSize, vocabSize, options);
148
391
  }
392
+
393
+ export async function runGatherSplit4(
394
+ indices,
395
+ splitEmbedding,
396
+ numTokens,
397
+ hiddenSize,
398
+ vocabSize,
399
+ options = {}
400
+ ) {
401
+ if ((splitEmbedding?.sections?.length ?? 0) > SPLIT4_GATHER_SECTION_COUNT) {
402
+ throw new Error(
403
+ `[Gather] split embeddings have ${splitEmbedding.sections.length} sections; ` +
404
+ `gather_split4 supports at most ${SPLIT4_GATHER_SECTION_COUNT}.`
405
+ );
406
+ }
407
+ return _gatherSplit(null, indices, splitEmbedding, numTokens, hiddenSize, vocabSize, {
408
+ ...options,
409
+ splitGatherSectionCount: SPLIT4_GATHER_SECTION_COUNT,
410
+ });
411
+ }
412
+
413
+ export async function recordGatherSplit4(
414
+ recorder,
415
+ indices,
416
+ splitEmbedding,
417
+ numTokens,
418
+ hiddenSize,
419
+ vocabSize,
420
+ options = {}
421
+ ) {
422
+ if ((splitEmbedding?.sections?.length ?? 0) > SPLIT4_GATHER_SECTION_COUNT) {
423
+ throw new Error(
424
+ `[Gather] split embeddings have ${splitEmbedding.sections.length} sections; ` +
425
+ `gather_split4 supports at most ${SPLIT4_GATHER_SECTION_COUNT}.`
426
+ );
427
+ }
428
+ return _gatherSplit(recorder, indices, splitEmbedding, numTokens, hiddenSize, vocabSize, {
429
+ ...options,
430
+ splitGatherSectionCount: SPLIT4_GATHER_SECTION_COUNT,
431
+ });
432
+ }
433
+
434
+ export async function runGatherSplit8(
435
+ indices,
436
+ splitEmbedding,
437
+ numTokens,
438
+ hiddenSize,
439
+ vocabSize,
440
+ options = {}
441
+ ) {
442
+ if ((splitEmbedding?.sections?.length ?? 0) > SPLIT8_GATHER_SECTION_COUNT) {
443
+ throw new Error(
444
+ `[Gather] split embeddings have ${splitEmbedding.sections.length} sections; ` +
445
+ `gather_split8 supports at most ${SPLIT8_GATHER_SECTION_COUNT}.`
446
+ );
447
+ }
448
+ return _gatherSplit(null, indices, splitEmbedding, numTokens, hiddenSize, vocabSize, {
449
+ ...options,
450
+ splitGatherSectionCount: SPLIT8_GATHER_SECTION_COUNT,
451
+ });
452
+ }
453
+
454
+ export async function recordGatherSplit8(
455
+ recorder,
456
+ indices,
457
+ splitEmbedding,
458
+ numTokens,
459
+ hiddenSize,
460
+ vocabSize,
461
+ options = {}
462
+ ) {
463
+ if ((splitEmbedding?.sections?.length ?? 0) > SPLIT8_GATHER_SECTION_COUNT) {
464
+ throw new Error(
465
+ `[Gather] split embeddings have ${splitEmbedding.sections.length} sections; ` +
466
+ `gather_split8 supports at most ${SPLIT8_GATHER_SECTION_COUNT}.`
467
+ );
468
+ }
469
+ return _gatherSplit(recorder, indices, splitEmbedding, numTokens, hiddenSize, vocabSize, {
470
+ ...options,
471
+ splitGatherSectionCount: SPLIT8_GATHER_SECTION_COUNT,
472
+ });
473
+ }
474
+
475
+ export async function runGatherSplit(
476
+ indices,
477
+ splitEmbedding,
478
+ numTokens,
479
+ hiddenSize,
480
+ vocabSize,
481
+ options = {}
482
+ ) {
483
+ return _gatherSplit(null, indices, splitEmbedding, numTokens, hiddenSize, vocabSize, options);
484
+ }
485
+
486
+ export async function recordGatherSplit(
487
+ recorder,
488
+ indices,
489
+ splitEmbedding,
490
+ numTokens,
491
+ hiddenSize,
492
+ vocabSize,
493
+ options = {}
494
+ ) {
495
+ return _gatherSplit(recorder, indices, splitEmbedding, numTokens, hiddenSize, vocabSize, options);
496
+ }
@@ -0,0 +1,75 @@
1
+ enable f16;
2
+
3
+ override WORKGROUP_SIZE_VEC4: u32 = 64u;
4
+ override WEIGHT_SCALE: f32 = 0.0625;
5
+ override STORAGE_OFFSET_BINARY: u32 = 0u;
6
+
7
+ struct Uniforms {
8
+ num_tokens: u32,
9
+ hidden_size: u32,
10
+ vocab_size: u32,
11
+ transpose: u32,
12
+ index_offset: u32,
13
+ input_hidden_size: u32,
14
+ hidden_offset: u32,
15
+ _pad0: u32,
16
+ _pad1: u32,
17
+ }
18
+
19
+ @group(0) @binding(0) var<uniform> u: Uniforms;
20
+ @group(0) @binding(1) var<storage, read> indices: array<u32>;
21
+ @group(0) @binding(2) var<storage, read> embeddings_words: array<u32>;
22
+ @group(0) @binding(4) var<storage, read_write> output_f16: array<f16>;
23
+
24
+ fn packed_byte(byte_index: u32) -> u32 {
25
+ let word = embeddings_words[byte_index >> 2u];
26
+ let shift = (byte_index & 3u) * 8u;
27
+ return (word >> shift) & 0xffu;
28
+ }
29
+
30
+ fn signed_int4(byte_value: u32, high_nibble: bool) -> f32 {
31
+ let value = select(byte_value & 0x0fu, (byte_value >> 4u) & 0x0fu, high_nibble);
32
+ if (STORAGE_OFFSET_BINARY == 1u) {
33
+ return f32(value) - 8.0;
34
+ }
35
+ return select(f32(value), f32(i32(value) - 16), value >= 8u);
36
+ }
37
+
38
+ fn read_weight(row_base: u32, dim: u32) -> f16 {
39
+ let byte_value = packed_byte(row_base + (dim >> 1u));
40
+ let value = signed_int4(byte_value, (dim & 1u) == 1u) * WEIGHT_SCALE;
41
+ return f16(value);
42
+ }
43
+
44
+ @compute @workgroup_size(WORKGROUP_SIZE_VEC4, 1, 1)
45
+ fn gather_litert_int4_vec4_f16_out(@builtin(global_invocation_id) gid: vec3<u32>) {
46
+ let tid = gid.x;
47
+ let vec4_per_row = u.hidden_size / 4u;
48
+ let total_vec4s = u.num_tokens * vec4_per_row;
49
+
50
+ if (tid >= total_vec4s) {
51
+ return;
52
+ }
53
+
54
+ let token_idx = tid / vec4_per_row;
55
+ let vec4_idx = tid % vec4_per_row;
56
+ let token_id = indices[token_idx + u.index_offset];
57
+ let out_base = tid * 4u;
58
+
59
+ if (token_id >= u.vocab_size) {
60
+ output_f16[out_base] = f16(0.0);
61
+ output_f16[out_base + 1u] = f16(0.0);
62
+ output_f16[out_base + 2u] = f16(0.0);
63
+ output_f16[out_base + 3u] = f16(0.0);
64
+ return;
65
+ }
66
+
67
+ let packed_row_bytes = (u.input_hidden_size + 1u) >> 1u;
68
+ let row_base = token_id * packed_row_bytes;
69
+ let dim_base = u.hidden_offset + vec4_idx * 4u;
70
+
71
+ output_f16[out_base] = read_weight(row_base, dim_base);
72
+ output_f16[out_base + 1u] = read_weight(row_base, dim_base + 1u);
73
+ output_f16[out_base + 2u] = read_weight(row_base, dim_base + 2u);
74
+ output_f16[out_base + 3u] = read_weight(row_base, dim_base + 3u);
75
+ }
@@ -0,0 +1,72 @@
1
+ override WORKGROUP_SIZE_VEC4: u32 = 64u;
2
+ override WEIGHT_SCALE: f32 = 0.0625;
3
+ override STORAGE_OFFSET_BINARY: u32 = 0u;
4
+
5
+ struct Uniforms {
6
+ num_tokens: u32,
7
+ hidden_size: u32,
8
+ vocab_size: u32,
9
+ transpose: u32,
10
+ index_offset: u32,
11
+ input_hidden_size: u32,
12
+ hidden_offset: u32,
13
+ _pad0: u32,
14
+ _pad1: u32,
15
+ }
16
+
17
+ @group(0) @binding(0) var<uniform> u: Uniforms;
18
+ @group(0) @binding(1) var<storage, read> indices: array<u32>;
19
+ @group(0) @binding(2) var<storage, read> embeddings_words: array<u32>;
20
+ @group(0) @binding(4) var<storage, read_write> output_f32: array<f32>;
21
+
22
+ fn packed_byte(byte_index: u32) -> u32 {
23
+ let word = embeddings_words[byte_index >> 2u];
24
+ let shift = (byte_index & 3u) * 8u;
25
+ return (word >> shift) & 0xffu;
26
+ }
27
+
28
+ fn signed_int4(byte_value: u32, high_nibble: bool) -> f32 {
29
+ let value = select(byte_value & 0x0fu, (byte_value >> 4u) & 0x0fu, high_nibble);
30
+ if (STORAGE_OFFSET_BINARY == 1u) {
31
+ return f32(value) - 8.0;
32
+ }
33
+ return select(f32(value), f32(i32(value) - 16), value >= 8u);
34
+ }
35
+
36
+ fn read_weight(row_base: u32, dim: u32) -> f32 {
37
+ let byte_value = packed_byte(row_base + (dim >> 1u));
38
+ return signed_int4(byte_value, (dim & 1u) == 1u) * WEIGHT_SCALE;
39
+ }
40
+
41
+ @compute @workgroup_size(WORKGROUP_SIZE_VEC4, 1, 1)
42
+ fn gather_litert_int4_vec4_f32_out(@builtin(global_invocation_id) gid: vec3<u32>) {
43
+ let tid = gid.x;
44
+ let vec4_per_row = u.hidden_size / 4u;
45
+ let total_vec4s = u.num_tokens * vec4_per_row;
46
+
47
+ if (tid >= total_vec4s) {
48
+ return;
49
+ }
50
+
51
+ let token_idx = tid / vec4_per_row;
52
+ let vec4_idx = tid % vec4_per_row;
53
+ let token_id = indices[token_idx + u.index_offset];
54
+ let out_base = tid * 4u;
55
+
56
+ if (token_id >= u.vocab_size) {
57
+ output_f32[out_base] = 0.0;
58
+ output_f32[out_base + 1u] = 0.0;
59
+ output_f32[out_base + 2u] = 0.0;
60
+ output_f32[out_base + 3u] = 0.0;
61
+ return;
62
+ }
63
+
64
+ let packed_row_bytes = (u.input_hidden_size + 1u) >> 1u;
65
+ let row_base = token_id * packed_row_bytes;
66
+ let dim_base = u.hidden_offset + vec4_idx * 4u;
67
+
68
+ output_f32[out_base] = read_weight(row_base, dim_base);
69
+ output_f32[out_base + 1u] = read_weight(row_base, dim_base + 1u);
70
+ output_f32[out_base + 2u] = read_weight(row_base, dim_base + 2u);
71
+ output_f32[out_base + 3u] = read_weight(row_base, dim_base + 3u);
72
+ }
@@ -0,0 +1,103 @@
1
+ // gather_split4_f16_vec4_f16_out.wgsl
2
+
3
+ /**
4
+ * Gather Kernel (split4 F16 input -> F16 output, vec4)
5
+ *
6
+ * Row-major embedding lookup for logical embedding tables split across up to
7
+ * four GPU buffers to stay under maxStorageBufferBindingSize.
8
+ */
9
+
10
+ enable f16;
11
+
12
+ override WORKGROUP_SIZE_VEC4: u32 = 64u;
13
+
14
+ struct Uniforms {
15
+ num_tokens: u32,
16
+ hidden_size: u32,
17
+ vocab_size: u32,
18
+ index_offset: u32,
19
+ input_hidden_size: u32,
20
+ hidden_offset: u32,
21
+ section0_rows: u32,
22
+ section1_rows: u32,
23
+ section2_rows: u32,
24
+ section3_rows: u32,
25
+ _pad0: u32,
26
+ _pad1: u32,
27
+ }
28
+
29
+ @group(0) @binding(0) var<uniform> u: Uniforms;
30
+ @group(0) @binding(1) var<storage, read> indices: array<u32>;
31
+ @group(0) @binding(2) var<storage, read> embeddings0: array<f16>;
32
+ @group(0) @binding(3) var<storage, read> embeddings1: array<f16>;
33
+ @group(0) @binding(4) var<storage, read> embeddings2: array<f16>;
34
+ @group(0) @binding(5) var<storage, read> embeddings3: array<f16>;
35
+ @group(0) @binding(6) var<storage, read_write> output_f16: array<f16>;
36
+
37
+ fn write_zero(out_base: u32) {
38
+ output_f16[out_base] = f16(0.0);
39
+ output_f16[out_base + 1u] = f16(0.0);
40
+ output_f16[out_base + 2u] = f16(0.0);
41
+ output_f16[out_base + 3u] = f16(0.0);
42
+ }
43
+
44
+ @compute @workgroup_size(WORKGROUP_SIZE_VEC4, 1, 1)
45
+ fn gather_vec4_f16_out(@builtin(global_invocation_id) gid: vec3<u32>) {
46
+ let tid = gid.x;
47
+ let vec4_per_row = u.hidden_size / 4u;
48
+ let total_vec4s = u.num_tokens * vec4_per_row;
49
+
50
+ if (tid >= total_vec4s) {
51
+ return;
52
+ }
53
+
54
+ let token_idx = tid / vec4_per_row;
55
+ let vec4_idx = tid % vec4_per_row;
56
+ let token_id = indices[token_idx + u.index_offset];
57
+ let out_base = tid * 4u;
58
+
59
+ if (token_id >= u.vocab_size) {
60
+ write_zero(out_base);
61
+ return;
62
+ }
63
+
64
+ let section1_start = u.section0_rows;
65
+ let section2_start = section1_start + u.section1_rows;
66
+ let section3_start = section2_start + u.section2_rows;
67
+ let section_end = section3_start + u.section3_rows;
68
+ let dim_base = u.hidden_offset + vec4_idx * 4u;
69
+
70
+ var local_token: u32;
71
+ var embed_base: u32;
72
+ if (token_id < section1_start) {
73
+ local_token = token_id;
74
+ embed_base = local_token * u.input_hidden_size + dim_base;
75
+ output_f16[out_base] = embeddings0[embed_base];
76
+ output_f16[out_base + 1u] = embeddings0[embed_base + 1u];
77
+ output_f16[out_base + 2u] = embeddings0[embed_base + 2u];
78
+ output_f16[out_base + 3u] = embeddings0[embed_base + 3u];
79
+ } else if (token_id < section2_start) {
80
+ local_token = token_id - section1_start;
81
+ embed_base = local_token * u.input_hidden_size + dim_base;
82
+ output_f16[out_base] = embeddings1[embed_base];
83
+ output_f16[out_base + 1u] = embeddings1[embed_base + 1u];
84
+ output_f16[out_base + 2u] = embeddings1[embed_base + 2u];
85
+ output_f16[out_base + 3u] = embeddings1[embed_base + 3u];
86
+ } else if (token_id < section3_start) {
87
+ local_token = token_id - section2_start;
88
+ embed_base = local_token * u.input_hidden_size + dim_base;
89
+ output_f16[out_base] = embeddings2[embed_base];
90
+ output_f16[out_base + 1u] = embeddings2[embed_base + 1u];
91
+ output_f16[out_base + 2u] = embeddings2[embed_base + 2u];
92
+ output_f16[out_base + 3u] = embeddings2[embed_base + 3u];
93
+ } else if (token_id < section_end) {
94
+ local_token = token_id - section3_start;
95
+ embed_base = local_token * u.input_hidden_size + dim_base;
96
+ output_f16[out_base] = embeddings3[embed_base];
97
+ output_f16[out_base + 1u] = embeddings3[embed_base + 1u];
98
+ output_f16[out_base + 2u] = embeddings3[embed_base + 2u];
99
+ output_f16[out_base + 3u] = embeddings3[embed_base + 3u];
100
+ } else {
101
+ write_zero(out_base);
102
+ }
103
+ }