doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -27,6 +27,23 @@ function selectQ4KFusedVariant(isM1, wantF16Output, aDtype, phase) {
27
27
  return selectKernelRuleValue('matmul', 'q4kFusedVariant', { useF16A, useF16Out, isM1, isPrefill });
28
28
  }
29
29
 
30
+ function selectLiteRTInt4FusedVariant(outputDtype, aDtype, transposeB) {
31
+ return selectKernelRuleValue('matmul', 'litertInt4FusedVariant', {
32
+ activationDtype: aDtype,
33
+ outputDtype,
34
+ transposeB,
35
+ });
36
+ }
37
+
38
+ function selectW4A16FusedVariant(outputDtype, aDtype, transposeB, isPrefill) {
39
+ return selectKernelRuleValue('matmul', 'w4a16FusedVariant', {
40
+ activationDtype: aDtype,
41
+ outputDtype,
42
+ transposeB,
43
+ isPrefill,
44
+ });
45
+ }
46
+
30
47
 
31
48
  export function resolveMatmulPhase(M, phaseOverride = null) {
32
49
  if (phaseOverride != null) {
@@ -207,6 +224,14 @@ export function getMatmulBindingSizes(label, A, B, M, N, K, aDtype, bDtype, tran
207
224
  const numBlocksPerRow = Math.ceil(K / QK_K);
208
225
  bBindingSize = Math.ceil((N * numBlocksPerRow * Q4K_BLOCK_BYTES) / 4) * 4;
209
226
  bRequired = bOffset + bBindingSize;
227
+ } else if (bDtype === 'litert_int4') {
228
+ const packedBytesPerRow = Math.ceil(K / 2);
229
+ bBindingSize = Math.ceil((N * packedBytesPerRow) / 4) * 4;
230
+ bRequired = bOffset + bBindingSize;
231
+ } else if (bDtype === 'w4a16') {
232
+ const groupsPerRow = Math.ceil(K / 32);
233
+ bBindingSize = Math.ceil((N * groupsPerRow * 16) / 4) * 4;
234
+ bRequired = bOffset + bBindingSize;
210
235
  } else {
211
236
  const bBytesPerElem = bDtype === 'f16' ? 2 : 4;
212
237
  const bElements = transposeB ? N * K : K * N;
@@ -229,6 +254,14 @@ function isQ4KFusedVariant(variant) {
229
254
  return variant.startsWith('q4_fused');
230
255
  }
231
256
 
257
+ function isLiteRTInt4FusedVariant(variant) {
258
+ return variant.startsWith('litert_int4_');
259
+ }
260
+
261
+ function isW4A16FusedVariant(variant) {
262
+ return variant.startsWith('w4a16_');
263
+ }
264
+
232
265
 
233
266
  function isGemvVariant(variant) {
234
267
  return variant.startsWith('gemv');
@@ -236,7 +269,11 @@ function isGemvVariant(variant) {
236
269
 
237
270
 
238
271
  function supportsF16Input(variant) {
239
- return variant === 'f16' || variant === 'f16_vec4' || variant === 'f16_tiled' || variant.endsWith('_f16a');
272
+ return variant === 'f16'
273
+ || variant === 'f16_vec4'
274
+ || variant === 'f16_tiled'
275
+ || variant.endsWith('_f16a')
276
+ || variant.includes('_f16a_');
240
277
  }
241
278
 
242
279
  export function requiresF32Input(variant) {
@@ -254,6 +291,7 @@ function resolveMatmulOverride(
254
291
  K,
255
292
  aDtype,
256
293
  bDtype,
294
+ transposeB,
257
295
  requestedOutputDtype,
258
296
  capabilities,
259
297
  strict,
@@ -319,6 +357,7 @@ function resolveMatmulOverride(
319
357
  }
320
358
 
321
359
  const useQ4KFused = isQ4KFusedVariant(override);
360
+ const useLiteRTInt4Fused = isLiteRTInt4FusedVariant(override);
322
361
  if (useQ4KFused) {
323
362
  if (bDtype !== 'q4k') {
324
363
  return failOrWarn(`Matmul kernel "${variantOverride}" requires Q4K weights but B dtype is ${bDtype}.`);
@@ -327,13 +366,30 @@ function resolveMatmulOverride(
327
366
  return failOrWarn(`Matmul kernel "${variantOverride}" blocked by kernel path (fused Q4K disabled).`);
328
367
  }
329
368
  }
369
+ if (useLiteRTInt4Fused) {
370
+ if (bDtype !== 'litert_int4') {
371
+ return failOrWarn(`Matmul kernel "${variantOverride}" requires LiteRT INT4 weights but B dtype is ${bDtype}.`);
372
+ }
373
+ if (transposeB !== true) {
374
+ return failOrWarn(`Matmul kernel "${variantOverride}" requires transposeB=true for row-packed [N,K] weights.`);
375
+ }
376
+ }
377
+ const useW4A16Fused = isW4A16FusedVariant(override);
378
+ if (useW4A16Fused) {
379
+ if (bDtype !== 'w4a16') {
380
+ return failOrWarn(`Matmul kernel "${variantOverride}" requires W4A16 weights but B dtype is ${bDtype}.`);
381
+ }
382
+ if (transposeB !== true) {
383
+ return failOrWarn(`Matmul kernel "${variantOverride}" requires transposeB=true for row-packed [N,K] weights.`);
384
+ }
385
+ }
330
386
 
331
387
  const useGemv = isGemvVariant(override);
332
388
  if (useGemv && M !== 1) {
333
389
  return failOrWarn(`Matmul kernel "${variantOverride}" requires M=1 but got M=${M}.`);
334
390
  }
335
391
 
336
- return { variant: override, useQ4KFused, useGemv };
392
+ return { variant: override, useQ4KFused, useGemv, useLiteRTInt4Fused, useW4A16Fused };
337
393
  }
338
394
 
339
395
  function resolveGemvPathVariant(pathVariant, aDtype, requestedOutputDtype, N, multicolThreshold) {
@@ -378,19 +434,20 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
378
434
  K,
379
435
  aDtype,
380
436
  bDtype,
437
+ transposeB,
381
438
  requestedOutputDtype,
382
439
  capabilities,
383
440
  strict,
384
441
  fusedQ4KDisabled
385
442
  );
386
443
  if (!override && strict) {
387
- // When weights resolved to Q4K (precision upgrade) but the path variant requires F16,
444
+ // When weights resolved to packed quantized storage but the path variant requires F16,
388
445
  // fall through to auto-selection rather than throwing. The auto path will pick
389
- // the correct fused Q4K variant for the actual weight dtype.
390
- if (bDtype === 'q4k') {
446
+ // the correct fused variant for the actual weight dtype.
447
+ if (bDtype === 'q4k' || bDtype === 'litert_int4' || bDtype === 'w4a16') {
391
448
  logKernelSelectionOnce('matmul', {
392
449
  variant: pathVariant,
393
- reason: 'path_override_q4k_fallthrough',
450
+ reason: 'path_override_quantized_fallthrough',
394
451
  });
395
452
  } else {
396
453
  throw new Error(`[Matmul] Path variant "${pathVariant}" rejected for role=${options.role ?? '?'} layerIdx=${options.layerIdx ?? '?'} phase=${phase} M=${M} K=${K} aDtype=${aDtype} bDtype=${bDtype} outDtype=${requestedOutputDtype}`);
@@ -421,6 +478,7 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
421
478
  variant: adaptiveVariant,
422
479
  useQ4KFused: false,
423
480
  useGemv: false,
481
+ useLiteRTInt4Fused: false,
424
482
  };
425
483
  logKernelSelectionOnce('matmul', {
426
484
  variant: adaptiveSelection.variant,
@@ -440,7 +498,29 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
440
498
 
441
499
  const fusedAllowed = !fusedQ4KDisabled;
442
500
  const isQ4K = bDtype === 'q4k';
501
+ const isLiteRTInt4 = bDtype === 'litert_int4';
502
+ const isW4A16 = bDtype === 'w4a16';
443
503
  const wantF16Output = requestedOutputDtype === 'f16' && capabilities.hasF16;
504
+ const litertInt4Variant = isLiteRTInt4 && capabilities.hasF16
505
+ ? selectLiteRTInt4FusedVariant(requestedOutputDtype, aDtype, transposeB)
506
+ : null;
507
+ const useLiteRTInt4Fused = litertInt4Variant != null;
508
+ const w4a16Variant = isW4A16 && capabilities.hasF16
509
+ ? selectW4A16FusedVariant(requestedOutputDtype, aDtype, transposeB, M > 1)
510
+ : null;
511
+ const useW4A16Fused = w4a16Variant != null;
512
+ if (isLiteRTInt4 && !useLiteRTInt4Fused) {
513
+ throw new Error(
514
+ `[Matmul] LiteRT INT4 weights require a fused kernel path. ` +
515
+ `No variant matched M=${M} K=${K} aDtype=${aDtype} output=${requestedOutputDtype} transposeB=${transposeB}.`
516
+ );
517
+ }
518
+ if (isW4A16 && !useW4A16Fused) {
519
+ throw new Error(
520
+ `[Matmul] W4A16 weights require a fused kernel path. ` +
521
+ `No variant matched M=${M} K=${K} aDtype=${aDtype} output=${requestedOutputDtype} transposeB=${transposeB}.`
522
+ );
523
+ }
444
524
  let q4kVariant = isQ4K && capabilities.hasSubgroups && fusedAllowed
445
525
  ? selectQ4KFusedVariant(M === 1, wantF16Output, aDtype, phase)
446
526
  : null;
@@ -488,6 +568,17 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
488
568
  q4kVariant = 'q4_fused_widetile';
489
569
  }
490
570
  }
571
+ if (
572
+ options.useWideTileQ4KDecode === true
573
+ && phase === 'decode'
574
+ && M === 1
575
+ && aDtype === 'f32'
576
+ && !wantF16Output
577
+ && capabilities.hasF16 === true
578
+ && (q4kVariant === 'q4_fused_multicol' || q4kVariant === 'q4_fused')
579
+ ) {
580
+ q4kVariant = 'q4_fused_widetile';
581
+ }
491
582
  // Residual-fused WideTile override: when the caller passes a residualTensor
492
583
  // and opts into the fusion via useWideTileResidualFusion, route the
493
584
  // f32-output WideTile variant to its residual-epilogue twin. Saves one
@@ -512,7 +603,7 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
512
603
  q4kVariant = 'q4_fused_rmsnorm_widetile';
513
604
  }
514
605
 
515
- const effectiveBDtype = bDtype === 'q4k' ? 'f32' : bDtype;
606
+ const effectiveBDtype = (bDtype === 'q4k' || bDtype === 'litert_int4' || bDtype === 'w4a16') ? 'f32' : bDtype;
516
607
  const matmulVariant = selectMatmulKernel({
517
608
  ...options,
518
609
  aDtype: aDtype === 'q4k' ? 'f32' : aDtype,
@@ -537,9 +628,13 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
537
628
  const selection = selectKernelRuleValue(
538
629
  'matmul',
539
630
  'matmulSelection',
540
- { isQ4K, hasSubgroups: capabilities.hasSubgroups, fusedAllowed, useGemv, q4kVariant, gemvVariant, matmulVariant }
631
+ { isQ4K, isLiteRTInt4, isW4A16, hasSubgroups: capabilities.hasSubgroups, fusedAllowed, useGemv, useLiteRTInt4Fused, useW4A16Fused, litertInt4Variant, w4a16Variant, q4kVariant, gemvVariant, matmulVariant }
541
632
  );
542
- const reason = selection.useQ4KFused
633
+ const reason = selection.useLiteRTInt4Fused
634
+ ? 'litert_int4_fused'
635
+ : selection.useW4A16Fused
636
+ ? 'w4a16_fused'
637
+ : selection.useQ4KFused
543
638
  ? 'q4k_fused'
544
639
  : selection.useGemv
545
640
  ? 'gemv'
@@ -38,7 +38,7 @@ export interface MatmulOptions extends OutputBufferOptions, OutputDtypeOptions,
38
38
  bOffset?: number;
39
39
  cOffset?: number;
40
40
  aDtype?: 'f16' | 'f32' | null;
41
- bDtype?: 'f16' | 'f32' | 'q4k' | null;
41
+ bDtype?: 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16' | null;
42
42
  preferF16?: boolean;
43
43
  /** WGSL override constants for pipeline creation */
44
44
  constants?: Record<string, number | boolean>;
@@ -3,6 +3,7 @@ import { createTensor } from '../tensor.js';
3
3
  import {
4
4
  getBuffer,
5
5
  getLayout,
6
+ getWeightMetadata,
6
7
  getWeightDtype,
7
8
  isWeightBuffer,
8
9
  resolveWeightBufferMaterialization,
@@ -37,7 +38,7 @@ import {
37
38
  createMatmulBindGroupLayout,
38
39
  getMatmulPipeline,
39
40
  } from './matmul-dispatch.js';
40
- import { __dbgRecord } from './utils.js';
41
+ import { RECORD_STAGE_DEBUG_ENABLED, __dbgRecord, getPipelineBindGroupLayout } from './utils.js';
41
42
 
42
43
  export { isFusedQ4KDisabled, selectMatmulKernel };
43
44
  export { createMatmulBindGroupLayout };
@@ -81,6 +82,36 @@ function buildProfileLabel(options = {}) {
81
82
  return `matmul${roleLabel}${layerLabel}`;
82
83
  }
83
84
 
85
+ function resolveLiteRTInt4StorageEncoding(weight, label) {
86
+ const storageEncoding = String(getWeightMetadata(weight)?.storageEncoding ?? '').toLowerCase();
87
+ if (storageEncoding !== 'signed' && storageEncoding !== 'offset_binary') {
88
+ throw new Error(
89
+ `[Matmul] LiteRT INT4 weight "${label ?? 'unknown'}" requires metadata.storageEncoding ` +
90
+ `"signed" or "offset_binary", got "${storageEncoding || 'missing'}".`
91
+ );
92
+ }
93
+ return storageEncoding;
94
+ }
95
+
96
+ function resolveW4A16ScaleDtype(weight, label) {
97
+ const metadata = getWeightMetadata(weight);
98
+ const scaleDtype = String(metadata?.scaleDtype ?? '').toLowerCase();
99
+ if (scaleDtype !== 'f16' && scaleDtype !== 'bf16' && scaleDtype !== 'f32') {
100
+ throw new Error(
101
+ `[Matmul] W4A16 weight "${label ?? 'unknown'}" requires metadata.scaleDtype ` +
102
+ `"f16", "bf16", or "f32", got "${scaleDtype || 'missing'}".`
103
+ );
104
+ }
105
+ return scaleDtype;
106
+ }
107
+
108
+ function resolveW4A16ScaleDtypeConstant(scaleDtype) {
109
+ if (scaleDtype === 'f16') return 0;
110
+ if (scaleDtype === 'bf16') return 1;
111
+ if (scaleDtype === 'f32') return 2;
112
+ throw new Error(`[Matmul] unsupported W4A16 scale dtype "${scaleDtype}".`);
113
+ }
114
+
84
115
  function assertBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel, buffer, details = []) {
85
116
  const isGpuBuffer = buffer && (
86
117
  typeof GPUBuffer === 'undefined'
@@ -98,7 +129,7 @@ function assertBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel,
98
129
  );
99
130
  }
100
131
 
101
- function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes, residualBuffer = null, normWeightBuffer = null) {
132
+ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes, residualBuffer = null, normWeightBuffer = null, scaleBuffer = null) {
102
133
  const isQ4KF16 = variant === 'q4_fused_multicol_f16'
103
134
  || variant === 'q4_fused_f16a'
104
135
  || variant === 'q4_fused_batched_f16'
@@ -109,11 +140,12 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
109
140
  || variant === 'q4_fused_prefill_tiled_f16'
110
141
  || variant === 'q4_fused_widetile_f16'
111
142
  || variant === 'q4_fused_widetile_f16a';
112
- // 5-entry WideTile epilogue/prologue variants: output at binding 3 + one
143
+ // 5-entry Q4K epilogue/prologue variants: output at binding 3 + one
113
144
  // extra read-only buffer at binding 4 (residual for _residual, norm weight
114
145
  // for _rmsnorm). Distinct from isQ4KF16 (which puts output at binding 4).
115
- const isWideTileResidual = variant === 'q4_fused_widetile_residual';
146
+ const isQ4KResidual = variant === 'q4_fused_widetile_residual';
116
147
  const isWideTileRmsnorm = variant === 'q4_fused_rmsnorm_widetile';
148
+ const isW4A16 = variant.startsWith('w4a16_');
117
149
 
118
150
  assertBindGroupBuffer('matmul', variant, 0, 'uniforms', uniformBuffer);
119
151
  assertBindGroupBuffer('matmul', variant, 1, 'input', matmulInput?.buffer, [
@@ -121,8 +153,11 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
121
153
  `inputDtype=${matmulInput?.dtype ?? 'unknown'}`,
122
154
  ]);
123
155
  assertBindGroupBuffer('matmul', variant, 2, 'weights', bBuffer);
124
- assertBindGroupBuffer('matmul', variant, isQ4KF16 ? 4 : 3, 'output', outputBuffer);
125
- if (isWideTileResidual) {
156
+ if (isW4A16) {
157
+ assertBindGroupBuffer('matmul', variant, 3, 'scales', scaleBuffer);
158
+ }
159
+ assertBindGroupBuffer('matmul', variant, (isQ4KF16 || isW4A16) ? 4 : 3, 'output', outputBuffer);
160
+ if (isQ4KResidual) {
126
161
  if (!residualBuffer) {
127
162
  throw new Error(`[Matmul] variant "${variant}" requires a residual buffer but none was provided.`);
128
163
  }
@@ -141,7 +176,16 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
141
176
  { binding: 2, resource: { buffer: bBuffer, offset: offsets.bOffset, size: bindingSizes.bBindingSize } },
142
177
  ];
143
178
 
144
- if (isQ4KF16) {
179
+ if (isW4A16) {
180
+ entries.push({
181
+ binding: 3,
182
+ resource: { buffer: scaleBuffer },
183
+ });
184
+ entries.push({
185
+ binding: 4,
186
+ resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
187
+ });
188
+ } else if (isQ4KF16) {
145
189
  entries.push({
146
190
  binding: 4,
147
191
  resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
@@ -151,7 +195,7 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
151
195
  binding: 3,
152
196
  resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
153
197
  });
154
- if (isWideTileResidual) {
198
+ if (isQ4KResidual) {
155
199
  entries.push({
156
200
  binding: 4,
157
201
  resource: { buffer: residualBuffer },
@@ -254,6 +298,7 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
254
298
  const effectiveOptions = (
255
299
  options.useTiledQ4KPrefill == null
256
300
  || options.useWideTileQ4KPrefill == null
301
+ || options.useWideTileQ4KDecode == null
257
302
  || options.useWideTileResidualFusion == null
258
303
  || options.useFusedRmsnormWideTile == null
259
304
  )
@@ -261,12 +306,13 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
261
306
  ...options,
262
307
  useTiledQ4KPrefill: options.useTiledQ4KPrefill ?? (runtimeSession?.useTiledQ4KPrefill === true),
263
308
  useWideTileQ4KPrefill: options.useWideTileQ4KPrefill ?? (runtimeSession?.useWideTileQ4KPrefill === true),
309
+ useWideTileQ4KDecode: options.useWideTileQ4KDecode ?? (runtimeSession?.useWideTileQ4KDecode === true),
264
310
  useWideTileResidualFusion: options.useWideTileResidualFusion ?? (runtimeSession?.useWideTileResidualFusion === true),
265
311
  useFusedRmsnormWideTile: options.useFusedRmsnormWideTile ?? (runtimeSession?.useFusedRmsnormWideTile === true),
266
312
  }
267
313
  : options;
268
314
 
269
- let { variant, useQ4KFused, useGemv } = selectMatmulVariantAndFlags(
315
+ let { variant, useQ4KFused, useGemv, useLiteRTInt4Fused = false, useW4A16Fused = false } = selectMatmulVariantAndFlags(
270
316
  mode,
271
317
  M,
272
318
  N,
@@ -290,6 +336,23 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
290
336
  if (variant === 'f32' && constants && options.constants == null) {
291
337
  constants = null;
292
338
  }
339
+ if (bDtype === 'litert_int4') {
340
+ const storageEncoding = resolveLiteRTInt4StorageEncoding(resolvedWeight, weightLabel);
341
+ constants = {
342
+ ...(constants ?? {}),
343
+ STORAGE_OFFSET_BINARY: storageEncoding === 'offset_binary' ? 1 : 0,
344
+ };
345
+ }
346
+ let w4a16ScaleBuffer = null;
347
+ if (bDtype === 'w4a16') {
348
+ const metadata = getWeightMetadata(resolvedWeight);
349
+ w4a16ScaleBuffer = metadata?.scaleBuffer ?? null;
350
+ const scaleDtype = resolveW4A16ScaleDtype(resolvedWeight, weightLabel);
351
+ constants = {
352
+ ...(constants ?? {}),
353
+ SCALE_DTYPE: resolveW4A16ScaleDtypeConstant(scaleDtype),
354
+ };
355
+ }
293
356
  // For the rmsnorm-fused WideTile variant, forward the caller's
294
357
  // rmsNormOffset flag as a pipeline override constant. Gemma-family norm
295
358
  // weights encode `(w - 1.0)`; other models encode `w`.
@@ -297,7 +360,10 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
297
360
  // kernel correctly unpacks f16-packed weights (Gemma hidden weights) vs
298
361
  // f32 weights.
299
362
  if (variant === 'q4_fused_rmsnorm_widetile' && options.rmsNormOffset != null) {
300
- const normWeightDtype = getWeightDtype(options.normWeight) ?? 'f32';
363
+ const normWeightDtype = getWeightDtype(options.normWeight);
364
+ if (!normWeightDtype) {
365
+ throw new Error('[Matmul] q4_fused_rmsnorm_widetile requires normWeight dtype metadata.');
366
+ }
301
367
  constants = {
302
368
  ...(constants ?? {}),
303
369
  RMS_NORM_OFFSET: options.rmsNormOffset === true,
@@ -372,12 +438,24 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
372
438
  log.warn('MatmulQKVProbe', `${err.message} | ${probeDetail}`);
373
439
  }
374
440
  if (shouldFailOnSmallAttentionWeightBuffer) {
441
+ if (!isRecord && castedInput) {
442
+ releaseBuffer(castedInput.buffer);
443
+ castedInput = null;
444
+ }
375
445
  throw new Error(`${err.message}${detail}`);
376
446
  }
377
447
  }
378
448
  if (err instanceof Error && err.message.includes('B buffer too small')) {
449
+ if (!isRecord && castedInput) {
450
+ releaseBuffer(castedInput.buffer);
451
+ castedInput = null;
452
+ }
379
453
  throw new Error(`${err.message}${detail}`);
380
454
  }
455
+ if (!isRecord && castedInput) {
456
+ releaseBuffer(castedInput.buffer);
457
+ castedInput = null;
458
+ }
381
459
  throw err;
382
460
  }
383
461
 
@@ -402,39 +480,66 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
402
480
  );
403
481
  }
404
482
 
405
- const __dbg = (typeof process !== "undefined" ? process : { env: {} }).env.DOPPLER_DBG_RECORD === '1';
406
- const __t0 = __dbg ? performance.now() : 0;
407
- const config = getMatmulConfig(variant, constants);
408
- const kernel = new MatmulKernel(device);
409
- const pipeline = await getMatmulPipeline(variant, constants);
410
- const __tPipeline = __dbg ? performance.now() : 0;
411
-
412
- const { output: C, outputSize, cBindingSize, actualOutputDtype } = resolveMatmulOutput(
413
- variant,
414
- M,
415
- N,
416
- outputBuffer
417
- );
418
- const ownsOutput = outputBuffer == null;
419
-
420
- if (isAttnProj && shouldLogAttentionWeightBuffer) {
421
- log.warn('MatmulVariantDiag',
422
- `role=${options.role ?? ''} layer=${Number.isFinite(options.layerIdx) ? options.layerIdx : '?'} mode=${mode} ` +
423
- `variant=${variant} useQ4KFused=${useQ4KFused} useGemv=${useGemv} ` +
424
- `aDtype=${aDtype} bDtype=${bDtype} output=${actualOutputDtype}`
483
+ let __dbg = false;
484
+ let __t0 = 0;
485
+ let __tPipeline = 0;
486
+ let config;
487
+ let kernel;
488
+ let pipeline;
489
+ let C = null;
490
+ let outputSize;
491
+ let cBindingSize;
492
+ let actualOutputDtype;
493
+ let ownsOutput = false;
494
+ let dispatchPlan;
495
+ try {
496
+ __dbg = RECORD_STAGE_DEBUG_ENABLED;
497
+ __t0 = __dbg ? performance.now() : 0;
498
+ config = getMatmulConfig(variant, constants);
499
+ kernel = new MatmulKernel(device);
500
+ pipeline = await getMatmulPipeline(variant, constants);
501
+ __tPipeline = __dbg ? performance.now() : 0;
502
+
503
+ const outputInfo = resolveMatmulOutput(
504
+ variant,
505
+ M,
506
+ N,
507
+ outputBuffer
425
508
  );
426
- }
509
+ C = outputInfo.output;
510
+ outputSize = outputInfo.outputSize;
511
+ cBindingSize = outputInfo.cBindingSize;
512
+ actualOutputDtype = outputInfo.actualOutputDtype;
513
+ ownsOutput = outputBuffer == null;
514
+
515
+ if (isAttnProj && shouldLogAttentionWeightBuffer) {
516
+ log.warn('MatmulVariantDiag',
517
+ `role=${options.role ?? ''} layer=${Number.isFinite(options.layerIdx) ? options.layerIdx : '?'} mode=${mode} ` +
518
+ `variant=${variant} useQ4KFused=${useQ4KFused} useGemv=${useGemv} useLiteRTInt4Fused=${useLiteRTInt4Fused} ` +
519
+ `aDtype=${aDtype} bDtype=${bDtype} output=${actualOutputDtype}`
520
+ );
521
+ }
427
522
 
428
- if (!Number.isFinite(outputSize) || outputSize <= 0) {
429
- throw new Error(`[${opLabel}] Invalid output size: ${outputSize} (M=${M}, N=${N})`);
430
- }
523
+ if (!Number.isFinite(outputSize) || outputSize <= 0) {
524
+ throw new Error(`[${opLabel}] Invalid output size: ${outputSize} (M=${M}, N=${N})`);
525
+ }
431
526
 
432
- const cRequired = cOffset + cBindingSize;
433
- if (C.size < cRequired) {
434
- throw new Error(`[${opLabel}] Output buffer too small: ${C.size} < ${cRequired} (M=${M}, N=${N})`);
435
- }
527
+ const cRequired = cOffset + cBindingSize;
528
+ if (C.size < cRequired) {
529
+ throw new Error(`[${opLabel}] Output buffer too small: ${C.size} < ${cRequired} (M=${M}, N=${N})`);
530
+ }
436
531
 
437
- const dispatchPlan = calculateMatmulDispatch(variant, useQ4KFused, useGemv, M, N, config);
532
+ dispatchPlan = calculateMatmulDispatch(variant, useQ4KFused, useGemv, useLiteRTInt4Fused, M, N, config, useW4A16Fused);
533
+ } catch (error) {
534
+ if (!isRecord && castedInput) {
535
+ releaseBuffer(castedInput.buffer);
536
+ castedInput = null;
537
+ }
538
+ if (ownsOutput && C) {
539
+ releaseBuffer(C);
540
+ }
541
+ throw error;
542
+ }
438
543
  let uniformBuffer = null;
439
544
  let completed = false;
440
545
  try {
@@ -470,13 +575,14 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
470
575
  cBindingSize,
471
576
  },
472
577
  residualBuffer,
473
- normWeightBuffer
578
+ normWeightBuffer,
579
+ w4a16ScaleBuffer
474
580
  );
475
581
 
476
582
  const __tBgStart = __dbg ? performance.now() : 0;
477
583
  const bindGroup = device.createBindGroup({
478
584
  label: 'matmul_bind_group',
479
- layout: pipeline.getBindGroupLayout(0),
585
+ layout: getPipelineBindGroupLayout(pipeline, 0),
480
586
  entries,
481
587
  });
482
588
  const __tBg = __dbg ? performance.now() : 0;
@@ -490,16 +596,18 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
490
596
  const __tEnd = performance.now();
491
597
  __dbgRecord('matmul', variant, __tPipeline - __t0, __tBgStart - __tPipeline, __tBg - __tBgStart, __tEnd - __tBg);
492
598
  }
599
+ const tensor = createTensor(C, actualOutputDtype, [M, N], 'matmul_output');
493
600
  completed = true;
494
- return createTensor(C, actualOutputDtype, [M, N], 'matmul_output');
601
+ return tensor;
495
602
  } finally {
496
603
  if (!isRecord && uniformBuffer) {
497
604
  releaseUniformBuffer(uniformBuffer);
498
605
  }
499
606
  if (!isRecord && castedInput) {
500
607
  releaseBuffer(castedInput.buffer);
608
+ castedInput = null;
501
609
  }
502
- if (!completed && ownsOutput) {
610
+ if (!completed && ownsOutput && C) {
503
611
  releaseBuffer(C);
504
612
  }
505
613
  }
@@ -16,6 +16,7 @@ export interface MoEOptions extends OutputBufferOptions {
16
16
  normalize?: boolean;
17
17
  maxTokensPerExpert?: number;
18
18
  weightsDtype?: 'f16' | 'f32';
19
+ perExpertScale?: GPUBuffer | null;
19
20
  }
20
21
 
21
22
  /** MoE gather result */