doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -9,8 +9,9 @@ import { f16ToF32, convertBF16ToF32GPU, shouldDequantizeToF16, applyBufferLayout
9
9
  import { QK_K, Q4K_BLOCK_BYTES, Q6K_BLOCK_BYTES } from '../quantization-constants.js';
10
10
  import { log, trace as debugTrace } from '../../debug/index.js';
11
11
  import { selectRuleValue } from '../../rules/rule-registry.js';
12
- import { dequantizeQ4KM, dequantizeQ4KMRowWise } from '../../converter/quantizer.js';
12
+ import { dequantizeQ4KM, dequantizeQ4KMRowWise, float32ToFloat16 } from '../../converter/quantizer.js';
13
13
  import { hasSourceTransform } from './source-transform.js';
14
+ import { loadFunctionalDescriptor } from './functional-descriptor-loader.js';
14
15
 
15
16
  // ============================================================================
16
17
  // Q4K Detection
@@ -77,6 +78,28 @@ function toUint8View(data) {
77
78
  return new Uint8Array(data);
78
79
  }
79
80
 
81
+ function toUint16View(data, label) {
82
+ const bytes = toUint8View(data);
83
+ if (bytes.byteLength % 2 !== 0) {
84
+ throw new Error(`${label}: byte length must be divisible by 2.`);
85
+ }
86
+ if (bytes.byteOffset % 2 === 0) {
87
+ return new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
88
+ }
89
+ return new Uint16Array(bytes.slice().buffer);
90
+ }
91
+
92
+ function toFloat32View(data, label) {
93
+ const bytes = toUint8View(data);
94
+ if (bytes.byteLength % 4 !== 0) {
95
+ throw new Error(`${label}: byte length must be divisible by 4.`);
96
+ }
97
+ if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
98
+ return new Float32Array(bytes.buffer);
99
+ }
100
+ return new Float32Array(bytes.slice().buffer);
101
+ }
102
+
80
103
  function resolveInputByteLength(data, fallbackSize) {
81
104
  if (data instanceof Uint8Array) return data.byteLength;
82
105
  if (ArrayBuffer.isView(data)) return data.byteLength;
@@ -108,12 +131,203 @@ function getShapeElementCount(shape) {
108
131
  return shape.reduce((product, value) => product * value, 1);
109
132
  }
110
133
 
134
+ function getStorageCompanion(shardData, location, name, role) {
135
+ const companion = shardData?.storageCompanions?.[role];
136
+ if (!companion || !(companion.bytes instanceof Uint8Array)) {
137
+ throw new Error(
138
+ `W4A16 tensor "${name}" is missing required storage companion "${role}".`
139
+ );
140
+ }
141
+ const declared = Array.isArray(location?.storage?.companions)
142
+ ? location.storage.companions.find((entry) => entry.role === role)
143
+ : null;
144
+ if (declared && companion.tensorId !== declared.tensorId) {
145
+ throw new Error(
146
+ `W4A16 tensor "${name}" companion "${role}" resolved to "${companion.tensorId}", expected "${declared.tensorId}".`
147
+ );
148
+ }
149
+ return companion;
150
+ }
151
+
152
+ function readW4A16LogicalShape(companion, fallbackShape, name) {
153
+ const location = companion.location ?? null;
154
+ const bytes = companion.bytes;
155
+ const dtype = String(location?.dtype || '').toUpperCase();
156
+ if (!Array.isArray(location?.shape) || location.shape.length !== 1 || location.shape[0] !== 2) {
157
+ throw new Error(`W4A16 tensor "${name}" shape companion must have shape [2].`);
158
+ }
159
+ if (dtype === 'I64') {
160
+ if (bytes.byteLength !== 16) {
161
+ throw new Error(`W4A16 tensor "${name}" I64 shape companion must be 16 bytes.`);
162
+ }
163
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
164
+ return [Number(view.getBigInt64(0, true)), Number(view.getBigInt64(8, true))];
165
+ }
166
+ if (dtype === 'I32' || dtype === 'U32') {
167
+ if (bytes.byteLength !== 8) {
168
+ throw new Error(`W4A16 tensor "${name}" ${dtype} shape companion must be 8 bytes.`);
169
+ }
170
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
171
+ const read = dtype === 'I32'
172
+ ? (offset) => view.getInt32(offset, true)
173
+ : (offset) => view.getUint32(offset, true);
174
+ return [read(0), read(4)];
175
+ }
176
+ if (Array.isArray(fallbackShape) && fallbackShape.length === 2) {
177
+ return fallbackShape;
178
+ }
179
+ throw new Error(`W4A16 tensor "${name}" has unsupported shape companion dtype "${location?.dtype}".`);
180
+ }
181
+
182
+ function assertW4A16Shape(shape, fallbackShape, name) {
183
+ if (!Array.isArray(shape) || shape.length !== 2) {
184
+ throw new Error(`W4A16 tensor "${name}" logical shape must be 2D.`);
185
+ }
186
+ const rows = Number(shape[0]);
187
+ const cols = Number(shape[1]);
188
+ if (!Number.isInteger(rows) || rows <= 0 || !Number.isInteger(cols) || cols <= 0) {
189
+ throw new Error(`W4A16 tensor "${name}" has invalid logical shape ${JSON.stringify(shape)}.`);
190
+ }
191
+ if (Array.isArray(fallbackShape) && fallbackShape.length === 2) {
192
+ if (rows !== fallbackShape[0] || cols !== fallbackShape[1]) {
193
+ throw new Error(
194
+ `W4A16 tensor "${name}" shape companion [${rows},${cols}] does not match manifest shape [${fallbackShape.join(',')}].`
195
+ );
196
+ }
197
+ }
198
+ return [rows, cols];
199
+ }
200
+
201
+ function bf16ToF32(bits) {
202
+ const floats = new Float32Array(1);
203
+ const uints = new Uint32Array(floats.buffer);
204
+ uints[0] = (bits & 0xffff) << 16;
205
+ return floats[0];
206
+ }
207
+
208
+ function readOffsetBinaryInt4(byte, highNibble) {
209
+ const value = highNibble ? ((byte >> 4) & 0x0f) : (byte & 0x0f);
210
+ return value - 8;
211
+ }
212
+
213
+ function readW4A16Scales(companion, expectedScales, name) {
214
+ const dtype = String(companion.location?.dtype || '').toUpperCase();
215
+ if (dtype === 'F16') {
216
+ const packed = toUint16View(companion.bytes, `W4A16 scales for ${name}`);
217
+ if (packed.length !== expectedScales) {
218
+ throw new Error(
219
+ `W4A16 tensor "${name}" scale count ${packed.length} does not match expected ${expectedScales}.`
220
+ );
221
+ }
222
+ const scales = new Float32Array(packed.length);
223
+ for (let i = 0; i < packed.length; i += 1) {
224
+ scales[i] = f16ToF32(packed[i]);
225
+ }
226
+ return scales;
227
+ }
228
+ if (dtype === 'BF16') {
229
+ const packed = toUint16View(companion.bytes, `W4A16 scales for ${name}`);
230
+ if (packed.length !== expectedScales) {
231
+ throw new Error(
232
+ `W4A16 tensor "${name}" scale count ${packed.length} does not match expected ${expectedScales}.`
233
+ );
234
+ }
235
+ const scales = new Float32Array(packed.length);
236
+ for (let i = 0; i < packed.length; i += 1) {
237
+ scales[i] = bf16ToF32(packed[i]);
238
+ }
239
+ return scales;
240
+ }
241
+ if (dtype === 'F32') {
242
+ const scales = toFloat32View(companion.bytes, `W4A16 scales for ${name}`);
243
+ if (scales.length !== expectedScales) {
244
+ throw new Error(
245
+ `W4A16 tensor "${name}" scale count ${scales.length} does not match expected ${expectedScales}.`
246
+ );
247
+ }
248
+ return scales;
249
+ }
250
+ throw new Error(`W4A16 tensor "${name}" has unsupported scale companion dtype "${companion.location?.dtype}".`);
251
+ }
252
+
253
+ function validateW4A16PackedStorage(shardData, location, name) {
254
+ const scaleCompanion = getStorageCompanion(shardData, location, name, 'scales');
255
+ const shapeCompanion = getStorageCompanion(shardData, location, name, 'shape');
256
+ const [rows, cols] = assertW4A16Shape(
257
+ readW4A16LogicalShape(shapeCompanion, location.shape, name),
258
+ location.shape,
259
+ name
260
+ );
261
+ const groupsPerRow = Math.ceil(cols / 32);
262
+ const expectedPackedBytes = rows * groupsPerRow * 16;
263
+ if (shardData.byteLength !== expectedPackedBytes) {
264
+ throw new Error(
265
+ `W4A16 tensor "${name}" packed byte length ${shardData.byteLength} does not match expected ${expectedPackedBytes}.`
266
+ );
267
+ }
268
+ const scaleDtype = String(scaleCompanion.location?.dtype || '').toUpperCase();
269
+ const scaleBytesPerElement = scaleDtype === 'F32'
270
+ ? 4
271
+ : (scaleDtype === 'F16' || scaleDtype === 'BF16' ? 2 : null);
272
+ if (scaleBytesPerElement == null) {
273
+ throw new Error(`W4A16 tensor "${name}" has unsupported scale companion dtype "${scaleCompanion.location?.dtype}".`);
274
+ }
275
+ const expectedScaleBytes = rows * groupsPerRow * scaleBytesPerElement;
276
+ if (scaleCompanion.bytes.byteLength !== expectedScaleBytes) {
277
+ throw new Error(
278
+ `W4A16 tensor "${name}" scale byte length ${scaleCompanion.bytes.byteLength} does not match expected ${expectedScaleBytes}.`
279
+ );
280
+ }
281
+ return {
282
+ rows,
283
+ cols,
284
+ groupsPerRow,
285
+ scaleDtype: scaleDtype.toLowerCase(),
286
+ scaleBytes: scaleCompanion.bytes,
287
+ };
288
+ }
289
+
290
+ function dequantizeW4A16ToF16(shardData, location, name) {
291
+ const scaleCompanion = getStorageCompanion(shardData, location, name, 'scales');
292
+ const shapeCompanion = getStorageCompanion(shardData, location, name, 'shape');
293
+ const [rows, cols] = assertW4A16Shape(
294
+ readW4A16LogicalShape(shapeCompanion, location.shape, name),
295
+ location.shape,
296
+ name
297
+ );
298
+ const groupsPerRow = Math.ceil(cols / 32);
299
+ const expectedPackedBytes = rows * groupsPerRow * 16;
300
+ if (shardData.byteLength !== expectedPackedBytes) {
301
+ throw new Error(
302
+ `W4A16 tensor "${name}" packed byte length ${shardData.byteLength} does not match expected ${expectedPackedBytes}.`
303
+ );
304
+ }
305
+ const expectedScales = rows * groupsPerRow;
306
+ const scales = readW4A16Scales(scaleCompanion, expectedScales, name);
307
+ const out = new Uint16Array(rows * cols);
308
+ for (let row = 0; row < rows; row += 1) {
309
+ for (let group = 0; group < groupsPerRow; group += 1) {
310
+ const scale = scales[(row * groupsPerRow) + group];
311
+ const packedOffset = ((row * groupsPerRow) + group) * 16;
312
+ for (let lane = 0; lane < 32; lane += 1) {
313
+ const col = (group * 32) + lane;
314
+ if (col >= cols) break;
315
+ const byte = shardData[packedOffset + Math.floor(lane / 2)];
316
+ const quant = readOffsetBinaryInt4(byte, (lane % 2) === 1);
317
+ out[(row * cols) + col] = float32ToFloat16(quant * scale);
318
+ }
319
+ }
320
+ }
321
+ return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
322
+ }
323
+
111
324
 
112
325
  export function isPackedQ4K(location) {
113
- if (!Array.isArray(location.shape) || location.shape.length !== 2) {
326
+ if (!Array.isArray(location.shape) || location.shape.length < 2) {
114
327
  return false;
115
328
  }
116
- const [rows, cols] = location.shape;
329
+ const cols = location.shape[location.shape.length - 1];
330
+ const rows = location.shape.slice(0, -1).reduce((a, b) => a * b, 1);
117
331
  const expectedRowwise = rows * Math.ceil(cols / QK_K) * Q4K_BLOCK_BYTES;
118
332
  return location.size < expectedRowwise;
119
333
  }
@@ -145,6 +359,56 @@ function canUseFusedQ4KStorage(location, config) {
145
359
  return true;
146
360
  }
147
361
 
362
+ export function isLiteRTAffineInt4FusedEligible(location, config) {
363
+ const caps = config?.gpuCapabilities || getKernelCapabilities();
364
+ if (caps?.hasF16 !== true) return false;
365
+
366
+ if (!Array.isArray(location?.shape) || location.shape.length !== 2) return false;
367
+ if (!location?.role) return false;
368
+ if (isEmbeddingRole(location)) return false;
369
+ if (location.role !== 'matmul' && location.role !== 'lm_head') return false;
370
+ if (!shouldDequantizeToF16(location)) return false;
371
+
372
+ const transform = location.sourceTransform;
373
+ if (!transform || typeof transform !== 'object') return false;
374
+ const sourceDtype = String(transform.sourceDtype || '').toUpperCase();
375
+ const targetDtype = String(transform.targetDtype || '').toUpperCase();
376
+ const locationDtype = String(location.dtype || '').toUpperCase();
377
+ const storageEncoding = String(transform.storageEncoding || '').toLowerCase();
378
+ const scale = Number(transform.scale);
379
+ const zeroPoint = Number(transform.zeroPoint);
380
+ const storageEncodingSupported = storageEncoding === 'signed' || storageEncoding === 'offset_binary';
381
+
382
+ return transform.kind === 'affine_dequant'
383
+ && transform.scheme === 'per_tensor_affine'
384
+ && sourceDtype === 'INT4'
385
+ && targetDtype === 'F16'
386
+ && locationDtype === 'F16'
387
+ && storageEncodingSupported
388
+ && Number.isFinite(scale)
389
+ && Math.abs(scale - 0.0625) <= Number.EPSILON
390
+ && Number.isSafeInteger(zeroPoint)
391
+ && zeroPoint === 0;
392
+ }
393
+
394
+ export function isW4A16FusedEligible(location, config) {
395
+ const caps = config?.gpuCapabilities || getKernelCapabilities();
396
+ if (caps?.hasF16 !== true) return false;
397
+
398
+ if (!Array.isArray(location?.shape) || location.shape.length !== 2) return false;
399
+ if (String(location?.dtype || '').toUpperCase() !== 'W4A16') return false;
400
+ if (!location?.role) return false;
401
+ if (isEmbeddingRole(location)) return false;
402
+ if (location.role !== 'matmul' && location.role !== 'lm_head') return false;
403
+ if (!shouldDequantizeToF16(location)) return false;
404
+ if (location?.storage?.packing !== 'w4a16') return false;
405
+ if (!Array.isArray(location?.storage?.companions)) return false;
406
+
407
+ const hasScales = location.storage.companions.some((entry) => entry.role === 'scales');
408
+ const hasShape = location.storage.companions.some((entry) => entry.role === 'shape');
409
+ return hasScales && hasShape;
410
+ }
411
+
148
412
  function getQ4KDenseMaterializedSizeBytes(location, config) {
149
413
  if (!Array.isArray(location.shape) || location.shape.length === 0) {
150
414
  return null;
@@ -293,6 +557,80 @@ export async function loadQ4KFused(shardData, location, name) {
293
557
  }
294
558
  }
295
559
 
560
+ export async function loadLiteRTInt4Fused(shardData, location, name, config = null) {
561
+ if (isGpuBufferInstance(shardData)) {
562
+ throw new Error(
563
+ `LiteRT INT4 tensor "${name}" requires raw packed source bytes before GPU upload.`
564
+ );
565
+ }
566
+ if (!isLiteRTAffineInt4FusedEligible(location, config ?? { gpuCapabilities: getKernelCapabilities() })) {
567
+ throw new Error(
568
+ `LiteRT INT4 tensor "${name}" does not match the fused fixed-affine contract ` +
569
+ '(INT4 -> F16, storageEncoding=signed|offset_binary, per_tensor_affine, scale=0.0625, zeroPoint=0, 2D matmul/lm_head role).'
570
+ );
571
+ }
572
+
573
+ const [rows, cols] = location.shape;
574
+ const expectedBytes = rows * Math.ceil(cols / 2);
575
+ const actualBytes = resolveInputByteLength(shardData, location.size);
576
+ if (actualBytes !== expectedBytes) {
577
+ throw new Error(
578
+ `LiteRT INT4 tensor "${name}" packed byte size mismatch. ` +
579
+ `Expected ${expectedBytes} bytes for shape [${rows},${cols}], got ${actualBytes}.`
580
+ );
581
+ }
582
+
583
+ const device = getDevice();
584
+ const buffer = acquireAlignedBuffer(actualBytes, `litert_int4_${name}`);
585
+ try {
586
+ writeBufferAligned(device, buffer, shardData);
587
+ return {
588
+ data: createWeightBuffer(buffer, 'litert_int4', 'row', location.shape, name, null, {
589
+ storageEncoding: String(location.sourceTransform.storageEncoding).toLowerCase(),
590
+ }),
591
+ allocatedBuffers: [buffer],
592
+ };
593
+ } catch (error) {
594
+ releaseBuffer(buffer);
595
+ throw error;
596
+ }
597
+ }
598
+
599
+ export async function loadW4A16Fused(shardData, location, name, config = null) {
600
+ if (isGpuBufferInstance(shardData)) {
601
+ throw new Error(
602
+ `W4A16 tensor "${name}" requires raw packed source bytes before GPU upload.`
603
+ );
604
+ }
605
+ if (!isW4A16FusedEligible(location, config ?? { gpuCapabilities: getKernelCapabilities() })) {
606
+ throw new Error(
607
+ `W4A16 tensor "${name}" does not match the fused packed contract ` +
608
+ '(dtype=W4A16, storage.packing=w4a16, scales+shape companions, 2D matmul/lm_head role).'
609
+ );
610
+ }
611
+
612
+ const storage = validateW4A16PackedStorage(shardData, location, name);
613
+ const device = getDevice();
614
+ const weightBuffer = acquireAlignedBuffer(shardData.byteLength, `w4a16_${name}`);
615
+ const scaleBuffer = acquireAlignedBuffer(storage.scaleBytes.byteLength, `w4a16_scales_${name}`);
616
+ try {
617
+ writeBufferAligned(device, weightBuffer, shardData);
618
+ writeBufferAligned(device, scaleBuffer, storage.scaleBytes);
619
+ return {
620
+ data: createWeightBuffer(weightBuffer, 'w4a16', 'row', location.shape, name, null, {
621
+ scaleBuffer,
622
+ scaleDtype: storage.scaleDtype,
623
+ groupsPerRow: storage.groupsPerRow,
624
+ }),
625
+ allocatedBuffers: [weightBuffer, scaleBuffer],
626
+ };
627
+ } catch (error) {
628
+ releaseBuffer(weightBuffer);
629
+ releaseBuffer(scaleBuffer);
630
+ throw error;
631
+ }
632
+ }
633
+
296
634
  async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name, config) {
297
635
  let dequantized = null;
298
636
  const outputDtype = getQ4KOutputDtype(location, config);
@@ -341,16 +679,23 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
341
679
  }
342
680
 
343
681
  let numBlocks = null;
682
+ let rowCount = null;
683
+ let blocksPerRow = null;
684
+ let denseElementCount = null;
344
685
  let dequantizedTensor;
345
686
  if (needsRowwise) {
346
- const rows = location.shape[0];
687
+ rowCount = location.shape.slice(0, -1).reduce((a, b) => a * b, 1);
688
+ blocksPerRow = Math.ceil(K / QK_K);
689
+ numBlocks = rowCount * blocksPerRow;
690
+ denseElementCount = rowCount * K;
347
691
  debugTrace.loader(
348
- `Dequantizing ${name} (row-wise): [${rows},${K}], K not 256-aligned, ` +
692
+ `Dequantizing ${name} (row-wise): [${rowCount},${K}], K not 256-aligned, ` +
349
693
  `outputDtype=${outputDtype}`
350
694
  );
351
- dequantizedTensor = await dequantizeRowwise(quantBuffer, rows, K, { outputDtype });
695
+ dequantizedTensor = await dequantizeRowwise(quantBuffer, rowCount, K, { outputDtype });
352
696
  } else {
353
697
  numBlocks = Math.ceil(location.size / Q4K_BLOCK_BYTES);
698
+ denseElementCount = numBlocks * QK_K;
354
699
  debugTrace.loader(
355
700
  `Dequantizing ${name}: size=${location.size}, numBlocks=${numBlocks}, ` +
356
701
  `outputDtype=${outputDtype}, expectedOutput=${numBlocks * QK_K * (outputDtype === 'f16' ? 2 : 4)}`
@@ -369,9 +714,9 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
369
714
  if (isProbeTarget) {
370
715
  try {
371
716
  const bytesPerElem = outputDtype === 'f16' ? 2 : 4;
372
- const requestedOutputBytes = numBlocks * QK_K * bytesPerElem;
717
+ const requestedOutputBytes = denseElementCount * bytesPerElem;
373
718
  const sampleCount = paritySamples;
374
- const readSize = Math.min(sampleCount * bytesPerElem, dequantized.size);
719
+ const readSize = Math.min(sampleCount * bytesPerElem, requestedOutputBytes, dequantized.size);
375
720
  const gpuRaw = await readBuffer(dequantized, readSize);
376
721
  const gpuBytes = gpuRaw instanceof ArrayBuffer
377
722
  ? new Uint8Array(gpuRaw)
@@ -407,7 +752,8 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
407
752
 
408
753
  log.warn('DequantProbe',
409
754
  `tensor="${name}" shape=[${location.shape}] ` +
410
- `location.size=${location.size} numBlocks=${numBlocks} outputDtype=${outputDtype} ` +
755
+ `location.size=${location.size} numBlocks=${numBlocks} blocksPerRow=${blocksPerRow} ` +
756
+ `rows=${rowCount} K=${K} denseElements=${denseElementCount} outputDtype=${outputDtype} ` +
411
757
  `bytesPerElem=${bytesPerElem} requestedOutputBytes=${requestedOutputBytes} bufSize=${dequantized.size} ` +
412
758
  `runParity=true sampleCount=${sampleCount}`
413
759
  );
@@ -509,10 +855,10 @@ async function loadQ4KMixed(shardData, location, name, config) {
509
855
 
510
856
  function getQ4KCpuReferenceContext(shardData, location, config) {
511
857
  const outputDtype = getQ4KOutputDtype(location, config);
512
- const is2DMatrix = Array.isArray(location.shape) && location.shape.length === 2;
513
- const K = is2DMatrix ? location.shape[1] : 0;
514
- const needsRowwise = is2DMatrix && K > 0 && K % QK_K !== 0;
858
+ const isMatrixLike = Array.isArray(location.shape) && location.shape.length >= 2;
859
+ const K = isMatrixLike ? location.shape[location.shape.length - 1] : 0;
515
860
  const layout = getWeightLayout(location, config);
861
+ const needsRowwise = isMatrixLike && layout === 'row' && K > 0 && K % QK_K !== 0;
516
862
  const eligible = outputDtype === 'f32'
517
863
  && !isGpuBufferInstance(shardData)
518
864
  && (!needsRowwise || layout === 'row');
@@ -728,12 +1074,58 @@ export async function loadFloat(shardData, location, name, config) {
728
1074
  }
729
1075
  }
730
1076
 
1077
+ export async function loadW4A16Dequant(shardData, location, name, config) {
1078
+ if (!config) {
1079
+ throw new Error('Tensor load config is required.');
1080
+ }
1081
+ if (isGpuBufferInstance(shardData)) {
1082
+ throw new Error(
1083
+ `W4A16 tensor "${name}" requires CPU-side storage companion materialization before GPU upload.`
1084
+ );
1085
+ }
1086
+ const f16Bytes = dequantizeW4A16ToF16(shardData, location, name);
1087
+ const device = getDevice();
1088
+ const buffer = acquireAlignedBuffer(f16Bytes.byteLength, `w4a16_dequant_${name}`);
1089
+ try {
1090
+ writeBufferAligned(device, buffer, f16Bytes);
1091
+ const layout = selectRuleValue('loader', 'weights', 'weightLayout', {
1092
+ layout: location.layout ?? null,
1093
+ useColumnWise: false,
1094
+ });
1095
+ if (shouldDequantizeToF16(location)) {
1096
+ return {
1097
+ data: createWeightBuffer(buffer, 'f16', layout, location.shape, name),
1098
+ allocatedBuffers: [buffer],
1099
+ };
1100
+ }
1101
+ return {
1102
+ data: applyBufferLayout(buffer, location, 'f16'),
1103
+ allocatedBuffers: [buffer],
1104
+ };
1105
+ } catch (error) {
1106
+ releaseBuffer(buffer);
1107
+ throw error;
1108
+ }
1109
+ }
1110
+
731
1111
  // ============================================================================
732
1112
  // Main GPU Loading Entry Point
733
1113
  // ============================================================================
734
1114
 
735
1115
 
736
1116
  const GPU_LOADER_DISPATCH = {
1117
+ functional_descriptor: (shardData, location, name, config) => {
1118
+ debugTrace.loader(`Loading functional descriptor: ${name}`);
1119
+ return loadFunctionalDescriptor(shardData, location, name, config);
1120
+ },
1121
+ litert_int4_fused: (shardData, location, name, config) => {
1122
+ debugTrace.loader(`Loading LiteRT INT4 weight (fused): ${name} (size=${location.size})`);
1123
+ return loadLiteRTInt4Fused(shardData, location, name, config);
1124
+ },
1125
+ w4a16_fused: (shardData, location, name, config) => {
1126
+ debugTrace.loader(`Loading W4A16 weight (fused): ${name} (size=${location.size})`);
1127
+ return loadW4A16Fused(shardData, location, name, config);
1128
+ },
737
1129
  q4k_mixed: (shardData, location, name, config) => loadQ4KMixed(shardData, location, name, config),
738
1130
  q4k_fused: (shardData, location, name, _config) => {
739
1131
  debugTrace.loader(`Loading Q4K weight (fused): ${name} (size=${location.size})`);
@@ -759,8 +1151,15 @@ const GPU_LOADER_DISPATCH = {
759
1151
  }
760
1152
  ),
761
1153
  q6k: (shardData, location, name, _config) => loadQ6K(shardData, location, name),
1154
+ w4a16_dequant_reference: (shardData, location, name, config) => loadW4A16Dequant(shardData, location, name, config),
762
1155
  bf16: (shardData, location, name, config) => loadBF16(shardData, location, name, config),
763
1156
  float: (shardData, location, name, config) => loadFloat(shardData, location, name, config),
1157
+ unsupported_packed_quantization: (_shardData, location, name, _config) => {
1158
+ throw new Error(
1159
+ `Unsupported packed quantization dtype "${location.dtype}" for tensor "${name}". ` +
1160
+ 'Add a native loader and kernel path before enabling runtime execution.'
1161
+ );
1162
+ },
764
1163
  };
765
1164
 
766
1165
  export async function loadTensorToGPU(shardData, location, name, config) {
@@ -774,9 +1173,13 @@ export async function loadTensorToGPU(shardData, location, name, config) {
774
1173
  const q4kBasicBackendClass = platformId === 'basic'
775
1174
  || (caps?.hasSubgroups !== true && caps?.hasF16 !== true);
776
1175
  const q4kLimitFallback = resolveQ4KLimitFallback(location, config);
1176
+ const litertAffineInt4FusedEligible = isLiteRTAffineInt4FusedEligible(location, { ...config, gpuCapabilities: caps });
1177
+ const w4a16FusedEligible = isW4A16FusedEligible(location, { ...config, gpuCapabilities: caps });
777
1178
  const loaderPath = selectRuleValue('loader', 'tensorLoader', 'gpuLoaderPath', {
778
1179
  dtype,
779
1180
  role: location.role ?? null,
1181
+ litertAffineInt4FusedEligible,
1182
+ w4a16FusedEligible,
780
1183
  useFusedQ4K,
781
1184
  requiresFusedQ4KRole,
782
1185
  q4kMaterializationMode: config.q4kMaterializationMode ?? 'dense',
@@ -797,24 +1200,37 @@ export async function loadTensorToGPU(shardData, location, name, config) {
797
1200
 
798
1201
 
799
1202
  const CPU_LOADER_DISPATCH = {
1203
+ unsupported_functional_descriptor: (_shardData, _location, name) => {
1204
+ throw new Error(
1205
+ `FUNCTIONAL_DESCRIPTOR tensor "${name ?? 'unknown'}" requires GPU materialization through loadTensorToGPU.`
1206
+ );
1207
+ },
800
1208
  raw: (shardData, _location) => shardData,
801
- bf16_to_f32: (shardData, _location) => {
802
- const bf16 = new Uint16Array(shardData.slice().buffer);
803
- return convertBF16ToF32CPU(bf16);
1209
+ w4a16_dequant_reference: (shardData, location) => {
1210
+ const f16Bytes = dequantizeW4A16ToF16(shardData, location, 'cpu');
1211
+ return convertF16ToF32CPU(toUint16View(f16Bytes, 'W4A16 CPU dequantized tensor load'));
804
1212
  },
805
- f16_to_f32: (shardData, _location) => {
806
- const f16 = new Uint16Array(shardData.slice().buffer);
807
- return convertF16ToF32CPU(f16);
1213
+ unsupported_packed_quantization: (_shardData, location) => {
1214
+ throw new Error(
1215
+ `Unsupported packed quantization dtype "${location.dtype}" for CPU tensor load. ` +
1216
+ 'Add a native loader before enabling runtime execution.'
1217
+ );
808
1218
  },
809
- f32: (shardData, _location) => new Float32Array(shardData.slice().buffer),
1219
+ bf16_to_f32: (shardData, _location) => convertBF16ToF32CPU(
1220
+ toUint16View(shardData, 'BF16 CPU tensor load')
1221
+ ),
1222
+ f16_to_f32: (shardData, _location) => convertF16ToF32CPU(
1223
+ toUint16View(shardData, 'F16 CPU tensor load')
1224
+ ),
1225
+ f32: (shardData, _location) => toFloat32View(shardData, 'F32 CPU tensor load'),
810
1226
  };
811
1227
 
812
- export function loadTensorToCPU(shardData, location) {
1228
+ export function loadTensorToCPU(shardData, location, name = null) {
813
1229
  const dtype = location.dtype;
814
1230
  const loaderPath = selectRuleValue('loader', 'tensorLoader', 'cpuLoaderPath', { dtype });
815
1231
  const loader = CPU_LOADER_DISPATCH[loaderPath];
816
1232
  if (!loader) {
817
1233
  throw new Error(`Unknown CPU loader path: "${loaderPath}" for dtype "${dtype}"`);
818
1234
  }
819
- return loader(shardData, location);
1235
+ return loader(shardData, location, name);
820
1236
  }
@@ -18,7 +18,10 @@ export declare function assembleShardData(
18
18
  location: TensorLocation,
19
19
  name: string,
20
20
  loadShard: (index: number, options?: ShardLoadOptions) => Promise<ArrayBuffer>,
21
- loadShardRange?: (index: number, offset: number, length: number) => Promise<ArrayBuffer>
21
+ loadShardRange?: (index: number, offset: number, length: number) => Promise<ArrayBuffer>,
22
+ options?: {
23
+ materializeSourceTransform?: boolean;
24
+ }
22
25
  ): Promise<Uint8Array>;
23
26
 
24
27
  export declare function loadTensorRange(
@@ -205,7 +205,8 @@ async function materializeLocationBytes(rawBytes, location, name, loadShard, loa
205
205
  return materializeTensorSourceTransform(rawBytes, location, name);
206
206
  }
207
207
 
208
- export async function assembleShardData(location, name, loadShard, loadShardRange = null) {
208
+ export async function assembleShardData(location, name, loadShard, loadShardRange = null, options = {}) {
209
+ const shouldMaterializeSourceTransform = options.materializeSourceTransform !== false;
209
210
  const spans = getLocationSpans(location);
210
211
  if (spans) {
211
212
  trace.loader(`Assembling tensor "${name}" from ${spans.length} spans`);
@@ -240,7 +241,9 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
240
241
  combined.set(chunk, offset);
241
242
  offset += chunk.length;
242
243
  }
243
- return materializeLocationBytes(combined, location, name, loadShard, loadShardRange);
244
+ return shouldMaterializeSourceTransform
245
+ ? materializeLocationBytes(combined, location, name, loadShard, loadShardRange)
246
+ : combined;
244
247
  }
245
248
 
246
249
  // Single shard - use view to avoid copying
@@ -255,7 +258,9 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
255
258
  );
256
259
  }
257
260
  const bytes = new Uint8Array(slice, 0, size);
258
- return materializeLocationBytes(bytes, location, name, loadShard, loadShardRange);
261
+ return shouldMaterializeSourceTransform
262
+ ? materializeLocationBytes(bytes, location, name, loadShard, loadShardRange)
263
+ : bytes;
259
264
  }
260
265
 
261
266
  const fullShard = await loadShard(shardIndex);
@@ -265,7 +270,9 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
265
270
  );
266
271
  }
267
272
  const bytes = new Uint8Array(fullShard, offset, size);
268
- return materializeLocationBytes(bytes, location, name, loadShard, loadShardRange);
273
+ return shouldMaterializeSourceTransform
274
+ ? materializeLocationBytes(bytes, location, name, loadShard, loadShardRange)
275
+ : bytes;
269
276
  }
270
277
 
271
278
  export async function loadTensorRange(location, name, byteOffset, byteLength, loadShardRange) {
@@ -10,9 +10,11 @@ export interface ExpertWeights {
10
10
  gate?: GPUBuffer | WeightBuffer | Float32Array | null;
11
11
  up?: GPUBuffer | WeightBuffer | Float32Array | null;
12
12
  down?: GPUBuffer | WeightBuffer | Float32Array | null;
13
- expertFormat?: 'mixtral' | 'gpt-oss';
13
+ gateUp?: GPUBuffer | WeightBuffer | Float32Array | null;
14
+ expertFormat?: 'mixtral' | 'gpt-oss' | 'gemma4';
14
15
  expertIdx?: number;
15
16
  numExperts?: number;
17
+ expertIntermediateSize?: number;
16
18
  gateUpBlocks?: GPUBuffer | null;
17
19
  gateUpScales?: GPUBuffer | null;
18
20
  gateUpBias?: GPUBuffer | null;