doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -0,0 +1,216 @@
1
+ import { getDevice } from '../gpu/device.js';
2
+ import { DTYPE_SIZES } from '../config/schema/index.js';
3
+ import { selectRuleValue } from '../rules/rule-registry.js';
4
+ import { getLargeWeightMaxBytes } from './manifest-config.js';
5
+
6
+ export const MAX_SPLIT4_EMBEDDING_SECTIONS = 4;
7
+ export const MAX_SPLIT8_EMBEDDING_SECTIONS = 8;
8
+ export const MAX_SPLIT_EMBEDDING_SECTIONS = MAX_SPLIT8_EMBEDDING_SECTIONS;
9
+ export const SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD = 2;
10
+
11
+ export function getEmbeddingFloatDtype(location) {
12
+ return selectRuleValue('loader', 'weights', 'floatLocationDtype', {
13
+ locationDtype: location?.dtype,
14
+ });
15
+ }
16
+
17
+ export function expectsSplitGpuEmbeddingKernel(embeddingKernel) {
18
+ return getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) > 0;
19
+ }
20
+
21
+ export function getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) {
22
+ const kernel = embeddingKernel?.kernel;
23
+ const entry = embeddingKernel?.entry;
24
+ if (kernel === 'gather_split4_f16_vec4_f16_out.wgsl' && entry === 'gather_vec4_f16_out') {
25
+ return MAX_SPLIT4_EMBEDDING_SECTIONS;
26
+ }
27
+ if (kernel === 'gather_split8_f16_vec4_f16_out.wgsl' && entry === 'gather_vec4_f16_out') {
28
+ return MAX_SPLIT8_EMBEDDING_SECTIONS;
29
+ }
30
+ if (kernel === 'gather_split8_f16_vec4_f32_out.wgsl' && entry === 'gather_vec4_f32_out') {
31
+ return MAX_SPLIT8_EMBEDDING_SECTIONS;
32
+ }
33
+ return 0;
34
+ }
35
+
36
+ export function getSplitGpuEmbeddingRequiredStorageBuffers(sectionCount) {
37
+ return sectionCount + SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD;
38
+ }
39
+
40
+ export function getMaxSplitGpuEmbeddingSectionsForDevice(embeddingKernel = null, device = getDevice()) {
41
+ const kernelSections = getSplitGpuEmbeddingKernelSectionCount(embeddingKernel);
42
+ const maxStorageBuffersPerShaderStage = device?.limits?.maxStorageBuffersPerShaderStage;
43
+ if (kernelSections > 0) {
44
+ const requiredStorageBuffers = getSplitGpuEmbeddingRequiredStorageBuffers(kernelSections);
45
+ if (
46
+ Number.isFinite(maxStorageBuffersPerShaderStage)
47
+ && maxStorageBuffersPerShaderStage < requiredStorageBuffers
48
+ ) {
49
+ return 0;
50
+ }
51
+ return kernelSections;
52
+ }
53
+ if (!Number.isFinite(maxStorageBuffersPerShaderStage)) {
54
+ return MAX_SPLIT_EMBEDDING_SECTIONS;
55
+ }
56
+ return Math.max(
57
+ 0,
58
+ Math.min(
59
+ MAX_SPLIT_EMBEDDING_SECTIONS,
60
+ maxStorageBuffersPerShaderStage - SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD
61
+ )
62
+ );
63
+ }
64
+
65
+ function alignByteLength(byteLength) {
66
+ return Math.ceil(byteLength / 4) * 4;
67
+ }
68
+
69
+ function estimateEmbeddingTensorBytes(location) {
70
+ if (!location?.shape || location.shape.length !== 2) {
71
+ return null;
72
+ }
73
+ const [rows, hidden] = location.shape;
74
+ if (!Number.isFinite(rows) || rows <= 0 || !Number.isFinite(hidden) || hidden <= 0) {
75
+ return null;
76
+ }
77
+ const dtype = getEmbeddingFloatDtype(location);
78
+ const bytesPerElement = DTYPE_SIZES[dtype];
79
+ if (!Number.isFinite(bytesPerElement) || bytesPerElement <= 0) {
80
+ return null;
81
+ }
82
+ const tensorSizeBytes = rows * hidden * bytesPerElement;
83
+ if (!Number.isFinite(tensorSizeBytes) || tensorSizeBytes <= 0) {
84
+ return null;
85
+ }
86
+ return {
87
+ dtype,
88
+ rows,
89
+ hidden,
90
+ bytesPerElement,
91
+ tensorSizeBytes,
92
+ alignedTensorSizeBytes: alignByteLength(tensorSizeBytes),
93
+ };
94
+ }
95
+
96
+ export function createGpuResidentEmbeddingLimitError({ name, location, embeddingKernel = null }) {
97
+ const estimate = estimateEmbeddingTensorBytes(location);
98
+ if (!estimate) {
99
+ return null;
100
+ }
101
+ const device = getDevice();
102
+ const maxStorageBufferBindingSize = device?.limits?.maxStorageBufferBindingSize;
103
+ const maxBufferSize = device?.limits?.maxBufferSize;
104
+ const maxStorageBuffersPerShaderStage = device?.limits?.maxStorageBuffersPerShaderStage;
105
+ if (
106
+ !Number.isFinite(maxStorageBufferBindingSize)
107
+ || maxStorageBufferBindingSize <= 0
108
+ || !Number.isFinite(maxBufferSize)
109
+ || maxBufferSize <= 0
110
+ ) {
111
+ return null;
112
+ }
113
+ const maxGpuResidentBytes = Math.min(maxStorageBufferBindingSize, maxBufferSize);
114
+ if (estimate.alignedTensorSizeBytes <= maxGpuResidentBytes) {
115
+ return null;
116
+ }
117
+
118
+ const rowBytes = estimate.hidden * estimate.bytesPerElement;
119
+ const largeWeightMaxBytes = getLargeWeightMaxBytes();
120
+ const rowsPerSplitSection = largeWeightMaxBytes
121
+ ? Math.floor(largeWeightMaxBytes / rowBytes)
122
+ : null;
123
+ const requiredSplitSections = rowsPerSplitSection && rowsPerSplitSection > 0
124
+ ? Math.ceil(estimate.rows / rowsPerSplitSection)
125
+ : null;
126
+ const splitKernelExpected = expectsSplitGpuEmbeddingKernel(embeddingKernel);
127
+ const activeSplitKernelMaxSections = getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) || null;
128
+ const maxSplitEmbeddingSections = getMaxSplitGpuEmbeddingSectionsForDevice(embeddingKernel, device);
129
+ if (
130
+ splitKernelExpected
131
+ && Number.isFinite(requiredSplitSections)
132
+ && requiredSplitSections <= maxSplitEmbeddingSections
133
+ ) {
134
+ return null;
135
+ }
136
+ const error = new Error(
137
+ `[Loader] Embedding "${name}" cannot be GPU-resident on this device: ` +
138
+ `requires ${estimate.alignedTensorSizeBytes} bytes but max storage binding is ` +
139
+ `${maxGpuResidentBytes} bytes. Configure a split embedding kernel that fits this device ` +
140
+ 'or use a device with a larger storage binding limit.'
141
+ );
142
+ error.details = {
143
+ weightLoadFailure: {
144
+ tensorName: name,
145
+ tensorRole: location.role ?? null,
146
+ tensorDtype: location.dtype ?? null,
147
+ tensorShape: Array.isArray(location.shape) ? [...location.shape] : null,
148
+ tensorSizeBytes: estimate.alignedTensorSizeBytes,
149
+ tensorLoadStage: 'gpuResidentEmbeddingLimitPreflight',
150
+ toGPU: true,
151
+ streamedUpload: false,
152
+ deviceLimitFailure: {
153
+ kind: 'gpu_resident_embedding_exceeds_device_limit',
154
+ maxGpuResidentBytes,
155
+ maxStorageBufferBindingSize,
156
+ maxBufferSize,
157
+ maxStorageBuffersPerShaderStage: Number.isFinite(maxStorageBuffersPerShaderStage)
158
+ ? maxStorageBuffersPerShaderStage
159
+ : null,
160
+ largeWeightMaxBytes,
161
+ embeddingKernel: embeddingKernel
162
+ ? {
163
+ kernel: embeddingKernel.kernel ?? null,
164
+ entry: embeddingKernel.entry ?? null,
165
+ }
166
+ : null,
167
+ splitKernelExpected,
168
+ activeSplitKernelMaxSections,
169
+ maxSplitEmbeddingSections,
170
+ requiredSplitSections,
171
+ },
172
+ },
173
+ };
174
+ return error;
175
+ }
176
+
177
+ function resolveLargeWeightGpuResidentOverrides(manifestOverrides, runtimeConfig) {
178
+ const runtimeOverrides = runtimeConfig?.inference?.largeWeights?.gpuResidentOverrides;
179
+ if (Array.isArray(runtimeOverrides)) {
180
+ return runtimeOverrides;
181
+ }
182
+ if (Array.isArray(manifestOverrides)) {
183
+ return manifestOverrides;
184
+ }
185
+ return null;
186
+ }
187
+
188
+ export function resolveManifestGpuResidentEmbeddingLimitError(manifest, options = {}) {
189
+ const tensorLocations = options.storageManifest?.tensors ?? manifest?.tensors;
190
+ if (!tensorLocations || typeof tensorLocations !== 'object') {
191
+ return null;
192
+ }
193
+ const overrides = resolveLargeWeightGpuResidentOverrides(
194
+ manifest?.inference?.largeWeights?.gpuResidentOverrides,
195
+ options.runtimeConfig
196
+ );
197
+ if (!Array.isArray(overrides) || overrides.length === 0) {
198
+ return null;
199
+ }
200
+ const embeddingKernel = manifest?.inference?.execution?.kernels?.embed ?? null;
201
+ for (const name of overrides) {
202
+ const location = tensorLocations[name];
203
+ if (location?.role !== 'embedding') {
204
+ continue;
205
+ }
206
+ const error = createGpuResidentEmbeddingLimitError({
207
+ name,
208
+ location,
209
+ embeddingKernel,
210
+ });
211
+ if (error) {
212
+ return error;
213
+ }
214
+ }
215
+ return null;
216
+ }
@@ -14,6 +14,7 @@ import type {
14
14
  WeightBuffer,
15
15
  WeightLayout,
16
16
  CpuWeightBuffer,
17
+ SplitWeightBuffer,
17
18
  } from '../gpu/weight-buffer.js';
18
19
  import type { TensorLocation } from './loader-types.js';
19
20
 
@@ -22,7 +23,7 @@ export type TensorLoader = (
22
23
  name: string,
23
24
  toGPU?: boolean,
24
25
  silent?: boolean
25
- ) => Promise<GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | Uint8Array | null>;
26
+ ) => Promise<GPUBuffer | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer | Float32Array | Uint8Array | null>;
26
27
 
27
28
  /**
28
29
  * Context required for embedding loading.
@@ -46,10 +47,15 @@ export interface EmbeddingLoaderContext {
46
47
  preserveF32Embeddings?: boolean;
47
48
  /** Host shader-f16 capability, used to choose CPU F16->F32 fallback on no-f16 devices */
48
49
  hostHasShaderF16?: boolean | null;
50
+ /** Manifest-declared embedding kernel identity. */
51
+ embeddingKernel?: {
52
+ kernel?: string;
53
+ entry?: string;
54
+ } | null;
49
55
  }
50
56
 
51
57
  /** Result of embedding loading */
52
- export type EmbeddingResult = GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
58
+ export type EmbeddingResult = GPUBuffer | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer | Float32Array | null;
53
59
 
54
60
  /**
55
61
  * Load embedding weights.
@@ -3,20 +3,32 @@
3
3
  import {
4
4
  createWeightBuffer,
5
5
  createCpuWeightBuffer,
6
+ createSplitWeightBuffer,
6
7
  isWeightBuffer,
7
8
  isCpuWeightBuffer,
9
+ isSplitWeightBuffer,
8
10
  getWeightDtype,
9
11
  getLayout,
10
12
  } from '../gpu/weight-buffer.js';
13
+ import { getDevice } from '../gpu/device.js';
11
14
  import { maybeDowncastToF16 } from './weight-downcast.js';
12
15
  import { getTensorNamesByRole } from './tensors/tensor-role.js';
13
16
  import { log } from '../debug/index.js';
14
17
  import { selectRuleValue } from '../rules/rule-registry.js';
18
+ import { DTYPE_SIZES } from '../config/schema/index.js';
15
19
  import { createTensor } from '../gpu/tensor.js';
16
20
  import { castF16ToF32 } from '../gpu/kernel-selector.js';
17
21
  import { releaseBuffer } from '../memory/buffer-pool.js';
18
- import { loadTensorRange } from './tensors/tensor-reader.js';
22
+ import { assembleShardData, loadTensorRange } from './tensors/tensor-reader.js';
19
23
  import { hasSourceTransform } from './tensors/source-transform.js';
24
+ import { getLargeWeightMaxBytes } from './manifest-config.js';
25
+ import {
26
+ createGpuResidentEmbeddingLimitError,
27
+ expectsSplitGpuEmbeddingKernel,
28
+ getEmbeddingFloatDtype,
29
+ getSplitGpuEmbeddingKernelSectionCount,
30
+ getSplitGpuEmbeddingRequiredStorageBuffers,
31
+ } from './embedding-limit-preflight.js';
20
32
 
21
33
  // ============================================================================
22
34
  // Constants
@@ -25,6 +37,7 @@ import { hasSourceTransform } from './tensors/source-transform.js';
25
37
 
26
38
  const EMBEDDING_ROLE = 'embedding';
27
39
  const EMBEDDING_GROUP = 'embed';
40
+ const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
28
41
 
29
42
  function isPerLayerEmbeddingTensor(name) {
30
43
  return String(name ?? '').toLowerCase().includes('embed_tokens_per_layer.weight');
@@ -34,6 +47,17 @@ function isGpuBufferInstance(value) {
34
47
  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
35
48
  }
36
49
 
50
+ function normalizeLiteRTInt4StorageEncoding(transform, name) {
51
+ const storageEncoding = String(transform?.storageEncoding ?? '').toLowerCase();
52
+ if (storageEncoding !== 'signed' && storageEncoding !== 'offset_binary') {
53
+ throw new Error(
54
+ `[Loader] Embedding "${name}" LiteRT INT4 sourceTransform.storageEncoding must be ` +
55
+ `"signed" or "offset_binary", got "${transform?.storageEncoding ?? 'missing'}".`
56
+ );
57
+ }
58
+ return storageEncoding;
59
+ }
60
+
37
61
  function createRangeBackedTensorSource(ctx, name, location) {
38
62
  if (typeof ctx.loadShardRange !== 'function') {
39
63
  return null;
@@ -62,6 +86,219 @@ function createRangeBackedWeightBuffer(ctx, name, location) {
62
86
  return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
63
87
  }
64
88
 
89
+ function isFixedLiteRTInt4AffineEmbedding(ctx, name, location) {
90
+ if (ctx.hostHasShaderF16 === false) {
91
+ return false;
92
+ }
93
+ if (!location?.shape || location.shape.length !== 2) {
94
+ return false;
95
+ }
96
+ if (location.role !== EMBEDDING_ROLE) {
97
+ return false;
98
+ }
99
+ if (ctx.resolveWeightLayout(location) !== 'row') {
100
+ return false;
101
+ }
102
+ const transform = location.sourceTransform;
103
+ if (!transform || transform.kind !== 'affine_dequant') {
104
+ return false;
105
+ }
106
+ if (transform.scheme !== 'per_tensor_affine') {
107
+ return false;
108
+ }
109
+ if (String(transform.sourceDtype ?? '').toUpperCase() !== 'INT4') {
110
+ return false;
111
+ }
112
+ if (String(transform.targetDtype ?? '').toUpperCase() !== 'F16') {
113
+ return false;
114
+ }
115
+ normalizeLiteRTInt4StorageEncoding(transform, name);
116
+ if (Number(transform.scale) !== 0.0625 || Number(transform.zeroPoint) !== 0) {
117
+ return false;
118
+ }
119
+ const [, hiddenSize] = location.shape;
120
+ const expectedBytes = location.shape[0] * Math.ceil(hiddenSize / 2);
121
+ if (location.size !== expectedBytes) {
122
+ throw new Error(
123
+ `[Loader] Embedding "${name}" LiteRT INT4 source size mismatch: ` +
124
+ `manifest size=${location.size}, expected=${expectedBytes}.`
125
+ );
126
+ }
127
+ return typeof ctx.loadShardRange === 'function';
128
+ }
129
+
130
+ async function createLiteRTInt4GpuEmbeddingWeightBuffer(ctx, name, location) {
131
+ if (!isFixedLiteRTInt4AffineEmbedding(ctx, name, location)) {
132
+ return null;
133
+ }
134
+ const device = getDevice();
135
+ if (!device) {
136
+ return null;
137
+ }
138
+ const rawBytes = await assembleShardData(location, name, null, ctx.loadShardRange, {
139
+ materializeSourceTransform: false,
140
+ });
141
+ if (rawBytes.byteLength !== location.size) {
142
+ throw new Error(
143
+ `[Loader] Embedding "${name}" LiteRT INT4 raw read ${rawBytes.byteLength} bytes, ` +
144
+ `expected ${location.size}.`
145
+ );
146
+ }
147
+ const buffer = device.createBuffer({
148
+ label: `${name}:litert_int4`,
149
+ size: alignByteLength(rawBytes.byteLength),
150
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
151
+ });
152
+ try {
153
+ writeBufferInChunks(device.queue, buffer, rawBytes);
154
+ } catch (error) {
155
+ buffer.destroy();
156
+ throw error;
157
+ }
158
+ ctx.gpuBuffers.add(buffer);
159
+ log.info(
160
+ 'Loader',
161
+ `Embedding "${name}" stored as packed LiteRT INT4 GPU source ` +
162
+ `(bytes=${rawBytes.byteLength}, layout=row).`
163
+ );
164
+ return createWeightBuffer(buffer, 'litert_int4', 'row', location.shape, name, null, {
165
+ storageEncoding: normalizeLiteRTInt4StorageEncoding(location.sourceTransform, name),
166
+ });
167
+ }
168
+
169
+ function alignByteLength(byteLength) {
170
+ return Math.ceil(byteLength / 4) * 4;
171
+ }
172
+
173
+ function writeBufferInChunks(queue, buffer, bytes) {
174
+ for (let offset = 0; offset < bytes.byteLength; offset += SPLIT_UPLOAD_CHUNK_BYTES) {
175
+ const end = Math.min(offset + SPLIT_UPLOAD_CHUNK_BYTES, bytes.byteLength);
176
+ queue.writeBuffer(buffer, offset, bytes, offset, end - offset);
177
+ }
178
+ }
179
+
180
+ function expectsSplitGpuEmbedding(ctx) {
181
+ return expectsSplitGpuEmbeddingKernel(ctx.embeddingKernel);
182
+ }
183
+
184
+ async function createSplitGpuEmbeddingWeightBuffer(ctx, name, location) {
185
+ if (!expectsSplitGpuEmbedding(ctx)) {
186
+ return null;
187
+ }
188
+ if (ctx.hostHasShaderF16 === false) {
189
+ return null;
190
+ }
191
+ if (!location?.shape || location.shape.length !== 2) {
192
+ return null;
193
+ }
194
+ if (hasSourceTransform(location)) {
195
+ return null;
196
+ }
197
+ if (typeof ctx.loadShardRange !== 'function') {
198
+ return null;
199
+ }
200
+
201
+ const layout = ctx.resolveWeightLayout(location);
202
+ if (layout !== 'row') {
203
+ return null;
204
+ }
205
+
206
+ const dtype = getEmbeddingFloatDtype(location);
207
+ if (dtype !== 'f16') {
208
+ return null;
209
+ }
210
+
211
+ const device = getDevice();
212
+ const maxBytes = getLargeWeightMaxBytes();
213
+ if (!device || !maxBytes) {
214
+ return null;
215
+ }
216
+ const maxSplitSections = getSplitGpuEmbeddingKernelSectionCount(ctx.embeddingKernel);
217
+ if (maxSplitSections <= 0) {
218
+ return null;
219
+ }
220
+ const requiredStorageBuffers = getSplitGpuEmbeddingRequiredStorageBuffers(maxSplitSections);
221
+ if ((device.limits.maxStorageBuffersPerShaderStage ?? 0) < requiredStorageBuffers) {
222
+ log.warn(
223
+ 'Loader',
224
+ `Embedding "${name}" cannot use split GPU residency: ` +
225
+ `maxStorageBuffersPerShaderStage is below ${requiredStorageBuffers}.`
226
+ );
227
+ return null;
228
+ }
229
+
230
+ const [rows, hidden] = location.shape;
231
+ const bytesPerElement = DTYPE_SIZES[dtype];
232
+ const rowBytes = hidden * bytesPerElement;
233
+ if (!Number.isFinite(rowBytes) || rowBytes <= 0 || rowBytes > maxBytes) {
234
+ return null;
235
+ }
236
+
237
+ const totalBytes = rows * rowBytes;
238
+ if (totalBytes <= maxBytes) {
239
+ return null;
240
+ }
241
+
242
+ const rowsPerSection = Math.floor(maxBytes / rowBytes);
243
+ if (rowsPerSection <= 0) {
244
+ return null;
245
+ }
246
+ const sectionCount = Math.ceil(rows / rowsPerSection);
247
+ if (sectionCount > maxSplitSections) {
248
+ log.warn(
249
+ 'Loader',
250
+ `Embedding "${name}" needs ${sectionCount} GPU sections; active split kernel supports ` +
251
+ `at most ${maxSplitSections}. ` +
252
+ 'Using CPU-backed streaming.'
253
+ );
254
+ return null;
255
+ }
256
+
257
+ const createdBuffers = [];
258
+ try {
259
+ const sections = [];
260
+ for (let rowStart = 0; rowStart < rows; rowStart += rowsPerSection) {
261
+ const rowCount = Math.min(rowsPerSection, rows - rowStart);
262
+ const byteOffset = rowStart * rowBytes;
263
+ const byteLength = rowCount * rowBytes;
264
+ const bytes = await loadTensorRange(location, name, byteOffset, byteLength, ctx.loadShardRange);
265
+ if (bytes.byteLength !== byteLength) {
266
+ throw new Error(
267
+ `[Loader] Embedding "${name}" split section read ${bytes.byteLength} bytes, expected ${byteLength}.`
268
+ );
269
+ }
270
+ const buffer = device.createBuffer({
271
+ label: `${name}:split:${sections.length}`,
272
+ size: alignByteLength(byteLength),
273
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
274
+ });
275
+ createdBuffers.push(buffer);
276
+ writeBufferInChunks(device.queue, buffer, bytes);
277
+ sections.push({ buffer, rowStart, rowCount });
278
+ }
279
+ for (const buffer of createdBuffers) {
280
+ ctx.gpuBuffers.add(buffer);
281
+ }
282
+ log.warn(
283
+ 'Loader',
284
+ `Embedding "${name}" stored as split GPU sections (${sections.length} sections, dtype=${dtype}, layout=${layout}).`
285
+ );
286
+ return createSplitWeightBuffer(sections, dtype, layout, location.shape, name, {
287
+ splitGatherSectionCount: maxSplitSections,
288
+ sourceKernel: ctx.embeddingKernel,
289
+ });
290
+ } catch (error) {
291
+ for (const buffer of createdBuffers) {
292
+ try {
293
+ buffer.destroy();
294
+ } catch {
295
+ // Ignore cleanup failure while preserving the original load error.
296
+ }
297
+ }
298
+ throw error;
299
+ }
300
+ }
301
+
65
302
  function shouldUseRangeBackedEmbeddingSource(ctx, name, location) {
66
303
  if (!location) {
67
304
  return false;
@@ -112,29 +349,42 @@ export async function loadEmbeddings(ctx) {
112
349
 
113
350
  for (const name of candidates) {
114
351
  const loc = ctx.tensorLocations.get(name);
115
- const shouldStream = shouldUseRangeBackedEmbeddingSource(ctx, name, loc);
352
+ const packedLiteRTTensor = await createLiteRTInt4GpuEmbeddingWeightBuffer(ctx, name, loc);
353
+ const shouldStream = packedLiteRTTensor ? false : shouldUseRangeBackedEmbeddingSource(ctx, name, loc);
354
+ const splitGpuTensor = packedLiteRTTensor ? null : await createSplitGpuEmbeddingWeightBuffer(ctx, name, loc);
355
+ if (!packedLiteRTTensor && !splitGpuTensor && !shouldStream) {
356
+ const limitError = createGpuResidentEmbeddingLimitError({
357
+ name,
358
+ location: loc,
359
+ embeddingKernel: ctx.embeddingKernel,
360
+ });
361
+ if (limitError) {
362
+ throw limitError;
363
+ }
364
+ }
116
365
 
117
366
  // Load tensor (to CPU if streaming, to GPU otherwise)
118
- const tensor = shouldStream
367
+ const tensor = packedLiteRTTensor ?? splitGpuTensor ?? (shouldStream
119
368
  ? (
120
369
  createRangeBackedWeightBuffer(ctx, name, loc)
121
370
  ?? await ctx.loadTensor(name, false, true)
122
371
  )
123
- : await ctx.loadTensor(name, true, true);
372
+ : await ctx.loadTensor(name, true, true));
373
+ const tensorShouldStream = splitGpuTensor ? false : shouldStream;
124
374
 
125
375
  // Skip if not found
126
376
  if (!tensor) continue;
127
377
 
128
378
  // Handle streaming path (CPU)
129
- if (shouldStream && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
379
+ if (tensorShouldStream && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
130
380
  throw new Error(
131
381
  `[Loader] Embedding "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc?.dtype ?? 'unknown'}).`
132
382
  );
133
383
  }
134
384
 
135
385
  // Handle valid tensor types
136
- if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || tensor instanceof Float32Array) {
137
- const result = await processEmbeddingTensor(ctx, tensor, name, loc, shouldStream);
386
+ if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || isSplitWeightBuffer(tensor) || tensor instanceof Float32Array) {
387
+ const result = await processEmbeddingTensor(ctx, tensor, name, loc, tensorShouldStream);
138
388
  if (result) {
139
389
  return result;
140
390
  }
@@ -163,6 +413,10 @@ async function processEmbeddingTensor(ctx, tensor, name, loc, shouldStream) {
163
413
  // but loaded with an F32 gather kernel.
164
414
  const promoted = await maybePromoteEmbeddingsToF32(ctx, tensor, name, loc);
165
415
 
416
+ if (isSplitWeightBuffer(promoted)) {
417
+ return promoted;
418
+ }
419
+
166
420
  // WeightBuffer already has layout set correctly from _loadTensor
167
421
  if (isWeightBuffer(promoted)) {
168
422
  return maybeDowncastEmbeddings(ctx, promoted, name, loc);
@@ -205,6 +459,7 @@ async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
205
459
  if (!ctx.preserveF32Embeddings) return current;
206
460
  if (current instanceof Float32Array) return current;
207
461
  if (isCpuWeightBuffer(current)) return current;
462
+ if (isSplitWeightBuffer(current)) return current;
208
463
 
209
464
  if (isWeightBuffer(current)) {
210
465
  const dtype = getWeightDtype(current);
@@ -244,7 +499,7 @@ async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
244
499
 
245
500
  async function maybeDowncastEmbeddings(ctx, current, name, loc) {
246
501
  // Can't downcast Float32Array or CpuWeightBuffer
247
- if (current instanceof Float32Array || isCpuWeightBuffer(current)) {
502
+ if (current instanceof Float32Array || isCpuWeightBuffer(current) || isSplitWeightBuffer(current)) {
248
503
  return current;
249
504
  }
250
505
 
@@ -10,6 +10,7 @@
10
10
 
11
11
  import type { RDRRManifest } from '../../formats/rdrr/index.js';
12
12
  import type { ExpertCache } from './expert-cache.js';
13
+ import type { TensorLocation } from '../loader-types.js';
13
14
  import type { ExpertWeights } from '../weights.js';
14
15
  import type { WeightBuffer } from '../../gpu/weight-buffer.js';
15
16
  import type { ShardLoadOptions } from '../loader-types.js';
@@ -38,6 +39,8 @@ export interface ShardCacheInterface {
38
39
  export interface ExpertLoaderContext {
39
40
  /** Model manifest */
40
41
  manifest: RDRRManifest | null;
42
+ /** Resolved tensor locations from manifest.tensors or tensors.json */
43
+ tensorLocations?: Map<string, TensorLocation>;
41
44
  /** Load a tensor by name */
42
45
  loadTensor: TensorLoader;
43
46
  /** Load a shard by index */
@@ -46,7 +49,7 @@ export interface ExpertLoaderContext {
46
49
  shardCache: ShardCacheInterface;
47
50
  /** Expert LRU cache */
48
51
  expertCache: ExpertCache | null;
49
- /** Simple map for GPT-OSS packed experts */
52
+ /** Simple map for packed experts */
50
53
  experts: Map<string, ExpertWeights>;
51
54
  /** GPU buffers to track for cleanup */
52
55
  gpuBuffers: Set<GPUBuffer>;