doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,31 @@
1
+ export const MAX_SPLIT4_EMBEDDING_SECTIONS: 4;
2
+ export const MAX_SPLIT8_EMBEDDING_SECTIONS: 8;
3
+ export const MAX_SPLIT_EMBEDDING_SECTIONS: 8;
4
+ export const SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD: 2;
5
+
6
+ export function getEmbeddingFloatDtype(location: unknown): string;
7
+
8
+ export function expectsSplitGpuEmbeddingKernel(embeddingKernel: unknown): boolean;
9
+
10
+ export function getSplitGpuEmbeddingKernelSectionCount(embeddingKernel: unknown): number;
11
+
12
+ export function getSplitGpuEmbeddingRequiredStorageBuffers(sectionCount: number): number;
13
+
14
+ export function getMaxSplitGpuEmbeddingSectionsForDevice(
15
+ embeddingKernel?: unknown,
16
+ device?: unknown
17
+ ): number;
18
+
19
+ export function createGpuResidentEmbeddingLimitError(options: {
20
+ name: string;
21
+ location: unknown;
22
+ embeddingKernel?: unknown;
23
+ }): Error | null;
24
+
25
+ export function resolveManifestGpuResidentEmbeddingLimitError(
26
+ manifest: unknown,
27
+ options?: {
28
+ storageManifest?: unknown;
29
+ runtimeConfig?: unknown;
30
+ }
31
+ ): Error | null;
@@ -0,0 +1,216 @@
1
+ import { getDevice } from '../gpu/device.js';
2
+ import { DTYPE_SIZES } from '../config/schema/index.js';
3
+ import { selectRuleValue } from '../rules/rule-registry.js';
4
+ import { getLargeWeightMaxBytes } from './manifest-config.js';
5
+
6
+ export const MAX_SPLIT4_EMBEDDING_SECTIONS = 4;
7
+ export const MAX_SPLIT8_EMBEDDING_SECTIONS = 8;
8
+ export const MAX_SPLIT_EMBEDDING_SECTIONS = MAX_SPLIT8_EMBEDDING_SECTIONS;
9
+ export const SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD = 2;
10
+
11
+ export function getEmbeddingFloatDtype(location) {
12
+ return selectRuleValue('loader', 'weights', 'floatLocationDtype', {
13
+ locationDtype: location?.dtype,
14
+ });
15
+ }
16
+
17
+ export function expectsSplitGpuEmbeddingKernel(embeddingKernel) {
18
+ return getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) > 0;
19
+ }
20
+
21
+ export function getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) {
22
+ const kernel = embeddingKernel?.kernel;
23
+ const entry = embeddingKernel?.entry;
24
+ if (kernel === 'gather_split4_f16_vec4_f16_out.wgsl' && entry === 'gather_vec4_f16_out') {
25
+ return MAX_SPLIT4_EMBEDDING_SECTIONS;
26
+ }
27
+ if (kernel === 'gather_split8_f16_vec4_f16_out.wgsl' && entry === 'gather_vec4_f16_out') {
28
+ return MAX_SPLIT8_EMBEDDING_SECTIONS;
29
+ }
30
+ if (kernel === 'gather_split8_f16_vec4_f32_out.wgsl' && entry === 'gather_vec4_f32_out') {
31
+ return MAX_SPLIT8_EMBEDDING_SECTIONS;
32
+ }
33
+ return 0;
34
+ }
35
+
36
+ export function getSplitGpuEmbeddingRequiredStorageBuffers(sectionCount) {
37
+ return sectionCount + SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD;
38
+ }
39
+
40
+ export function getMaxSplitGpuEmbeddingSectionsForDevice(embeddingKernel = null, device = getDevice()) {
41
+ const kernelSections = getSplitGpuEmbeddingKernelSectionCount(embeddingKernel);
42
+ const maxStorageBuffersPerShaderStage = device?.limits?.maxStorageBuffersPerShaderStage;
43
+ if (kernelSections > 0) {
44
+ const requiredStorageBuffers = getSplitGpuEmbeddingRequiredStorageBuffers(kernelSections);
45
+ if (
46
+ Number.isFinite(maxStorageBuffersPerShaderStage)
47
+ && maxStorageBuffersPerShaderStage < requiredStorageBuffers
48
+ ) {
49
+ return 0;
50
+ }
51
+ return kernelSections;
52
+ }
53
+ if (!Number.isFinite(maxStorageBuffersPerShaderStage)) {
54
+ return MAX_SPLIT_EMBEDDING_SECTIONS;
55
+ }
56
+ return Math.max(
57
+ 0,
58
+ Math.min(
59
+ MAX_SPLIT_EMBEDDING_SECTIONS,
60
+ maxStorageBuffersPerShaderStage - SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD
61
+ )
62
+ );
63
+ }
64
+
65
+ function alignByteLength(byteLength) {
66
+ return Math.ceil(byteLength / 4) * 4;
67
+ }
68
+
69
+ function estimateEmbeddingTensorBytes(location) {
70
+ if (!location?.shape || location.shape.length !== 2) {
71
+ return null;
72
+ }
73
+ const [rows, hidden] = location.shape;
74
+ if (!Number.isFinite(rows) || rows <= 0 || !Number.isFinite(hidden) || hidden <= 0) {
75
+ return null;
76
+ }
77
+ const dtype = getEmbeddingFloatDtype(location);
78
+ const bytesPerElement = DTYPE_SIZES[dtype];
79
+ if (!Number.isFinite(bytesPerElement) || bytesPerElement <= 0) {
80
+ return null;
81
+ }
82
+ const tensorSizeBytes = rows * hidden * bytesPerElement;
83
+ if (!Number.isFinite(tensorSizeBytes) || tensorSizeBytes <= 0) {
84
+ return null;
85
+ }
86
+ return {
87
+ dtype,
88
+ rows,
89
+ hidden,
90
+ bytesPerElement,
91
+ tensorSizeBytes,
92
+ alignedTensorSizeBytes: alignByteLength(tensorSizeBytes),
93
+ };
94
+ }
95
+
96
+ export function createGpuResidentEmbeddingLimitError({ name, location, embeddingKernel = null }) {
97
+ const estimate = estimateEmbeddingTensorBytes(location);
98
+ if (!estimate) {
99
+ return null;
100
+ }
101
+ const device = getDevice();
102
+ const maxStorageBufferBindingSize = device?.limits?.maxStorageBufferBindingSize;
103
+ const maxBufferSize = device?.limits?.maxBufferSize;
104
+ const maxStorageBuffersPerShaderStage = device?.limits?.maxStorageBuffersPerShaderStage;
105
+ if (
106
+ !Number.isFinite(maxStorageBufferBindingSize)
107
+ || maxStorageBufferBindingSize <= 0
108
+ || !Number.isFinite(maxBufferSize)
109
+ || maxBufferSize <= 0
110
+ ) {
111
+ return null;
112
+ }
113
+ const maxGpuResidentBytes = Math.min(maxStorageBufferBindingSize, maxBufferSize);
114
+ if (estimate.alignedTensorSizeBytes <= maxGpuResidentBytes) {
115
+ return null;
116
+ }
117
+
118
+ const rowBytes = estimate.hidden * estimate.bytesPerElement;
119
+ const largeWeightMaxBytes = getLargeWeightMaxBytes();
120
+ const rowsPerSplitSection = largeWeightMaxBytes
121
+ ? Math.floor(largeWeightMaxBytes / rowBytes)
122
+ : null;
123
+ const requiredSplitSections = rowsPerSplitSection && rowsPerSplitSection > 0
124
+ ? Math.ceil(estimate.rows / rowsPerSplitSection)
125
+ : null;
126
+ const splitKernelExpected = expectsSplitGpuEmbeddingKernel(embeddingKernel);
127
+ const activeSplitKernelMaxSections = getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) || null;
128
+ const maxSplitEmbeddingSections = getMaxSplitGpuEmbeddingSectionsForDevice(embeddingKernel, device);
129
+ if (
130
+ splitKernelExpected
131
+ && Number.isFinite(requiredSplitSections)
132
+ && requiredSplitSections <= maxSplitEmbeddingSections
133
+ ) {
134
+ return null;
135
+ }
136
+ const error = new Error(
137
+ `[Loader] Embedding "${name}" cannot be GPU-resident on this device: ` +
138
+ `requires ${estimate.alignedTensorSizeBytes} bytes but max storage binding is ` +
139
+ `${maxGpuResidentBytes} bytes. Configure a split embedding kernel that fits this device ` +
140
+ 'or use a device with a larger storage binding limit.'
141
+ );
142
+ error.details = {
143
+ weightLoadFailure: {
144
+ tensorName: name,
145
+ tensorRole: location.role ?? null,
146
+ tensorDtype: location.dtype ?? null,
147
+ tensorShape: Array.isArray(location.shape) ? [...location.shape] : null,
148
+ tensorSizeBytes: estimate.alignedTensorSizeBytes,
149
+ tensorLoadStage: 'gpuResidentEmbeddingLimitPreflight',
150
+ toGPU: true,
151
+ streamedUpload: false,
152
+ deviceLimitFailure: {
153
+ kind: 'gpu_resident_embedding_exceeds_device_limit',
154
+ maxGpuResidentBytes,
155
+ maxStorageBufferBindingSize,
156
+ maxBufferSize,
157
+ maxStorageBuffersPerShaderStage: Number.isFinite(maxStorageBuffersPerShaderStage)
158
+ ? maxStorageBuffersPerShaderStage
159
+ : null,
160
+ largeWeightMaxBytes,
161
+ embeddingKernel: embeddingKernel
162
+ ? {
163
+ kernel: embeddingKernel.kernel ?? null,
164
+ entry: embeddingKernel.entry ?? null,
165
+ }
166
+ : null,
167
+ splitKernelExpected,
168
+ activeSplitKernelMaxSections,
169
+ maxSplitEmbeddingSections,
170
+ requiredSplitSections,
171
+ },
172
+ },
173
+ };
174
+ return error;
175
+ }
176
+
177
+ function resolveLargeWeightGpuResidentOverrides(manifestOverrides, runtimeConfig) {
178
+ const runtimeOverrides = runtimeConfig?.inference?.largeWeights?.gpuResidentOverrides;
179
+ if (Array.isArray(runtimeOverrides)) {
180
+ return runtimeOverrides;
181
+ }
182
+ if (Array.isArray(manifestOverrides)) {
183
+ return manifestOverrides;
184
+ }
185
+ return null;
186
+ }
187
+
188
+ export function resolveManifestGpuResidentEmbeddingLimitError(manifest, options = {}) {
189
+ const tensorLocations = options.storageManifest?.tensors ?? manifest?.tensors;
190
+ if (!tensorLocations || typeof tensorLocations !== 'object') {
191
+ return null;
192
+ }
193
+ const overrides = resolveLargeWeightGpuResidentOverrides(
194
+ manifest?.inference?.largeWeights?.gpuResidentOverrides,
195
+ options.runtimeConfig
196
+ );
197
+ if (!Array.isArray(overrides) || overrides.length === 0) {
198
+ return null;
199
+ }
200
+ const embeddingKernel = manifest?.inference?.execution?.kernels?.embed ?? null;
201
+ for (const name of overrides) {
202
+ const location = tensorLocations[name];
203
+ if (location?.role !== 'embedding') {
204
+ continue;
205
+ }
206
+ const error = createGpuResidentEmbeddingLimitError({
207
+ name,
208
+ location,
209
+ embeddingKernel,
210
+ });
211
+ if (error) {
212
+ return error;
213
+ }
214
+ }
215
+ return null;
216
+ }
@@ -14,6 +14,7 @@ import type {
14
14
  WeightBuffer,
15
15
  WeightLayout,
16
16
  CpuWeightBuffer,
17
+ SplitWeightBuffer,
17
18
  } from '../gpu/weight-buffer.js';
18
19
  import type { TensorLocation } from './loader-types.js';
19
20
 
@@ -22,7 +23,7 @@ export type TensorLoader = (
22
23
  name: string,
23
24
  toGPU?: boolean,
24
25
  silent?: boolean
25
- ) => Promise<GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | Uint8Array | null>;
26
+ ) => Promise<GPUBuffer | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer | Float32Array | Uint8Array | null>;
26
27
 
27
28
  /**
28
29
  * Context required for embedding loading.
@@ -46,10 +47,15 @@ export interface EmbeddingLoaderContext {
46
47
  preserveF32Embeddings?: boolean;
47
48
  /** Host shader-f16 capability, used to choose CPU F16->F32 fallback on no-f16 devices */
48
49
  hostHasShaderF16?: boolean | null;
50
+ /** Manifest-declared embedding kernel identity. */
51
+ embeddingKernel?: {
52
+ kernel?: string;
53
+ entry?: string;
54
+ } | null;
49
55
  }
50
56
 
51
57
  /** Result of embedding loading */
52
- export type EmbeddingResult = GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
58
+ export type EmbeddingResult = GPUBuffer | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer | Float32Array | null;
53
59
 
54
60
  /**
55
61
  * Load embedding weights.
@@ -3,20 +3,32 @@
3
3
  import {
4
4
  createWeightBuffer,
5
5
  createCpuWeightBuffer,
6
+ createSplitWeightBuffer,
6
7
  isWeightBuffer,
7
8
  isCpuWeightBuffer,
9
+ isSplitWeightBuffer,
8
10
  getWeightDtype,
9
11
  getLayout,
10
12
  } from '../gpu/weight-buffer.js';
13
+ import { getDevice } from '../gpu/device.js';
11
14
  import { maybeDowncastToF16 } from './weight-downcast.js';
12
15
  import { getTensorNamesByRole } from './tensors/tensor-role.js';
13
16
  import { log } from '../debug/index.js';
14
17
  import { selectRuleValue } from '../rules/rule-registry.js';
18
+ import { DTYPE_SIZES } from '../config/schema/index.js';
15
19
  import { createTensor } from '../gpu/tensor.js';
16
20
  import { castF16ToF32 } from '../gpu/kernel-selector.js';
17
21
  import { releaseBuffer } from '../memory/buffer-pool.js';
18
- import { loadTensorRange } from './tensors/tensor-reader.js';
22
+ import { assembleShardData, loadTensorRange } from './tensors/tensor-reader.js';
19
23
  import { hasSourceTransform } from './tensors/source-transform.js';
24
+ import { getLargeWeightMaxBytes } from './manifest-config.js';
25
+ import {
26
+ createGpuResidentEmbeddingLimitError,
27
+ expectsSplitGpuEmbeddingKernel,
28
+ getEmbeddingFloatDtype,
29
+ getSplitGpuEmbeddingKernelSectionCount,
30
+ getSplitGpuEmbeddingRequiredStorageBuffers,
31
+ } from './embedding-limit-preflight.js';
20
32
 
21
33
  // ============================================================================
22
34
  // Constants
@@ -25,6 +37,7 @@ import { hasSourceTransform } from './tensors/source-transform.js';
25
37
 
26
38
  const EMBEDDING_ROLE = 'embedding';
27
39
  const EMBEDDING_GROUP = 'embed';
40
+ const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
28
41
 
29
42
  function isPerLayerEmbeddingTensor(name) {
30
43
  return String(name ?? '').toLowerCase().includes('embed_tokens_per_layer.weight');
@@ -34,6 +47,17 @@ function isGpuBufferInstance(value) {
34
47
  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
35
48
  }
36
49
 
50
+ function normalizeLiteRTInt4StorageEncoding(transform, name) {
51
+ const storageEncoding = String(transform?.storageEncoding ?? '').toLowerCase();
52
+ if (storageEncoding !== 'signed' && storageEncoding !== 'offset_binary') {
53
+ throw new Error(
54
+ `[Loader] Embedding "${name}" LiteRT INT4 sourceTransform.storageEncoding must be ` +
55
+ `"signed" or "offset_binary", got "${transform?.storageEncoding ?? 'missing'}".`
56
+ );
57
+ }
58
+ return storageEncoding;
59
+ }
60
+
37
61
  function createRangeBackedTensorSource(ctx, name, location) {
38
62
  if (typeof ctx.loadShardRange !== 'function') {
39
63
  return null;
@@ -62,22 +86,245 @@ function createRangeBackedWeightBuffer(ctx, name, location) {
62
86
  return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
63
87
  }
64
88
 
89
+ function isFixedLiteRTInt4AffineEmbedding(ctx, name, location) {
90
+ if (ctx.hostHasShaderF16 === false) {
91
+ return false;
92
+ }
93
+ if (!location?.shape || location.shape.length !== 2) {
94
+ return false;
95
+ }
96
+ if (location.role !== EMBEDDING_ROLE) {
97
+ return false;
98
+ }
99
+ if (ctx.resolveWeightLayout(location) !== 'row') {
100
+ return false;
101
+ }
102
+ const transform = location.sourceTransform;
103
+ if (!transform || transform.kind !== 'affine_dequant') {
104
+ return false;
105
+ }
106
+ if (transform.scheme !== 'per_tensor_affine') {
107
+ return false;
108
+ }
109
+ if (String(transform.sourceDtype ?? '').toUpperCase() !== 'INT4') {
110
+ return false;
111
+ }
112
+ if (String(transform.targetDtype ?? '').toUpperCase() !== 'F16') {
113
+ return false;
114
+ }
115
+ normalizeLiteRTInt4StorageEncoding(transform, name);
116
+ if (Number(transform.scale) !== 0.0625 || Number(transform.zeroPoint) !== 0) {
117
+ return false;
118
+ }
119
+ const [, hiddenSize] = location.shape;
120
+ const expectedBytes = location.shape[0] * Math.ceil(hiddenSize / 2);
121
+ if (location.size !== expectedBytes) {
122
+ throw new Error(
123
+ `[Loader] Embedding "${name}" LiteRT INT4 source size mismatch: ` +
124
+ `manifest size=${location.size}, expected=${expectedBytes}.`
125
+ );
126
+ }
127
+ return typeof ctx.loadShardRange === 'function';
128
+ }
129
+
130
+ async function createLiteRTInt4GpuEmbeddingWeightBuffer(ctx, name, location) {
131
+ if (!isFixedLiteRTInt4AffineEmbedding(ctx, name, location)) {
132
+ return null;
133
+ }
134
+ const device = getDevice();
135
+ if (!device) {
136
+ return null;
137
+ }
138
+ const rawBytes = await assembleShardData(location, name, null, ctx.loadShardRange, {
139
+ materializeSourceTransform: false,
140
+ });
141
+ if (rawBytes.byteLength !== location.size) {
142
+ throw new Error(
143
+ `[Loader] Embedding "${name}" LiteRT INT4 raw read ${rawBytes.byteLength} bytes, ` +
144
+ `expected ${location.size}.`
145
+ );
146
+ }
147
+ const buffer = device.createBuffer({
148
+ label: `${name}:litert_int4`,
149
+ size: alignByteLength(rawBytes.byteLength),
150
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
151
+ });
152
+ try {
153
+ writeBufferInChunks(device.queue, buffer, rawBytes);
154
+ } catch (error) {
155
+ buffer.destroy();
156
+ throw error;
157
+ }
158
+ ctx.gpuBuffers.add(buffer);
159
+ log.info(
160
+ 'Loader',
161
+ `Embedding "${name}" stored as packed LiteRT INT4 GPU source ` +
162
+ `(bytes=${rawBytes.byteLength}, layout=row).`
163
+ );
164
+ return createWeightBuffer(buffer, 'litert_int4', 'row', location.shape, name, null, {
165
+ storageEncoding: normalizeLiteRTInt4StorageEncoding(location.sourceTransform, name),
166
+ });
167
+ }
168
+
169
+ function alignByteLength(byteLength) {
170
+ return Math.ceil(byteLength / 4) * 4;
171
+ }
172
+
173
+ function writeBufferInChunks(queue, buffer, bytes) {
174
+ for (let offset = 0; offset < bytes.byteLength; offset += SPLIT_UPLOAD_CHUNK_BYTES) {
175
+ const end = Math.min(offset + SPLIT_UPLOAD_CHUNK_BYTES, bytes.byteLength);
176
+ queue.writeBuffer(buffer, offset, bytes, offset, end - offset);
177
+ }
178
+ }
179
+
180
+ function expectsSplitGpuEmbedding(ctx) {
181
+ return expectsSplitGpuEmbeddingKernel(ctx.embeddingKernel);
182
+ }
183
+
184
+ async function createSplitGpuEmbeddingWeightBuffer(ctx, name, location) {
185
+ if (!expectsSplitGpuEmbedding(ctx)) {
186
+ return null;
187
+ }
188
+ if (ctx.hostHasShaderF16 === false) {
189
+ return null;
190
+ }
191
+ if (!location?.shape || location.shape.length !== 2) {
192
+ return null;
193
+ }
194
+ if (hasSourceTransform(location)) {
195
+ return null;
196
+ }
197
+ if (typeof ctx.loadShardRange !== 'function') {
198
+ return null;
199
+ }
200
+
201
+ const layout = ctx.resolveWeightLayout(location);
202
+ if (layout !== 'row') {
203
+ return null;
204
+ }
205
+
206
+ const dtype = getEmbeddingFloatDtype(location);
207
+ if (dtype !== 'f16') {
208
+ return null;
209
+ }
210
+
211
+ const device = getDevice();
212
+ const maxBytes = getLargeWeightMaxBytes();
213
+ if (!device || !maxBytes) {
214
+ return null;
215
+ }
216
+ const maxSplitSections = getSplitGpuEmbeddingKernelSectionCount(ctx.embeddingKernel);
217
+ if (maxSplitSections <= 0) {
218
+ return null;
219
+ }
220
+ const requiredStorageBuffers = getSplitGpuEmbeddingRequiredStorageBuffers(maxSplitSections);
221
+ if ((device.limits.maxStorageBuffersPerShaderStage ?? 0) < requiredStorageBuffers) {
222
+ log.warn(
223
+ 'Loader',
224
+ `Embedding "${name}" cannot use split GPU residency: ` +
225
+ `maxStorageBuffersPerShaderStage is below ${requiredStorageBuffers}.`
226
+ );
227
+ return null;
228
+ }
229
+
230
+ const [rows, hidden] = location.shape;
231
+ const bytesPerElement = DTYPE_SIZES[dtype];
232
+ const rowBytes = hidden * bytesPerElement;
233
+ if (!Number.isFinite(rowBytes) || rowBytes <= 0 || rowBytes > maxBytes) {
234
+ return null;
235
+ }
236
+
237
+ const totalBytes = rows * rowBytes;
238
+ if (totalBytes <= maxBytes) {
239
+ return null;
240
+ }
241
+
242
+ const rowsPerSection = Math.floor(maxBytes / rowBytes);
243
+ if (rowsPerSection <= 0) {
244
+ return null;
245
+ }
246
+ const sectionCount = Math.ceil(rows / rowsPerSection);
247
+ if (sectionCount > maxSplitSections) {
248
+ log.warn(
249
+ 'Loader',
250
+ `Embedding "${name}" needs ${sectionCount} GPU sections; active split kernel supports ` +
251
+ `at most ${maxSplitSections}. ` +
252
+ 'Using CPU-backed streaming.'
253
+ );
254
+ return null;
255
+ }
256
+
257
+ const createdBuffers = [];
258
+ try {
259
+ const sections = [];
260
+ for (let rowStart = 0; rowStart < rows; rowStart += rowsPerSection) {
261
+ const rowCount = Math.min(rowsPerSection, rows - rowStart);
262
+ const byteOffset = rowStart * rowBytes;
263
+ const byteLength = rowCount * rowBytes;
264
+ const bytes = await loadTensorRange(location, name, byteOffset, byteLength, ctx.loadShardRange);
265
+ if (bytes.byteLength !== byteLength) {
266
+ throw new Error(
267
+ `[Loader] Embedding "${name}" split section read ${bytes.byteLength} bytes, expected ${byteLength}.`
268
+ );
269
+ }
270
+ const buffer = device.createBuffer({
271
+ label: `${name}:split:${sections.length}`,
272
+ size: alignByteLength(byteLength),
273
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
274
+ });
275
+ createdBuffers.push(buffer);
276
+ writeBufferInChunks(device.queue, buffer, bytes);
277
+ sections.push({ buffer, rowStart, rowCount });
278
+ }
279
+ for (const buffer of createdBuffers) {
280
+ ctx.gpuBuffers.add(buffer);
281
+ }
282
+ log.warn(
283
+ 'Loader',
284
+ `Embedding "${name}" stored as split GPU sections (${sections.length} sections, dtype=${dtype}, layout=${layout}).`
285
+ );
286
+ return createSplitWeightBuffer(sections, dtype, layout, location.shape, name, {
287
+ splitGatherSectionCount: maxSplitSections,
288
+ sourceKernel: ctx.embeddingKernel,
289
+ });
290
+ } catch (error) {
291
+ for (const buffer of createdBuffers) {
292
+ try {
293
+ buffer.destroy();
294
+ } catch {
295
+ // Ignore cleanup failure while preserving the original load error.
296
+ }
297
+ }
298
+ throw error;
299
+ }
300
+ }
301
+
65
302
  function shouldUseRangeBackedEmbeddingSource(ctx, name, location) {
66
303
  if (!location) {
67
304
  return false;
68
305
  }
69
306
  if (String(location.dtype ?? '').toUpperCase() === 'F16' && ctx.hostHasShaderF16 === false) {
307
+ log.info(
308
+ 'Loader',
309
+ `Embedding "${name}" range-backed: F16 source on device without shader-f16 support.`
310
+ );
70
311
  return true;
71
312
  }
72
313
  if (hasSourceTransform(location) && typeof ctx.loadShardRange === 'function') {
73
314
  log.info(
74
315
  'Loader',
75
- `Embedding "${name}" uses sourceTransform.kind="${location.sourceTransform.kind}". ` +
76
- 'Using range-backed CPU source to avoid eager full materialization.'
316
+ `Embedding "${name}" range-backed: sourceTransform.kind="${location.sourceTransform.kind}" defers full materialization.`
77
317
  );
78
318
  return true;
79
319
  }
80
- return ctx.shouldStreamLargeWeight(name, location, 'Embedding');
320
+ const stream = ctx.shouldStreamLargeWeight(name, location, 'Embedding');
321
+ if (!stream) {
322
+ log.info(
323
+ 'Loader',
324
+ `Embedding "${name}" GPU-resident: shouldStreamLargeWeight returned false.`
325
+ );
326
+ }
327
+ return stream;
81
328
  }
82
329
 
83
330
  // ============================================================================
@@ -102,29 +349,42 @@ export async function loadEmbeddings(ctx) {
102
349
 
103
350
  for (const name of candidates) {
104
351
  const loc = ctx.tensorLocations.get(name);
105
- const shouldStream = shouldUseRangeBackedEmbeddingSource(ctx, name, loc);
352
+ const packedLiteRTTensor = await createLiteRTInt4GpuEmbeddingWeightBuffer(ctx, name, loc);
353
+ const shouldStream = packedLiteRTTensor ? false : shouldUseRangeBackedEmbeddingSource(ctx, name, loc);
354
+ const splitGpuTensor = packedLiteRTTensor ? null : await createSplitGpuEmbeddingWeightBuffer(ctx, name, loc);
355
+ if (!packedLiteRTTensor && !splitGpuTensor && !shouldStream) {
356
+ const limitError = createGpuResidentEmbeddingLimitError({
357
+ name,
358
+ location: loc,
359
+ embeddingKernel: ctx.embeddingKernel,
360
+ });
361
+ if (limitError) {
362
+ throw limitError;
363
+ }
364
+ }
106
365
 
107
366
  // Load tensor (to CPU if streaming, to GPU otherwise)
108
- const tensor = shouldStream
367
+ const tensor = packedLiteRTTensor ?? splitGpuTensor ?? (shouldStream
109
368
  ? (
110
369
  createRangeBackedWeightBuffer(ctx, name, loc)
111
370
  ?? await ctx.loadTensor(name, false, true)
112
371
  )
113
- : await ctx.loadTensor(name, true, true);
372
+ : await ctx.loadTensor(name, true, true));
373
+ const tensorShouldStream = splitGpuTensor ? false : shouldStream;
114
374
 
115
375
  // Skip if not found
116
376
  if (!tensor) continue;
117
377
 
118
378
  // Handle streaming path (CPU)
119
- if (shouldStream && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
379
+ if (tensorShouldStream && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
120
380
  throw new Error(
121
381
  `[Loader] Embedding "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc?.dtype ?? 'unknown'}).`
122
382
  );
123
383
  }
124
384
 
125
385
  // Handle valid tensor types
126
- if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || tensor instanceof Float32Array) {
127
- const result = await processEmbeddingTensor(ctx, tensor, name, loc, shouldStream);
386
+ if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || isSplitWeightBuffer(tensor) || tensor instanceof Float32Array) {
387
+ const result = await processEmbeddingTensor(ctx, tensor, name, loc, tensorShouldStream);
128
388
  if (result) {
129
389
  return result;
130
390
  }
@@ -153,6 +413,10 @@ async function processEmbeddingTensor(ctx, tensor, name, loc, shouldStream) {
153
413
  // but loaded with an F32 gather kernel.
154
414
  const promoted = await maybePromoteEmbeddingsToF32(ctx, tensor, name, loc);
155
415
 
416
+ if (isSplitWeightBuffer(promoted)) {
417
+ return promoted;
418
+ }
419
+
156
420
  // WeightBuffer already has layout set correctly from _loadTensor
157
421
  if (isWeightBuffer(promoted)) {
158
422
  return maybeDowncastEmbeddings(ctx, promoted, name, loc);
@@ -195,6 +459,7 @@ async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
195
459
  if (!ctx.preserveF32Embeddings) return current;
196
460
  if (current instanceof Float32Array) return current;
197
461
  if (isCpuWeightBuffer(current)) return current;
462
+ if (isSplitWeightBuffer(current)) return current;
198
463
 
199
464
  if (isWeightBuffer(current)) {
200
465
  const dtype = getWeightDtype(current);
@@ -234,7 +499,7 @@ async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
234
499
 
235
500
  async function maybeDowncastEmbeddings(ctx, current, name, loc) {
236
501
  // Can't downcast Float32Array or CpuWeightBuffer
237
- if (current instanceof Float32Array || isCpuWeightBuffer(current)) {
502
+ if (current instanceof Float32Array || isCpuWeightBuffer(current) || isSplitWeightBuffer(current)) {
238
503
  return current;
239
504
  }
240
505
 
@@ -10,6 +10,7 @@
10
10
 
11
11
  import type { RDRRManifest } from '../../formats/rdrr/index.js';
12
12
  import type { ExpertCache } from './expert-cache.js';
13
+ import type { TensorLocation } from '../loader-types.js';
13
14
  import type { ExpertWeights } from '../weights.js';
14
15
  import type { WeightBuffer } from '../../gpu/weight-buffer.js';
15
16
  import type { ShardLoadOptions } from '../loader-types.js';
@@ -38,6 +39,8 @@ export interface ShardCacheInterface {
38
39
  export interface ExpertLoaderContext {
39
40
  /** Model manifest */
40
41
  manifest: RDRRManifest | null;
42
+ /** Resolved tensor locations from manifest.tensors or tensors.json */
43
+ tensorLocations?: Map<string, TensorLocation>;
41
44
  /** Load a tensor by name */
42
45
  loadTensor: TensorLoader;
43
46
  /** Load a shard by index */
@@ -46,7 +49,7 @@ export interface ExpertLoaderContext {
46
49
  shardCache: ShardCacheInterface;
47
50
  /** Expert LRU cache */
48
51
  expertCache: ExpertCache | null;
49
- /** Simple map for GPT-OSS packed experts */
52
+ /** Simple map for packed experts */
50
53
  experts: Map<string, ExpertWeights>;
51
54
  /** GPU buffers to track for cleanup */
52
55
  gpuBuffers: Set<GPUBuffer>;