doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -8,6 +8,8 @@ import {
8
8
  openModelStore,
9
9
  verifyIntegrity,
10
10
  loadManifestFromStore,
11
+ loadAuxFile,
12
+ computeHash,
11
13
  } from '../storage/shard-manager.js';
12
14
  import { clearManifest, parseManifest, setManifest as setCurrentManifest } from '../formats/rdrr/index.js';
13
15
  import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
@@ -39,12 +41,15 @@ import { MemoryMonitor } from './memory-monitor.js';
39
41
  import {
40
42
  loadTensorToGPU,
41
43
  loadTensorToCPU,
44
+ isLiteRTAffineInt4FusedEligible,
42
45
  } from './tensors/tensor-loader.js';
43
46
  import { annotateWeightLoadError } from '../inference/pipelines/text/load-errors.js';
44
47
  import { loadEmbeddings } from './embedding-loader.js';
45
48
  import { loadPerLayerInputWeights } from './per-layer-input-loader.js';
46
49
  import { loadLayer } from './layer-loader.js';
47
50
  import { loadFinalWeights } from './final-weights-loader.js';
51
+ import { cloneJsonValue } from '../utils/clone-json.js';
52
+ import { assertFunctionalDescriptorManifest } from '../formats/rdrr/functional-descriptor.js';
48
53
  import {
49
54
  loadExpert as loadExpertFromModule,
50
55
  prefetchExperts as prefetchExpertsFromModule,
@@ -96,6 +101,71 @@ function isGpuBufferInstance(value) {
96
101
  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
97
102
  }
98
103
 
104
+ function nowMs() {
105
+ return typeof performance !== 'undefined' && typeof performance.now === 'function'
106
+ ? performance.now()
107
+ : Date.now();
108
+ }
109
+
110
+ function roundLoadTimingMs(value) {
111
+ return Number.isFinite(value) ? Number(value.toFixed(3)) : null;
112
+ }
113
+
114
+ function createLoadTiming(modelId, hasCustomLoader) {
115
+ return {
116
+ schemaVersion: 1,
117
+ source: 'doppler-loader',
118
+ modelId: typeof modelId === 'string' ? modelId : null,
119
+ status: 'running',
120
+ customShardLoader: hasCustomLoader === true,
121
+ byteAccountingMode: hasCustomLoader === true
122
+ ? 'custom-loader-progress-unavailable'
123
+ : 'full-shard-progress',
124
+ totalBytes: null,
125
+ totalShards: null,
126
+ bytesLoaded: 0,
127
+ shardsLoaded: 0,
128
+ bytesPerSecond: null,
129
+ phasesMs: {
130
+ preflight: null,
131
+ tensorLocations: null,
132
+ embeddings: null,
133
+ layers: null,
134
+ finalWeights: null,
135
+ cleanup: null,
136
+ },
137
+ layers: {
138
+ count: null,
139
+ totalMs: null,
140
+ meanMs: null,
141
+ maxMs: null,
142
+ maxLayer: null,
143
+ },
144
+ totalMs: null,
145
+ failedPhase: null,
146
+ error: null,
147
+ };
148
+ }
149
+
150
+ function finishLoadPhase(loadTiming, phase, startMs) {
151
+ if (!loadTiming?.phasesMs || !phase) return;
152
+ loadTiming.phasesMs[phase] = roundLoadTimingMs(nowMs() - startMs);
153
+ }
154
+
155
+ function finishLoadTiming(loadTiming, status, startMs, error = null, failedPhase = null) {
156
+ if (!loadTiming) return;
157
+ const totalMs = nowMs() - startMs;
158
+ loadTiming.status = status;
159
+ loadTiming.totalMs = roundLoadTimingMs(totalMs);
160
+ loadTiming.bytesPerSecond = totalMs > 0 && Number.isFinite(loadTiming.bytesLoaded)
161
+ ? Math.round((loadTiming.bytesLoaded / totalMs) * 1000)
162
+ : null;
163
+ if (status === 'failed') {
164
+ loadTiming.failedPhase = failedPhase;
165
+ loadTiming.error = error?.message ?? String(error ?? 'unknown load error');
166
+ }
167
+ }
168
+
99
169
  function getTensorShardIndices(location) {
100
170
  if (Array.isArray(location?.spans) && location.spans.length > 0) {
101
171
  return Array.from(new Set(location.spans.map((span) => span?.shardIndex).filter(Number.isInteger)));
@@ -158,6 +228,8 @@ export class DopplerLoader {
158
228
 
159
229
  embeddingPostprocessor = null;
160
230
 
231
+ diffusionGemmaSelfConditioning = null;
232
+
161
233
  perLayerInputWeights = null;
162
234
 
163
235
  // Memory management
@@ -176,6 +248,8 @@ export class DopplerLoader {
176
248
 
177
249
  tensorLocations = new Map();
178
250
 
251
+ loadTiming = null;
252
+
179
253
  // Shard cache (LRU with request deduplication)
180
254
 
181
255
  shardCache;
@@ -186,6 +260,7 @@ export class DopplerLoader {
186
260
  #loaderDebug = null;
187
261
 
188
262
  #perLayerInputSession = null;
263
+ #loadAuxiliaryFile = null;
189
264
 
190
265
  // Fused Q4_K matmul: skip dequantization for matmul weights, use fused kernel
191
266
 
@@ -315,6 +390,13 @@ export class DopplerLoader {
315
390
  loadRange: options.loadShardRange ?? null,
316
391
  streamRange: options.streamShardRange ?? null,
317
392
  });
393
+ this.#loadAuxiliaryFile = typeof options.loadAuxiliaryFile === 'function'
394
+ ? options.loadAuxiliaryFile
395
+ : null;
396
+ }
397
+
398
+ setAuxiliaryFileLoader(loadAuxiliaryFile) {
399
+ this.#loadAuxiliaryFile = typeof loadAuxiliaryFile === 'function' ? loadAuxiliaryFile : null;
318
400
  }
319
401
 
320
402
 
@@ -452,6 +534,10 @@ export class DopplerLoader {
452
534
 
453
535
  log.info('Loader', `Loading: ${modelId}`);
454
536
  this.modelId = modelId;
537
+ const loadTimingStart = nowMs();
538
+ let activeLoadPhase = 'preflight';
539
+ let phaseStart = loadTimingStart;
540
+ this.loadTiming = createLoadTiming(modelId, this.shardCache.hasCustomLoader);
455
541
 
456
542
  this.#startMemoryLogging();
457
543
  this.#assertResidentBudget('load start');
@@ -488,10 +574,17 @@ export class DopplerLoader {
488
574
  }
489
575
  }
490
576
 
491
- await this.#buildTensorLocations();
492
-
493
577
  const totalBytes = (this.manifest.shards || []).reduce((sum, s) => sum + (s.size || 0), 0);
494
578
  const totalShards = this.manifest.shards?.length || 0;
579
+ this.loadTiming.totalBytes = totalBytes;
580
+ this.loadTiming.totalShards = totalShards;
581
+ finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
582
+
583
+ activeLoadPhase = 'tensorLocations';
584
+ phaseStart = nowMs();
585
+ await this.#buildTensorLocations();
586
+ finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
587
+
495
588
  const loadStartTime = Date.now();
496
589
  let bytesLoaded = 0;
497
590
  let shardsLoaded = 0;
@@ -549,6 +642,8 @@ export class DopplerLoader {
549
642
  loadedShardIndices.add(shardIndex);
550
643
  bytesLoaded += shardSize;
551
644
  shardsLoaded++;
645
+ this.loadTiming.bytesLoaded = bytesLoaded;
646
+ this.loadTiming.shardsLoaded = shardsLoaded;
552
647
  if (!inLayerPhase) {
553
648
  const pct = 0.1 + Math.min(bytesLoaded / totalBytes, 1.0) * 0.7;
554
649
  const elapsed = (Date.now() - loadStartTime) / 1000;
@@ -582,7 +677,10 @@ export class DopplerLoader {
582
677
  let loadError = null;
583
678
  try {
584
679
  reportProgress('shards', 0.1, 'Loading embeddings...');
680
+ activeLoadPhase = 'embeddings';
681
+ phaseStart = nowMs();
585
682
  await this.#loadEmbeddings(onProgress);
683
+ finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
586
684
  this.#assertResidentBudget('embeddings');
587
685
 
588
686
  const resolveNumLayers = (value) => {
@@ -616,14 +714,24 @@ export class DopplerLoader {
616
714
  log.info('Loader', `Layers: 0-${numLayers - 1}`);
617
715
 
618
716
  inLayerPhase = true;
717
+ activeLoadPhase = 'layers';
619
718
  const layersStartTime = performance.now();
719
+ let layerTotalMs = 0;
720
+ let maxLayerMs = 0;
721
+ let maxLayer = null;
620
722
 
621
723
  for (let l = 0; l < numLayers; l++) {
622
724
  const layerStart = performance.now();
623
725
  const layerPromise = this.#loadLayer(l, onProgress);
624
726
  this.#prefetchLayerShards(l);
625
727
  await layerPromise;
626
- const layerElapsed = ((performance.now() - layerStart) / 1000).toFixed(2);
728
+ const layerElapsedMs = performance.now() - layerStart;
729
+ layerTotalMs += layerElapsedMs;
730
+ if (layerElapsedMs > maxLayerMs) {
731
+ maxLayerMs = layerElapsedMs;
732
+ maxLayer = l;
733
+ }
734
+ const layerElapsed = (layerElapsedMs / 1000).toFixed(2);
627
735
  log.verbose('Loader', ` Layer ${l}: ${layerElapsed}s`);
628
736
 
629
737
  await new Promise(r => setTimeout(r, 0));
@@ -661,10 +769,21 @@ export class DopplerLoader {
661
769
  }
662
770
 
663
771
  const layersTotalTime = ((performance.now() - layersStartTime) / 1000).toFixed(2);
772
+ this.loadTiming.layers = {
773
+ count: numLayers,
774
+ totalMs: roundLoadTimingMs(layerTotalMs),
775
+ meanMs: numLayers > 0 ? roundLoadTimingMs(layerTotalMs / numLayers) : null,
776
+ maxMs: maxLayer == null ? null : roundLoadTimingMs(maxLayerMs),
777
+ maxLayer,
778
+ };
779
+ finishLoadPhase(this.loadTiming, activeLoadPhase, layersStartTime);
664
780
  log.info('Loader', `Layers: ${numLayers} complete (${layersTotalTime}s)`);
665
781
 
666
782
  reportProgress('gpu_transfer', 0.85, 'Loading final weights...');
783
+ activeLoadPhase = 'finalWeights';
784
+ phaseStart = nowMs();
667
785
  await this.#loadFinalWeights(onProgress);
786
+ finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
668
787
  this.#assertResidentBudget('final weights');
669
788
 
670
789
  if (onProgress) {
@@ -676,11 +795,16 @@ export class DopplerLoader {
676
795
  const avgSpeed = formatBytes(bytesLoaded / (Date.now() - loadStartTime) * 1000);
677
796
  log.info('Loader', `Complete: ${formatBytes(bytesLoaded)} in ${totalTime}s (${avgSpeed}/s)`);
678
797
 
798
+ activeLoadPhase = 'cleanup';
799
+ phaseStart = nowMs();
679
800
  this.shardCache.clear();
801
+ finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
802
+ finishLoadTiming(this.loadTiming, 'complete', loadTimingStart);
680
803
 
681
804
  return (this.manifest.config) || {};
682
805
  } catch (error) {
683
806
  loadError = error;
807
+ finishLoadTiming(this.loadTiming, 'failed', loadTimingStart, error, activeLoadPhase);
684
808
  } finally {
685
809
  this.#loadShardOverride = null;
686
810
  if (this.#memoryMonitor) {
@@ -734,12 +858,8 @@ export class DopplerLoader {
734
858
  this.#layerShardMap.set(layerIdx, shards);
735
859
  }
736
860
 
737
- if (location.spans) {
738
- for (const span of location.spans) {
739
- shards.add(span.shardIndex);
740
- }
741
- } else {
742
- shards.add(location.shardIndex);
861
+ for (const shardIndex of getTensorShardIndices(location)) {
862
+ shards.add(shardIndex);
743
863
  }
744
864
  }
745
865
  }
@@ -827,9 +947,16 @@ export class DopplerLoader {
827
947
  const streamedUpload = toGPU && this.#shouldStreamUploadToGPU(location);
828
948
  let shardData;
829
949
  try {
830
- shardData = streamedUpload
950
+ const preserveRawSourceBytes = toGPU && isLiteRTAffineInt4FusedEligible(location, {
951
+ gpuCapabilities: this.gpuCapabilities,
952
+ });
953
+ shardData = this.#isFunctionalDescriptorLocation(location)
954
+ ? await this.#assembleFunctionalDescriptorData(location, name)
955
+ : streamedUpload
831
956
  ? await this.#assembleShardDataToGpuBuffer(location, name)
832
- : await this.#assembleShardData(location, name);
957
+ : await this.#assembleShardData(location, name, {
958
+ materializeSourceTransform: !preserveRawSourceBytes,
959
+ });
833
960
  } catch (error) {
834
961
  throw annotateTensorLoadError(error, name, location, {
835
962
  tensorLoadStage: streamedUpload ? 'streamShardToGpuBuffer' : 'assembleShardData',
@@ -846,7 +973,7 @@ export class DopplerLoader {
846
973
  releaseBuffer(shardData);
847
974
  shardData = await this.#assembleShardData(location, name);
848
975
  }
849
- return loadTensorToCPU(shardData, location);
976
+ return loadTensorToCPU(shardData, location, name);
850
977
  }
851
978
 
852
979
 
@@ -891,19 +1018,145 @@ export class DopplerLoader {
891
1018
  releaseBuffer(shardData);
892
1019
  shardData = await this.#assembleShardData(location, name);
893
1020
  }
894
- return loadTensorToCPU(shardData, location);
1021
+ return loadTensorToCPU(shardData, location, name);
1022
+ }
1023
+
1024
+ #isFunctionalDescriptorLocation(location) {
1025
+ return String(location?.dtype || '').trim().toUpperCase() === 'FUNCTIONAL_DESCRIPTOR';
1026
+ }
1027
+
1028
+ #getDescriptorShardFiles(location, name) {
1029
+ if (!location?.descriptorManifest) {
1030
+ throw new Error(
1031
+ `[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" is missing descriptorManifest.`
1032
+ );
1033
+ }
1034
+ const manifest = assertFunctionalDescriptorManifest(
1035
+ location.descriptorManifest,
1036
+ `FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptorManifest`
1037
+ );
1038
+ const components = manifest.components;
1039
+ if (!components || typeof components !== 'object') {
1040
+ throw new Error(
1041
+ `[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptorManifest.components is required.`
1042
+ );
1043
+ }
1044
+ const files = [
1045
+ components.kronecker_sum?.shard_file,
1046
+ components.coordinate_inr?.shard_file,
1047
+ components.sparse_outliers?.shard_file,
1048
+ ];
1049
+ if (files.some((file) => typeof file !== 'string' || file.trim().length === 0)) {
1050
+ throw new Error(
1051
+ `[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" must declare kronecker, SIREN, and sparse shard_file values.`
1052
+ );
1053
+ }
1054
+ return files.map((file) => file.trim());
1055
+ }
1056
+
1057
+ async #loadDescriptorShardFile(file, name) {
1058
+ const loadAuxiliaryFile = this.#loadAuxiliaryFile;
1059
+ const payload = loadAuxiliaryFile
1060
+ ? await loadAuxiliaryFile(file)
1061
+ : await loadAuxFile(file);
1062
+ if (payload == null) {
1063
+ throw new Error(
1064
+ `[DopplerLoader] Descriptor shard "${file}" for tensor "${name}" was not found.`
1065
+ );
1066
+ }
1067
+ if (payload instanceof Uint8Array) {
1068
+ return payload;
1069
+ }
1070
+ if (ArrayBuffer.isView(payload)) {
1071
+ return new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength);
1072
+ }
1073
+ if (payload instanceof ArrayBuffer) {
1074
+ return new Uint8Array(payload);
1075
+ }
1076
+ throw new Error(
1077
+ `[DopplerLoader] Descriptor shard "${file}" for tensor "${name}" must load as ArrayBuffer or Uint8Array.`
1078
+ );
1079
+ }
1080
+
1081
+ async #assertDescriptorHash(location, name, descriptorShards) {
1082
+ const descriptorHash = location?.descriptorManifest?.descriptor_hash;
1083
+ if (typeof descriptorHash !== 'string' || !descriptorHash.trim()) {
1084
+ return;
1085
+ }
1086
+ const match = /^sha256:([a-f0-9]{64})$/i.exec(descriptorHash.trim());
1087
+ if (!match) {
1088
+ throw new Error(
1089
+ `[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptor_hash must be sha256:<64 hex chars>.`
1090
+ );
1091
+ }
1092
+ const totalBytes = Array.from(descriptorShards.values()).reduce((sum, bytes) => sum + bytes.byteLength, 0);
1093
+ const combined = new Uint8Array(totalBytes);
1094
+ let offset = 0;
1095
+ for (const bytes of descriptorShards.values()) {
1096
+ combined.set(bytes, offset);
1097
+ offset += bytes.byteLength;
1098
+ }
1099
+ const actual = await computeHash(combined, 'sha256');
1100
+ if (actual.toLowerCase() !== match[1].toLowerCase()) {
1101
+ throw new Error(
1102
+ `[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptor hash mismatch. ` +
1103
+ `Expected ${descriptorHash}, got sha256:${actual}.`
1104
+ );
1105
+ }
1106
+ }
1107
+
1108
+ async #assembleFunctionalDescriptorData(location, name) {
1109
+ const shardFiles = this.#getDescriptorShardFiles(location, name);
1110
+ const descriptorShards = new Map();
1111
+ for (const file of shardFiles) {
1112
+ descriptorShards.set(file, await this.#loadDescriptorShardFile(file, name));
1113
+ }
1114
+ await this.#assertDescriptorHash(location, name, descriptorShards);
1115
+ const data = new Uint8Array(0);
1116
+ Object.defineProperty(data, 'descriptorShards', {
1117
+ value: descriptorShards,
1118
+ enumerable: false,
1119
+ });
1120
+ return data;
895
1121
  }
896
1122
 
897
1123
 
898
- async #assembleShardData(location, name) {
1124
+ async #assembleShardData(location, name, options = {}) {
899
1125
  const loadShard = this.#getLoadShard();
900
1126
  const loadShardRange = (idx, offset, length) => this.shardCache.loadRange(idx, offset, length);
901
- return assembleShardData(location, name, loadShard, loadShardRange);
1127
+ const data = await assembleShardData(location, name, loadShard, loadShardRange, options);
1128
+ const companions = Array.isArray(location?.storage?.companions)
1129
+ ? location.storage.companions
1130
+ : [];
1131
+ if (companions.length === 0) {
1132
+ return data;
1133
+ }
1134
+ const storageCompanions = {};
1135
+ for (const companion of companions) {
1136
+ const companionLocation = this.tensorLocations.get(companion.tensorId);
1137
+ if (!companionLocation) {
1138
+ throw new Error(
1139
+ `[DopplerLoader] Tensor "${name}" storage companion "${companion.tensorId}" for role "${companion.role}" was not found.`
1140
+ );
1141
+ }
1142
+ storageCompanions[companion.role] = {
1143
+ tensorId: companion.tensorId,
1144
+ location: companionLocation,
1145
+ bytes: await assembleShardData(companionLocation, companion.tensorId, loadShard, loadShardRange),
1146
+ };
1147
+ }
1148
+ Object.defineProperty(data, 'storageCompanions', {
1149
+ value: storageCompanions,
1150
+ enumerable: false,
1151
+ });
1152
+ return data;
902
1153
  }
903
1154
 
904
1155
  #shouldStreamUploadToGPU(location) {
1156
+ if (this.#isFunctionalDescriptorLocation(location)) return false;
905
1157
  if (!location?.size || location.size <= 0) return false;
906
1158
  if (hasSourceTransform(location)) return false;
1159
+ if (Array.isArray(location?.storage?.companions) && location.storage.companions.length > 0) return false;
907
1160
  if (this.shardCache.hasCustomLoader && !this.shardCache.canStreamRanges) return false;
908
1161
  const chunkBytes = this.#loadingConfig?.storage?.backend?.streaming?.readChunkBytes ?? 0;
909
1162
  if (!Number.isFinite(chunkBytes) || chunkBytes <= 0) return false;
@@ -1015,6 +1268,7 @@ export class DopplerLoader {
1015
1268
  // gather.wgsl reads embeddings as f32; downcasting here corrupts reads.
1016
1269
  preserveF32Embeddings: String(this.manifest?.quantizationInfo?.embeddings ?? '').toLowerCase() === 'f32',
1017
1270
  hostHasShaderF16: this.gpuCapabilities?.hasF16 ?? null,
1271
+ embeddingKernel: this.manifest?.inference?.execution?.kernels?.embed ?? null,
1018
1272
  };
1019
1273
 
1020
1274
  this.embeddings = await loadEmbeddings(ctx);
@@ -1049,6 +1303,7 @@ export class DopplerLoader {
1049
1303
  keepF32Weights: this.keepF32Weights,
1050
1304
  isMoE: this.isMoE,
1051
1305
  isExpertLayer: (idx) => this.#isExpertLayer(idx),
1306
+ loadDenseFfnForMoeLayers: this.manifest?.inference?.ffn?.branchMode === 'dense_plus_moe',
1052
1307
  numHeads: this.manifest?.architecture?.numAttentionHeads ?? null,
1053
1308
  numKVHeads: this.manifest?.architecture?.numKeyValueHeads ?? null,
1054
1309
  headDim: this.manifest?.architecture?.headDim ?? null,
@@ -1088,6 +1343,7 @@ export class DopplerLoader {
1088
1343
  const loadShard = this.#getLoadShard();
1089
1344
  return {
1090
1345
  manifest: this.manifest,
1346
+ tensorLocations: this.tensorLocations,
1091
1347
  loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
1092
1348
  loadShard,
1093
1349
  shardCache: this.shardCache,
@@ -1119,6 +1375,7 @@ export class DopplerLoader {
1119
1375
  resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
1120
1376
  embeddings: this.embeddings,
1121
1377
  embeddingPostprocessor: this.manifest?.inference?.output?.embeddingPostprocessor ?? null,
1378
+ diffusionGemmaSelfConditioning: this.manifest?.inference?.diffusionGemma?.selfConditioning === true,
1122
1379
  modelType: this.manifest?.modelType ?? null,
1123
1380
  tieWordEmbeddings,
1124
1381
  gpuBuffers: this.gpuBuffers,
@@ -1130,6 +1387,7 @@ export class DopplerLoader {
1130
1387
  this.finalNorm = result.finalNorm;
1131
1388
  this.lmHead = result.lmHead;
1132
1389
  this.embeddingPostprocessor = result.embeddingPostprocessor;
1390
+ this.diffusionGemmaSelfConditioning = result.diffusionGemmaSelfConditioning;
1133
1391
  this.#normOffsetDebugLogged = result.normOffsetDebugLogged;
1134
1392
  }
1135
1393
 
@@ -1166,9 +1424,14 @@ export class DopplerLoader {
1166
1424
  layersLoaded: this.layers.size,
1167
1425
  expertsLoaded: this.experts.size + expertCacheCount,
1168
1426
  gpuBuffers: this.gpuBuffers.size,
1427
+ loadTiming: this.getLoadTiming(),
1169
1428
  };
1170
1429
  }
1171
1430
 
1431
+ getLoadTiming() {
1432
+ return this.loadTiming ? cloneJsonValue(this.loadTiming) : null;
1433
+ }
1434
+
1172
1435
 
1173
1436
  getExpertCacheStats() {
1174
1437
  return this.expertCache?.getStats() || null;
@@ -1199,10 +1462,21 @@ export class DopplerLoader {
1199
1462
  }
1200
1463
  this.gpuBuffers.delete(gpuBuffer);
1201
1464
  };
1465
+ const releaseSplitCandidate = (value) => {
1466
+ const sections = value?.gpuSplitWeight?.sections;
1467
+ if (!Array.isArray(sections)) {
1468
+ return;
1469
+ }
1470
+ for (const section of sections) {
1471
+ releaseCandidate(section?.buffer ?? null);
1472
+ }
1473
+ value.gpuSplitWeight = null;
1474
+ };
1202
1475
 
1203
1476
  for (const buffer of this.gpuBuffers) {
1204
1477
  releaseCandidate(buffer);
1205
1478
  }
1479
+ releaseSplitCandidate(this.lmHead);
1206
1480
  this.gpuBuffers.clear();
1207
1481
 
1208
1482
  if (this.expertCache) {
@@ -1236,6 +1510,7 @@ export class DopplerLoader {
1236
1510
  this.modelId = null;
1237
1511
  this.loadedShards.clear();
1238
1512
  this.isLoaded = false;
1513
+ this.loadTiming = null;
1239
1514
  this.tensorLocations.clear();
1240
1515
  this.#layerShardMap.clear();
1241
1516
  this.shardCache.clear();
@@ -0,0 +1,31 @@
1
+ export const MAX_SPLIT4_EMBEDDING_SECTIONS: 4;
2
+ export const MAX_SPLIT8_EMBEDDING_SECTIONS: 8;
3
+ export const MAX_SPLIT_EMBEDDING_SECTIONS: 8;
4
+ export const SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD: 2;
5
+
6
+ export function getEmbeddingFloatDtype(location: unknown): string;
7
+
8
+ export function expectsSplitGpuEmbeddingKernel(embeddingKernel: unknown): boolean;
9
+
10
+ export function getSplitGpuEmbeddingKernelSectionCount(embeddingKernel: unknown): number;
11
+
12
+ export function getSplitGpuEmbeddingRequiredStorageBuffers(sectionCount: number): number;
13
+
14
+ export function getMaxSplitGpuEmbeddingSectionsForDevice(
15
+ embeddingKernel?: unknown,
16
+ device?: unknown
17
+ ): number;
18
+
19
+ export function createGpuResidentEmbeddingLimitError(options: {
20
+ name: string;
21
+ location: unknown;
22
+ embeddingKernel?: unknown;
23
+ }): Error | null;
24
+
25
+ export function resolveManifestGpuResidentEmbeddingLimitError(
26
+ manifest: unknown,
27
+ options?: {
28
+ storageManifest?: unknown;
29
+ runtimeConfig?: unknown;
30
+ }
31
+ ): Error | null;