doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -11,7 +11,13 @@ import {
11
11
  } from '../storage/shard-manager.js';
12
12
  import { clearManifest, parseManifest, setManifest as setCurrentManifest } from '../formats/rdrr/index.js';
13
13
  import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
14
- import { acquireBuffer, isBufferActive, releaseBuffer, forceBufferPoolReclaim } from '../memory/buffer-pool.js';
14
+ import {
15
+ PersistentBufferSet,
16
+ acquireBuffer,
17
+ isBufferActive,
18
+ releaseBuffer,
19
+ forceBufferPoolReclaim,
20
+ } from '../memory/buffer-pool.js';
15
21
  import { getExpertCache } from './experts/expert-cache.js';
16
22
  import { formatBytes } from '../storage/quota.js';
17
23
  import { log, trace as debugTrace } from '../debug/index.js';
@@ -35,6 +41,7 @@ import {
35
41
  loadTensorToCPU,
36
42
  } from './tensors/tensor-loader.js';
37
43
  import { loadEmbeddings } from './embedding-loader.js';
44
+ import { loadPerLayerInputWeights } from './per-layer-input-loader.js';
38
45
  import { loadLayer } from './layer-loader.js';
39
46
  import { loadFinalWeights } from './final-weights-loader.js';
40
47
  import {
@@ -42,8 +49,15 @@ import {
42
49
  prefetchExperts as prefetchExpertsFromModule,
43
50
  predictNextLayerExperts as predictNextLayerExpertsFromModule,
44
51
  } from './experts/expert-loader.js';
45
- import { loadLoRAWeights as loadLoRAWeightsFromModule } from '../adapters/lora-loader.js';
46
52
  import { assembleShardData } from './tensors/tensor-reader.js';
53
+ import { hasSourceTransform } from './tensors/source-transform.js';
54
+
55
+ let loraModulePromise = null;
56
+
57
+ async function getExperimentalLoRAModule() {
58
+ loraModulePromise ??= import('../experimental/adapters/lora-loader.js');
59
+ return loraModulePromise;
60
+ }
47
61
 
48
62
  function hasExpertGroups(manifest) {
49
63
  if (!manifest?.groups) return false;
@@ -113,11 +127,13 @@ export class DopplerLoader {
113
127
 
114
128
  embeddingPostprocessor = null;
115
129
 
130
+ perLayerInputWeights = null;
131
+
116
132
  // Memory management
117
133
 
118
134
  heapManager = null;
119
135
 
120
- gpuBuffers = new Set();
136
+ gpuBuffers = new PersistentBufferSet();
121
137
 
122
138
  // Expert cache for MoE models (LRU eviction)
123
139
 
@@ -138,6 +154,8 @@ export class DopplerLoader {
138
154
  #loadingConfig;
139
155
  #loaderDebug = null;
140
156
 
157
+ #perLayerInputSession = null;
158
+
141
159
  // Fused Q4_K matmul: skip dequantization for matmul weights, use fused kernel
142
160
 
143
161
  useFusedQ4K = false;
@@ -148,6 +166,8 @@ export class DopplerLoader {
148
166
 
149
167
  keepF32Weights = false;
150
168
 
169
+ q4kMaterializationMode = 'dense';
170
+
151
171
  // Internal tracking
152
172
 
153
173
  #normOffsetLogged = false;
@@ -193,11 +213,16 @@ export class DopplerLoader {
193
213
  this.#loaderDebug = loaderDebug ?? null;
194
214
  }
195
215
 
216
+ setPerLayerInputSession(sessionConfig) {
217
+ this.#perLayerInputSession = sessionConfig ?? null;
218
+ }
219
+
196
220
 
197
221
  setQ4KConfig(config) {
198
222
  this.useFusedQ4K = config.useFusedQ4K;
199
223
  this.q4kLayout = config.q4kLayout;
200
224
  this.keepF32Weights = config.keepF32Weights;
225
+ this.q4kMaterializationMode = config.q4kMaterializationMode ?? 'dense';
201
226
  }
202
227
 
203
228
 
@@ -216,6 +241,29 @@ export class DopplerLoader {
216
241
  this.#memoryMonitor.start(() => this.#getMemoryState());
217
242
  }
218
243
 
244
+ #assertResidentBudget(phase) {
245
+ const budgetConfig = this.#loadingConfig?.memoryManagement?.budget;
246
+ if (!budgetConfig || budgetConfig.enabled !== true) {
247
+ return;
248
+ }
249
+ const maxResidentBytes = Number(budgetConfig.maxResidentBytes);
250
+ if (!Number.isFinite(maxResidentBytes) || maxResidentBytes <= 0) {
251
+ return;
252
+ }
253
+ if (typeof process === 'undefined' || typeof process.memoryUsage !== 'function') {
254
+ return;
255
+ }
256
+ const rssBytes = process.memoryUsage().rss;
257
+ if (rssBytes <= maxResidentBytes) {
258
+ return;
259
+ }
260
+ throw new Error(
261
+ `Loader resident memory budget exceeded during ${phase}: ` +
262
+ `rss=${formatBytes(rssBytes)}, max=${formatBytes(maxResidentBytes)}. ` +
263
+ 'Lower the model working set or raise runtime.loading.memoryManagement.budget.maxResidentBytes.'
264
+ );
265
+ }
266
+
219
267
 
220
268
  #stopMemoryLogging(phase = 'complete') {
221
269
  if (this.#memoryMonitor) {
@@ -296,12 +344,13 @@ export class DopplerLoader {
296
344
  const prevLayerShardMap = new Map(this.#layerShardMap);
297
345
 
298
346
  try {
347
+ const { loadLoRAWeights } = await getExperimentalLoRAModule();
299
348
  this.manifest = manifest;
300
349
  // We must rebuild locations so _loadTensor finds them
301
350
  await this.#buildTensorLocations();
302
351
  this.#logWeightBreakdown();
303
352
 
304
- return await loadLoRAWeightsFromModule(
353
+ return await loadLoRAWeights(
305
354
  manifest,
306
355
  (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent)
307
356
  );
@@ -363,6 +412,7 @@ export class DopplerLoader {
363
412
  this.modelId = modelId;
364
413
 
365
414
  this.#startMemoryLogging();
415
+ this.#assertResidentBudget('load start');
366
416
 
367
417
  if (!this.shardCache.hasCustomLoader) {
368
418
  await openModelStore(modelId);
@@ -492,6 +542,7 @@ export class DopplerLoader {
492
542
  try {
493
543
  reportProgress('shards', 0.1, 'Loading embeddings...');
494
544
  await this.#loadEmbeddings(onProgress);
545
+ this.#assertResidentBudget('embeddings');
495
546
 
496
547
  const resolveNumLayers = (value) => {
497
548
  const normalized = Number(value);
@@ -565,6 +616,7 @@ export class DopplerLoader {
565
616
  message: `Layer ${l + 1}/${numLayers}`,
566
617
  });
567
618
  }
619
+ this.#assertResidentBudget(`layer ${l + 1}`);
568
620
  }
569
621
 
570
622
  const layersTotalTime = ((performance.now() - layersStartTime) / 1000).toFixed(2);
@@ -572,6 +624,7 @@ export class DopplerLoader {
572
624
 
573
625
  reportProgress('gpu_transfer', 0.85, 'Loading final weights...');
574
626
  await this.#loadFinalWeights(onProgress);
627
+ this.#assertResidentBudget('final weights');
575
628
 
576
629
  if (onProgress) {
577
630
  onProgress({ stage: 'complete', progress: 1.0 });
@@ -674,6 +727,10 @@ export class DopplerLoader {
674
727
  #prefetchLayerShards(layerIdx) {
675
728
  const prefetch = this.#loadingConfig.prefetch;
676
729
  if (!prefetch?.enabled) return;
730
+ // Range-capable custom loaders are expected to serve fine-grained tensor reads.
731
+ // Whole-shard prefetch defeats that contract and can force invalid >4 GiB reads
732
+ // for direct-source SafeTensors bundles.
733
+ if (this.shardCache.hasCustomLoader && this.shardCache.canStreamRanges) return;
677
734
 
678
735
  const layersAhead = prefetch.layersAhead;
679
736
  if (!Number.isFinite(layersAhead) || layersAhead <= 0) return;
@@ -734,6 +791,7 @@ export class DopplerLoader {
734
791
  }
735
792
  const config = {
736
793
  useFusedQ4K: this.useFusedQ4K,
794
+ q4kMaterializationMode: this.q4kMaterializationMode,
737
795
  keepF32Weights: this.keepF32Weights,
738
796
  q4kLayout: this.q4kLayout,
739
797
  loaderDebug: this.#loaderDebug,
@@ -767,6 +825,7 @@ export class DopplerLoader {
767
825
 
768
826
  #shouldStreamUploadToGPU(location) {
769
827
  if (!location?.size || location.size <= 0) return false;
828
+ if (hasSourceTransform(location)) return false;
770
829
  if (this.shardCache.hasCustomLoader && !this.shardCache.canStreamRanges) return false;
771
830
  const chunkBytes = this.#loadingConfig?.storage?.backend?.streaming?.readChunkBytes ?? 0;
772
831
  if (!Number.isFinite(chunkBytes) || chunkBytes <= 0) return false;
@@ -797,10 +856,27 @@ export class DopplerLoader {
797
856
 
798
857
  try {
799
858
  let dstOffset = 0;
800
- const uploadChunk = (bytes) => {
859
+ let pendingBytes = null;
860
+ const writeAlignedChunk = (bytes) => {
861
+ if (bytes.byteLength === 0) return;
801
862
  device.queue.writeBuffer(raw, dstOffset, bytes, bytes.byteOffset, bytes.byteLength);
802
863
  dstOffset += bytes.byteLength;
803
864
  };
865
+ const uploadChunk = (bytes) => {
866
+ let merged = bytes;
867
+ if (pendingBytes && pendingBytes.byteLength > 0) {
868
+ merged = new Uint8Array(pendingBytes.byteLength + bytes.byteLength);
869
+ merged.set(pendingBytes, 0);
870
+ merged.set(bytes, pendingBytes.byteLength);
871
+ pendingBytes = null;
872
+ }
873
+ const alignedLength = merged.byteLength - (merged.byteLength % 4);
874
+ if (alignedLength > 0) {
875
+ writeAlignedChunk(merged.subarray(0, alignedLength));
876
+ }
877
+ const remainder = merged.byteLength - alignedLength;
878
+ pendingBytes = remainder > 0 ? merged.slice(alignedLength) : null;
879
+ };
804
880
  const streamRange = (idx, offset, length) => this.shardCache.streamRange(idx, offset, length, { chunkBytes });
805
881
 
806
882
  if (location.spans) {
@@ -815,6 +891,14 @@ export class DopplerLoader {
815
891
  }
816
892
  }
817
893
 
894
+ if (pendingBytes && pendingBytes.byteLength > 0) {
895
+ const padded = new Uint8Array(4);
896
+ padded.set(pendingBytes, 0);
897
+ writeAlignedChunk(padded);
898
+ dstOffset -= (4 - pendingBytes.byteLength);
899
+ pendingBytes = null;
900
+ }
901
+
818
902
  if (dstOffset !== location.size) {
819
903
  throw new Error(
820
904
  `Stream upload short read for "${name}": got=${dstOffset}, expected=${location.size}.`
@@ -844,6 +928,7 @@ export class DopplerLoader {
844
928
  const ctx = {
845
929
  tensorLocations: this.tensorLocations,
846
930
  loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
931
+ loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
847
932
  shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
848
933
  resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
849
934
  gpuBuffers: this.gpuBuffers,
@@ -851,9 +936,20 @@ export class DopplerLoader {
851
936
  // Keep embedding weights in F32 when manifest quantization requires it.
852
937
  // gather.wgsl reads embeddings as f32; downcasting here corrupts reads.
853
938
  preserveF32Embeddings: String(this.manifest?.quantizationInfo?.embeddings ?? '').toLowerCase() === 'f32',
939
+ hostHasShaderF16: this.gpuCapabilities?.hasF16 ?? null,
854
940
  };
855
941
 
856
942
  this.embeddings = await loadEmbeddings(ctx);
943
+ this.perLayerInputWeights = await loadPerLayerInputWeights({
944
+ modelId: this.manifest?.modelId ?? null,
945
+ tensorLocations: this.tensorLocations,
946
+ gpuBuffers: this.gpuBuffers,
947
+ loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
948
+ shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
949
+ loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
950
+ resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
951
+ perLayerInputSession: this.#perLayerInputSession,
952
+ }, this.manifest?.architecture ?? null);
857
953
  }
858
954
 
859
955
 
@@ -939,6 +1035,7 @@ export class DopplerLoader {
939
1035
  const ctx = {
940
1036
  tensorLocations: this.tensorLocations,
941
1037
  loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
1038
+ loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
942
1039
  needsNormWeightOffset: () => this.#needsNormWeightOffset(),
943
1040
  shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
944
1041
  resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
@@ -1055,6 +1152,7 @@ export class DopplerLoader {
1055
1152
  this.lmHead = null;
1056
1153
  this.finalNorm = null;
1057
1154
  this.embeddingPostprocessor = null;
1155
+ this.perLayerInputWeights = null;
1058
1156
  this.manifest = null;
1059
1157
  clearManifest();
1060
1158
  this.modelId = null;
@@ -22,7 +22,7 @@ export type TensorLoader = (
22
22
  name: string,
23
23
  toGPU?: boolean,
24
24
  silent?: boolean
25
- ) => Promise<GPUBuffer | WeightBuffer | Float32Array | Uint8Array | null>;
25
+ ) => Promise<GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | Uint8Array | null>;
26
26
 
27
27
  /**
28
28
  * Context required for embedding loading.
@@ -34,6 +34,8 @@ export interface EmbeddingLoaderContext {
34
34
  loadTensor: TensorLoader;
35
35
  /** Check if large weight should stream to CPU */
36
36
  shouldStreamLargeWeight: (name: string, loc: TensorLocation, label: string) => boolean;
37
+ /** Load a shard byte range for range-backed CPU sources */
38
+ loadShardRange?: (index: number, offset: number, length: number) => Promise<ArrayBuffer>;
37
39
  /** Resolve weight layout from location */
38
40
  resolveWeightLayout: (loc: TensorLocation) => WeightLayout;
39
41
  /** GPU buffers to track for cleanup */
@@ -42,6 +44,8 @@ export interface EmbeddingLoaderContext {
42
44
  keepF32Weights: boolean;
43
45
  /** Preserve F32 embeddings when manifest quantization requires F32 embedding weights */
44
46
  preserveF32Embeddings?: boolean;
47
+ /** Host shader-f16 capability, used to choose CPU F16->F32 fallback on no-f16 devices */
48
+ hostHasShaderF16?: boolean | null;
45
49
  }
46
50
 
47
51
  /** Result of embedding loading */
@@ -4,6 +4,7 @@ import {
4
4
  createWeightBuffer,
5
5
  createCpuWeightBuffer,
6
6
  isWeightBuffer,
7
+ isCpuWeightBuffer,
7
8
  getWeightDtype,
8
9
  getLayout,
9
10
  } from '../gpu/weight-buffer.js';
@@ -14,6 +15,7 @@ import { selectRuleValue } from '../rules/rule-registry.js';
14
15
  import { createTensor } from '../gpu/tensor.js';
15
16
  import { castF16ToF32 } from '../gpu/kernel-selector.js';
16
17
  import { releaseBuffer } from '../memory/buffer-pool.js';
18
+ import { loadTensorRange } from './tensors/tensor-reader.js';
17
19
 
18
20
  // ============================================================================
19
21
  // Constants
@@ -23,10 +25,42 @@ import { releaseBuffer } from '../memory/buffer-pool.js';
23
25
  const EMBEDDING_ROLE = 'embedding';
24
26
  const EMBEDDING_GROUP = 'embed';
25
27
 
28
+ function isPerLayerEmbeddingTensor(name) {
29
+ return String(name ?? '').toLowerCase().includes('embed_tokens_per_layer.weight');
30
+ }
31
+
26
32
  function isGpuBufferInstance(value) {
27
33
  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
28
34
  }
29
35
 
36
+ function createRangeBackedTensorSource(ctx, name, location) {
37
+ if (typeof ctx.loadShardRange !== 'function') {
38
+ return null;
39
+ }
40
+ const normalizedLocationDtype = typeof location?.dtype === 'string'
41
+ ? location.dtype.toLowerCase()
42
+ : 'f32';
43
+ return {
44
+ kind: 'tensor_range_source',
45
+ sourceDtype: normalizedLocationDtype,
46
+ async loadRange(byteOffset, byteLength) {
47
+ return loadTensorRange(location, name, byteOffset, byteLength, ctx.loadShardRange);
48
+ },
49
+ };
50
+ }
51
+
52
+ function createRangeBackedWeightBuffer(ctx, name, location) {
53
+ const source = createRangeBackedTensorSource(ctx, name, location);
54
+ if (!source || !location?.shape || location.shape.length !== 2) {
55
+ return null;
56
+ }
57
+ const layout = ctx.resolveWeightLayout(location);
58
+ const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
59
+ locationDtype: location.dtype,
60
+ });
61
+ return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
62
+ }
63
+
30
64
  // ============================================================================
31
65
  // Main Function
32
66
  // ============================================================================
@@ -34,9 +68,11 @@ function isGpuBufferInstance(value) {
34
68
 
35
69
  export async function loadEmbeddings(ctx) {
36
70
  const embeddingNames = getTensorNamesByRole(ctx.tensorLocations, EMBEDDING_ROLE, EMBEDDING_GROUP);
37
- const candidates = embeddingNames.length > 0
71
+ const groupedCandidates = embeddingNames.length > 0
38
72
  ? embeddingNames
39
73
  : getTensorNamesByRole(ctx.tensorLocations, EMBEDDING_ROLE);
74
+ const filteredCandidates = groupedCandidates.filter((name) => !isPerLayerEmbeddingTensor(name));
75
+ const candidates = filteredCandidates.length > 0 ? filteredCandidates : groupedCandidates;
40
76
 
41
77
  if (candidates.length === 0) {
42
78
  throw new Error(
@@ -47,23 +83,32 @@ export async function loadEmbeddings(ctx) {
47
83
 
48
84
  for (const name of candidates) {
49
85
  const loc = ctx.tensorLocations.get(name);
50
- const shouldStream = loc ? ctx.shouldStreamLargeWeight(name, loc, 'Embedding') : false;
86
+ const requiresCpuFloatFallback = loc
87
+ && String(loc.dtype ?? '').toUpperCase() === 'F16'
88
+ && ctx.hostHasShaderF16 === false;
89
+ const shouldStream = requiresCpuFloatFallback
90
+ || (loc ? ctx.shouldStreamLargeWeight(name, loc, 'Embedding') : false);
51
91
 
52
92
  // Load tensor (to CPU if streaming, to GPU otherwise)
53
- const tensor = await ctx.loadTensor(name, !shouldStream, true);
93
+ const tensor = shouldStream
94
+ ? (
95
+ createRangeBackedWeightBuffer(ctx, name, loc)
96
+ ?? await ctx.loadTensor(name, false, true)
97
+ )
98
+ : await ctx.loadTensor(name, true, true);
54
99
 
55
100
  // Skip if not found
56
101
  if (!tensor) continue;
57
102
 
58
103
  // Handle streaming path (CPU)
59
- if (shouldStream && !(tensor instanceof Float32Array)) {
104
+ if (shouldStream && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
60
105
  throw new Error(
61
106
  `[Loader] Embedding "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc?.dtype ?? 'unknown'}).`
62
107
  );
63
108
  }
64
109
 
65
110
  // Handle valid tensor types
66
- if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || tensor instanceof Float32Array) {
111
+ if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || tensor instanceof Float32Array) {
67
112
  const result = await processEmbeddingTensor(ctx, tensor, name, loc, shouldStream);
68
113
  if (result) {
69
114
  return result;
@@ -98,6 +143,11 @@ async function processEmbeddingTensor(ctx, tensor, name, loc, shouldStream) {
98
143
  return maybeDowncastEmbeddings(ctx, promoted, name, loc);
99
144
  }
100
145
 
146
+ if (isCpuWeightBuffer(promoted)) {
147
+ log.warn('Loader', `Embeddings stored on CPU via range-backed source (layout=${promoted.layout})`);
148
+ return promoted;
149
+ }
150
+
101
151
  // Float32Array streaming path
102
152
  if (promoted instanceof Float32Array && loc?.shape && shouldStream) {
103
153
  const layout = ctx.resolveWeightLayout(loc);
@@ -129,6 +179,7 @@ async function processEmbeddingTensor(ctx, tensor, name, loc, shouldStream) {
129
179
  async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
130
180
  if (!ctx.preserveF32Embeddings) return current;
131
181
  if (current instanceof Float32Array) return current;
182
+ if (isCpuWeightBuffer(current)) return current;
132
183
 
133
184
  if (isWeightBuffer(current)) {
134
185
  const dtype = getWeightDtype(current);
@@ -168,7 +219,7 @@ async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
168
219
 
169
220
  async function maybeDowncastEmbeddings(ctx, current, name, loc) {
170
221
  // Can't downcast Float32Array or CpuWeightBuffer
171
- if (current instanceof Float32Array) {
222
+ if (current instanceof Float32Array || isCpuWeightBuffer(current)) {
172
223
  return current;
173
224
  }
174
225
 
@@ -10,6 +10,7 @@ import { maybeDowncastToF16 } from './weight-downcast.js';
10
10
  import { getTensorNamesByRole } from './tensors/tensor-role.js';
11
11
  import { log, trace as debugTrace } from '../debug/index.js';
12
12
  import { selectRuleValue } from '../rules/rule-registry.js';
13
+ import { loadTensorRange } from './tensors/tensor-reader.js';
13
14
 
14
15
  // ============================================================================
15
16
  // Constants
@@ -25,6 +26,34 @@ function isGpuBufferInstance(value) {
25
26
  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
26
27
  }
27
28
 
29
+ function createRangeBackedTensorSource(ctx, name, location) {
30
+ if (typeof ctx.loadShardRange !== 'function') {
31
+ return null;
32
+ }
33
+ const normalizedLocationDtype = typeof location?.dtype === 'string'
34
+ ? location.dtype.toLowerCase()
35
+ : 'f32';
36
+ return {
37
+ kind: 'tensor_range_source',
38
+ sourceDtype: normalizedLocationDtype,
39
+ async loadRange(byteOffset, byteLength) {
40
+ return loadTensorRange(location, name, byteOffset, byteLength, ctx.loadShardRange);
41
+ },
42
+ };
43
+ }
44
+
45
+ function createRangeBackedWeightBuffer(ctx, name, location) {
46
+ const source = createRangeBackedTensorSource(ctx, name, location);
47
+ if (!source || !location?.shape || location.shape.length !== 2) {
48
+ return null;
49
+ }
50
+ const layout = ctx.resolveWeightLayout(location);
51
+ const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
52
+ locationDtype: location.dtype,
53
+ });
54
+ return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
55
+ }
56
+
28
57
  function isLikelyFinalNormName(name) {
29
58
  const lower = String(name || '').toLowerCase();
30
59
  if (!lower) return false;
@@ -157,15 +186,20 @@ async function loadLmHead(ctx) {
157
186
  if (!loc) continue;
158
187
 
159
188
  const shouldStream = ctx.shouldStreamLargeWeight(name, loc, 'LM head');
160
- const tensor = await ctx.loadTensor(name, !shouldStream, true);
189
+ const tensor = shouldStream
190
+ ? (
191
+ createRangeBackedWeightBuffer(ctx, name, loc)
192
+ ?? await ctx.loadTensor(name, false, true)
193
+ )
194
+ : await ctx.loadTensor(name, true, true);
161
195
 
162
- if (shouldStream && tensor && !(tensor instanceof Float32Array)) {
196
+ if (shouldStream && tensor && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
163
197
  throw new Error(
164
198
  `[Loader] LM head "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc.dtype}).`
165
199
  );
166
200
  }
167
201
 
168
- if (tensor && (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || tensor instanceof Float32Array)) {
202
+ if (tensor && (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || tensor instanceof Float32Array)) {
169
203
  lmHeadName = name;
170
204
  lmHeadLoc = loc;
171
205
  lmHead = processLmHeadTensor(ctx, tensor, name, loc, shouldStream);
@@ -253,6 +287,11 @@ async function loadCpuFloatTensor(ctx, tensorName, label) {
253
287
 
254
288
 
255
289
  function processLmHeadTensor(ctx, tensor, name, loc, shouldStream) {
290
+ if (isCpuWeightBuffer(tensor)) {
291
+ log.warn('Loader', `LM head stored on CPU via range-backed source (layout=${tensor.layout})`);
292
+ return tensor;
293
+ }
294
+
256
295
  // Float32Array streaming path
257
296
  if (tensor instanceof Float32Array && shouldStream) {
258
297
  const layout = ctx.resolveWeightLayout(loc);
@@ -292,8 +331,8 @@ async function maybeDowncastLmHead(ctx, lmHead, lmHeadName, lmHeadLoc) {
292
331
  return lmHead;
293
332
  }
294
333
 
295
- // Can't downcast Float32Array
296
- if (lmHead instanceof Float32Array) {
334
+ // Can't downcast Float32Array or CpuWeightBuffer
335
+ if (lmHead instanceof Float32Array || isCpuWeightBuffer(lmHead)) {
297
336
  return lmHead;
298
337
  }
299
338