doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -3,19 +3,34 @@
3
3
  import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
4
4
  import { parseManifest, getExpectedShardHash } from '../formats/rdrr/index.js';
5
5
  import { createPipeline } from './pipelines/text.js';
6
+ import { createNodeFileShardStorageContext } from './pipelines/text/init.js';
6
7
  import { log as debugLog } from '../debug/index.js';
7
8
  import { getRuntimeConfig, setRuntimeConfig } from '../config/runtime.js';
8
- import { downloadShard as downloadShardFromDistribution } from '../distribution/shard-delivery.js';
9
- import {
10
- fetchHotSwapManifest,
11
- verifyHotSwapManifest,
12
- } from '../hotswap/manifest.js';
13
- import { evaluateHotSwapRollout, setHotSwapManifest } from '../hotswap/runtime.js';
14
- import {
15
- fetchIntentBundle,
16
- getKernelRegistryVersion,
17
- verifyIntentBundle,
18
- } from '../hotswap/intent-bundle.js';
9
+
10
+ let distributionModulePromise = null;
11
+ let hotSwapManifestModulePromise = null;
12
+ let hotSwapRuntimeModulePromise = null;
13
+ let intentBundleModulePromise = null;
14
+
15
+ async function loadDistributionModule() {
16
+ distributionModulePromise ??= import('../experimental/distribution/shard-delivery.js');
17
+ return distributionModulePromise;
18
+ }
19
+
20
+ async function loadHotSwapManifestModule() {
21
+ hotSwapManifestModulePromise ??= import('../experimental/hotswap/manifest.js');
22
+ return hotSwapManifestModulePromise;
23
+ }
24
+
25
+ async function loadHotSwapRuntimeModule() {
26
+ hotSwapRuntimeModulePromise ??= import('../experimental/hotswap/runtime.js');
27
+ return hotSwapRuntimeModulePromise;
28
+ }
29
+
30
+ async function loadIntentBundleModule() {
31
+ intentBundleModulePromise ??= import('../experimental/hotswap/intent-bundle.js');
32
+ return intentBundleModulePromise;
33
+ }
19
34
 
20
35
 
21
36
 
@@ -166,6 +181,7 @@ export function createHttpShardLoader(baseUrl, manifest, log) {
166
181
  // Start new load and track it as pending
167
182
  const loadPromise = (async () => {
168
183
  try {
184
+ const { downloadShard: downloadShardFromDistribution } = await loadDistributionModule();
169
185
  const result = await downloadShardFromDistribution(baseUrl, idx, shard, {
170
186
  distributionConfig,
171
187
  algorithm,
@@ -201,6 +217,16 @@ export function createHttpShardLoader(baseUrl, manifest, log) {
201
217
  };
202
218
  }
203
219
 
220
+ export function createHarnessShardStorageContext(modelUrl, manifest, log) {
221
+ const nodeFileStorageContext = createNodeFileShardStorageContext(modelUrl, manifest);
222
+ if (nodeFileStorageContext) {
223
+ return nodeFileStorageContext;
224
+ }
225
+ return {
226
+ loadShard: createHttpShardLoader(modelUrl, manifest, log),
227
+ };
228
+ }
229
+
204
230
  // ============================================================================
205
231
  // Pipeline Initialization
206
232
  // ============================================================================
@@ -231,7 +257,14 @@ export async function initializeInference(modelUrl, options = {}) {
231
257
  const hotSwapConfig = getRuntimeConfig().shared.hotSwap;
232
258
  const intentBundleConfig = getRuntimeConfig().shared.intentBundle;
233
259
  if (hotSwapConfig.enabled && hotSwapConfig.manifestUrl) {
234
- const rolloutDecision = evaluateHotSwapRollout(hotSwapConfig, {
260
+ const [
261
+ hotSwapManifestModule,
262
+ hotSwapRuntimeModule,
263
+ ] = await Promise.all([
264
+ loadHotSwapManifestModule(),
265
+ loadHotSwapRuntimeModule(),
266
+ ]);
267
+ const rolloutDecision = hotSwapRuntimeModule.evaluateHotSwapRollout(hotSwapConfig, {
235
268
  modelUrl,
236
269
  subjectId: options.modelId || null,
237
270
  sessionId: options.sessionId || null,
@@ -242,8 +275,8 @@ export async function initializeInference(modelUrl, options = {}) {
242
275
  } else {
243
276
  onProgress('hotswap', 0.05, 'Loading hot-swap manifest...');
244
277
  log(`Hot-swap: loading manifest ${hotSwapConfig.manifestUrl}`);
245
- const hotSwapManifest = await fetchHotSwapManifest(hotSwapConfig.manifestUrl);
246
- const verification = await verifyHotSwapManifest(hotSwapManifest, hotSwapConfig, {
278
+ const hotSwapManifest = await hotSwapManifestModule.fetchHotSwapManifest(hotSwapConfig.manifestUrl);
279
+ const verification = await hotSwapManifestModule.verifyHotSwapManifest(hotSwapManifest, hotSwapConfig, {
247
280
  source: {
248
281
  kind: 'remote',
249
282
  isLocal: false,
@@ -253,7 +286,7 @@ export async function initializeInference(modelUrl, options = {}) {
253
286
  if (!verification.ok) {
254
287
  throw new Error(`Hot-swap manifest rejected: ${verification.reason}`);
255
288
  }
256
- setHotSwapManifest(hotSwapManifest);
289
+ hotSwapRuntimeModule.setHotSwapManifest(hotSwapManifest);
257
290
  log(
258
291
  `Hot-swap manifest accepted: ${hotSwapManifest.bundleId} (${verification.reason}, rollout=${rolloutDecision.reason})`
259
292
  );
@@ -278,13 +311,14 @@ export async function initializeInference(modelUrl, options = {}) {
278
311
  const manifest = await fetchManifest(manifestUrl);
279
312
 
280
313
  if (intentBundleConfig.enabled && intentBundleConfig.bundleUrl) {
314
+ const intentBundleModule = await loadIntentBundleModule();
281
315
  onProgress('intent', 0.12, 'Loading intent bundle...');
282
316
  log(`Intent bundle: loading ${intentBundleConfig.bundleUrl}`);
283
- const bundle = await fetchIntentBundle(intentBundleConfig.bundleUrl);
317
+ const bundle = await intentBundleModule.fetchIntentBundle(intentBundleConfig.bundleUrl);
284
318
  const kernelRegistryVersion = intentBundleConfig.requireKernelRegistryVersion
285
- ? await getKernelRegistryVersion()
319
+ ? await intentBundleModule.getKernelRegistryVersion()
286
320
  : null;
287
- const verification = await verifyIntentBundle(bundle, {
321
+ const verification = await intentBundleModule.verifyIntentBundle(bundle, {
288
322
  manifest: intentBundleConfig.requireBaseModelHash ? manifest : null,
289
323
  kernelRegistryVersion,
290
324
  enforceDeterministicOutput: intentBundleConfig.enforceDeterministicOutput,
@@ -305,7 +339,7 @@ export async function initializeInference(modelUrl, options = {}) {
305
339
  log(`Model: ${modelLabel}`);
306
340
 
307
341
  // 3. Create shard loader
308
- const loadShard = createHttpShardLoader(modelUrl, manifest, log);
342
+ const storageContext = createHarnessShardStorageContext(modelUrl, manifest, log);
309
343
 
310
344
  // 4. Build runtime options
311
345
 
@@ -318,7 +352,7 @@ export async function initializeInference(modelUrl, options = {}) {
318
352
  log('Creating pipeline...');
319
353
 
320
354
  const pipeline = await createPipeline( ( (manifest)), {
321
- storage: { loadShard },
355
+ storage: storageContext,
322
356
  gpu: { device },
323
357
  runtime,
324
358
  baseUrl: modelUrl,
@@ -67,6 +67,11 @@ export declare class Tokenizer {
67
67
  * Get vocabulary size
68
68
  */
69
69
  getVocabSize(): number;
70
+
71
+ /**
72
+ * Get high-priority token IDs for bounded runtime hot caches, or null when unavailable
73
+ */
74
+ getHotTokenIds(limit: number): number[] | null;
70
75
  }
71
76
 
72
77
  export default Tokenizer;
@@ -6,6 +6,8 @@ import { TransformersTokenizer, BundledTokenizer } from './tokenizers/bundled.js
6
6
  import { SentencePieceTokenizer } from './tokenizers/sentencepiece.js';
7
7
  import { BPETokenizer } from './tokenizers/bpe.js';
8
8
 
9
+ const BUNDLED_TOKENIZER_CACHE = new Map();
10
+
9
11
  function hasScheme(value) {
10
12
  return /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value);
11
13
  }
@@ -33,6 +35,28 @@ function parseTokenizerJsonPayload(value) {
33
35
  throw new Error('Tokenizer JSON loader must return an object, JSON string, or null.');
34
36
  }
35
37
 
38
+ function buildBundledTokenizerCacheKey(modelId, tokenizerConfig) {
39
+ const tokenizerType = typeof tokenizerConfig?.type === 'string'
40
+ ? tokenizerConfig.type.trim().toLowerCase()
41
+ : '';
42
+ const tokenizerFile = typeof tokenizerConfig?.file === 'string'
43
+ ? tokenizerConfig.file.trim()
44
+ : '';
45
+ if (!tokenizerType || !tokenizerFile) {
46
+ return null;
47
+ }
48
+ return JSON.stringify({
49
+ modelId: String(modelId || '').trim() || 'unknown',
50
+ tokenizerType,
51
+ tokenizerFile,
52
+ vocabSize: Number.isFinite(tokenizerConfig?.vocabSize) ? tokenizerConfig.vocabSize : null,
53
+ eosToken: tokenizerConfig?.eosToken ?? null,
54
+ bosToken: tokenizerConfig?.bosToken ?? null,
55
+ padToken: tokenizerConfig?.padToken ?? null,
56
+ unkToken: tokenizerConfig?.unkToken ?? null,
57
+ });
58
+ }
59
+
36
60
  export class Tokenizer {
37
61
 
38
62
  backend = null;
@@ -71,6 +95,15 @@ export class Tokenizer {
71
95
  if (isBundled && tokenizerConfig.file) {
72
96
  tokenizerConfig.deferSpecialTokens = true;
73
97
  log.info('Tokenizer', `Loading ${tokenizerConfig.type} tokenizer from ${tokenizerConfig.file}`);
98
+ const cacheKey = buildBundledTokenizerCacheKey(modelId, tokenizerConfig);
99
+ const cachedBackend = cacheKey ? BUNDLED_TOKENIZER_CACHE.get(cacheKey) : null;
100
+ if (cachedBackend) {
101
+ log.info('Tokenizer', `Bundled tokenizer cache hit: ${tokenizerConfig.file}`);
102
+ this.backend = cachedBackend;
103
+ this.config = tokenizerConfig;
104
+ return;
105
+ }
106
+
74
107
  this.backend = new BundledTokenizer(tokenizerConfig);
75
108
 
76
109
  const baseUrl = options.baseUrl;
@@ -121,6 +154,9 @@ export class Tokenizer {
121
154
 
122
155
  if (tokenizerJson) {
123
156
  (this.backend).load(tokenizerJson);
157
+ if (cacheKey) {
158
+ BUNDLED_TOKENIZER_CACHE.set(cacheKey, this.backend);
159
+ }
124
160
  this.config = tokenizerConfig;
125
161
  return;
126
162
  }
@@ -246,6 +282,13 @@ export class Tokenizer {
246
282
  getVocabSize() {
247
283
  return this.backend?.getVocabSize() || 0;
248
284
  }
285
+
286
+ getHotTokenIds(limit) {
287
+ if (!this.backend || typeof this.backend.getHotTokenIds !== 'function') {
288
+ return null;
289
+ }
290
+ return this.backend.getHotTokenIds(limit);
291
+ }
249
292
  }
250
293
 
251
294
  export default Tokenizer;
@@ -32,6 +32,11 @@ export declare abstract class BaseTokenizer implements TokenizerBackend {
32
32
  */
33
33
  getVocabSize(): number;
34
34
 
35
+ /**
36
+ * Get high-priority token IDs for bounded runtime hot caches, or null when unavailable
37
+ */
38
+ getHotTokenIds(limit: number): number[] | null;
39
+
35
40
  /**
36
41
  * Check if token is special
37
42
  */
@@ -62,6 +62,11 @@ export class BaseTokenizer {
62
62
  return this.vocabSize;
63
63
  }
64
64
 
65
+ getHotTokenIds(limit) {
66
+ void limit;
67
+ return null;
68
+ }
69
+
65
70
 
66
71
  isSpecialToken(tokenId) {
67
72
  return Object.values(this.specialTokens).includes(tokenId);
@@ -0,0 +1,9 @@
1
+ export interface BundledTokenizerBehaviorFlags {
2
+ addBosToken: boolean | null;
3
+ addEosToken: boolean | null;
4
+ }
5
+
6
+ export function inferBundledTokenizerBehaviorFlags(
7
+ tokenizerJson: unknown,
8
+ specialTokens?: { bos?: number | null, eos?: number | null } | null
9
+ ): BundledTokenizerBehaviorFlags;
@@ -0,0 +1,100 @@
1
+ function normalizeSpecialTokenLabel(value) {
2
+ if (typeof value !== 'string') {
3
+ return '';
4
+ }
5
+ return value.trim().toLowerCase();
6
+ }
7
+
8
+ function isBosLikeLabel(value) {
9
+ const normalized = normalizeSpecialTokenLabel(value);
10
+ return normalized === '<bos>' || normalized === '<s>' || normalized.includes('bos');
11
+ }
12
+
13
+ function isEosLikeLabel(value) {
14
+ const normalized = normalizeSpecialTokenLabel(value);
15
+ return normalized === '<eos>' || normalized === '</s>' || normalized.includes('eos');
16
+ }
17
+
18
+ function classifySpecialTokenReference(ref, specialTokens) {
19
+ if (typeof ref === 'number' && Number.isFinite(ref)) {
20
+ if (specialTokens?.bos === ref) return 'bos';
21
+ if (specialTokens?.eos === ref) return 'eos';
22
+ return null;
23
+ }
24
+
25
+ if (Array.isArray(ref)) {
26
+ for (const item of ref) {
27
+ const kind = classifySpecialTokenReference(item, specialTokens);
28
+ if (kind) {
29
+ return kind;
30
+ }
31
+ }
32
+ return null;
33
+ }
34
+
35
+ if (ref && typeof ref === 'object') {
36
+ if (typeof ref.id === 'number' && Number.isFinite(ref.id)) {
37
+ return classifySpecialTokenReference(ref.id, specialTokens);
38
+ }
39
+ if (Array.isArray(ref.ids)) {
40
+ return classifySpecialTokenReference(ref.ids, specialTokens);
41
+ }
42
+ if (Array.isArray(ref.tokens)) {
43
+ return classifySpecialTokenReference(ref.tokens, specialTokens);
44
+ }
45
+ }
46
+
47
+ if (typeof ref === 'string') {
48
+ if (isBosLikeLabel(ref)) return 'bos';
49
+ if (isEosLikeLabel(ref)) return 'eos';
50
+ }
51
+
52
+ return null;
53
+ }
54
+
55
+ function getTemplateBoundaryKind(template, boundary, specialTokens) {
56
+ if (!Array.isArray(template) || template.length === 0) {
57
+ return null;
58
+ }
59
+
60
+ const entries = boundary === 'last'
61
+ ? [...template].reverse()
62
+ : template;
63
+
64
+ for (const entry of entries) {
65
+ const special = entry?.SpecialToken;
66
+ if (!special) {
67
+ continue;
68
+ }
69
+
70
+ const kind = classifySpecialTokenReference(
71
+ special.tokens ?? special.ids ?? special.id ?? null,
72
+ specialTokens
73
+ );
74
+ if (kind) {
75
+ return kind;
76
+ }
77
+ }
78
+
79
+ return null;
80
+ }
81
+
82
+ export function inferBundledTokenizerBehaviorFlags(tokenizerJson, specialTokens = null) {
83
+ if (!tokenizerJson || typeof tokenizerJson !== 'object' || Array.isArray(tokenizerJson)) {
84
+ return { addBosToken: null, addEosToken: null };
85
+ }
86
+
87
+ const postProcessor = tokenizerJson.post_processor;
88
+ if (!postProcessor || postProcessor.type !== 'TemplateProcessing') {
89
+ return { addBosToken: null, addEosToken: null };
90
+ }
91
+
92
+ const single = Array.isArray(postProcessor.single) ? postProcessor.single : null;
93
+ const firstKind = getTemplateBoundaryKind(single, 'first', specialTokens);
94
+ const lastKind = getTemplateBoundaryKind(single, 'last', specialTokens);
95
+
96
+ return {
97
+ addBosToken: firstKind === 'bos' ? true : null,
98
+ addEosToken: lastKind === 'eos' ? true : null,
99
+ };
100
+ }
@@ -39,6 +39,8 @@ export declare class TransformersTokenizer extends BaseTokenizer {
39
39
  * Batch encode multiple texts
40
40
  */
41
41
  batchEncode(texts: string[]): number[][];
42
+
43
+ getHotTokenIds(limit: number): number[] | null;
42
44
  }
43
45
 
44
46
  /**
@@ -60,4 +62,6 @@ export declare class BundledTokenizer extends BaseTokenizer {
60
62
  encode(text: string): number[];
61
63
 
62
64
  decode(ids: number[], skipSpecialTokens?: boolean, trim?: boolean): string;
65
+
66
+ getHotTokenIds(limit: number): number[] | null;
63
67
  }
@@ -3,6 +3,7 @@
3
3
  import { BaseTokenizer } from './base.js';
4
4
  import { log } from '../../debug/index.js';
5
5
  import { getRuntimeConfig } from '../../config/runtime.js';
6
+ import { inferBundledTokenizerBehaviorFlags } from './behavior-flags.js';
6
7
 
7
8
  function pickCandidate(...values) {
8
9
  for (const value of values) {
@@ -94,6 +95,78 @@ function resolveByteLevelPretokenizerConfig(preTokenizer) {
94
95
  };
95
96
  }
96
97
 
98
+ function isSpecialLikeHotTokenCandidate(token) {
99
+ if (typeof token !== 'string' || token.length === 0) {
100
+ return true;
101
+ }
102
+ if ((token.startsWith('<') && token.endsWith('>')) || (token.startsWith('[') && token.endsWith(']'))) {
103
+ return true;
104
+ }
105
+ return /unused|reserved|multimodal|image|video|audio/i.test(token);
106
+ }
107
+
108
+ function normalizeBpeHotTokenCandidate(token) {
109
+ if (typeof token !== 'string') {
110
+ return '';
111
+ }
112
+ return token
113
+ .replace(/^[▁Ġ]+/, '')
114
+ .replace(/Ċ/g, '\n');
115
+ }
116
+
117
+ function scoreBpeHotTokenCandidate(token, id) {
118
+ if (isSpecialLikeHotTokenCandidate(token)) {
119
+ return Number.NEGATIVE_INFINITY;
120
+ }
121
+ const normalized = normalizeBpeHotTokenCandidate(token);
122
+ if (normalized.length === 0) {
123
+ return Number.NEGATIVE_INFINITY;
124
+ }
125
+
126
+ const hasBoundaryMarker = token.startsWith('▁') || token.startsWith('Ġ');
127
+ const isAscii = /^[\x00-\x7F]+$/.test(normalized);
128
+ const isLowerAlpha = /^[a-z]+$/.test(normalized);
129
+ const isTitleAlpha = /^[A-Z][a-z]+$/.test(normalized);
130
+ const isAlpha = /^[A-Za-z]+$/.test(normalized);
131
+ const isPunctuation = /^[.,!?;:'"()%-]+$/.test(normalized);
132
+ const isDigits = /^\d+$/.test(normalized);
133
+ const length = normalized.length;
134
+
135
+ let score = 0;
136
+ if (hasBoundaryMarker) score += 40;
137
+ if (isLowerAlpha) score += 32;
138
+ else if (isTitleAlpha) score += 24;
139
+ else if (isAlpha) score += 20;
140
+ if (isPunctuation) score += 20;
141
+ if (isDigits) score += 8;
142
+ if (isAscii) score += 12;
143
+
144
+ if (length === 1) score += hasBoundaryMarker ? 8 : 2;
145
+ else if (length <= 4) score += 18 - (length * 2);
146
+ else if (length <= 8) score += 12 - (length - 4);
147
+ else if (length <= 12) score += 4 - (length - 8);
148
+ else score -= Math.min(12, length - 12);
149
+
150
+ score -= id / 1e7;
151
+ return score;
152
+ }
153
+
154
+ function rankFallbackBpeHotTokenIds(reverseVocab, limit, isSpecialToken) {
155
+ const ranked = [];
156
+ for (const [id, token] of reverseVocab.entries()) {
157
+ if (typeof isSpecialToken === 'function' && isSpecialToken(id)) {
158
+ continue;
159
+ }
160
+ const score = scoreBpeHotTokenCandidate(token, id);
161
+ if (!Number.isFinite(score)) {
162
+ continue;
163
+ }
164
+ ranked.push({ id, score });
165
+ }
166
+ ranked.sort((a, b) => b.score - a.score || a.id - b.id);
167
+ return ranked.slice(0, limit).map((entry) => entry.id);
168
+ }
169
+
97
170
  function registerAddedTokens(addedTokens, vocab, reverseVocab, patterns, specialTokenIds, derivedSpecialTokens = null) {
98
171
  let maxId = -1;
99
172
  for (const token of addedTokens) {
@@ -187,6 +260,11 @@ export class TransformersTokenizer extends BaseTokenizer {
187
260
  batchEncode(texts) {
188
261
  return texts.map(t => this.encode(t));
189
262
  }
263
+
264
+ getHotTokenIds(limit) {
265
+ void limit;
266
+ return null;
267
+ }
190
268
  }
191
269
 
192
270
 
@@ -437,15 +515,18 @@ export class BundledTokenizer extends BaseTokenizer {
437
515
  const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(hf.pre_tokenizer);
438
516
  const configuredAddBosToken = this.addBosToken;
439
517
  const configuredAddEosToken = this.addEosToken;
518
+ const inferredFlags = inferBundledTokenizerBehaviorFlags(hf, this.specialTokens);
440
519
  this.addBosToken =
441
520
  hf.add_bos_token
442
521
  ?? hf.addBosToken
443
522
  ?? configuredAddBosToken
523
+ ?? inferredFlags.addBosToken
444
524
  ?? runtimeDefaults.addBosToken;
445
525
  this.addEosToken =
446
526
  hf.add_eos_token
447
527
  ?? hf.addEosToken
448
528
  ?? configuredAddEosToken
529
+ ?? inferredFlags.addEosToken
449
530
  ?? runtimeDefaults.addEosToken;
450
531
  if (this.addBosToken && this.specialTokens.bos == null) {
451
532
  throw new Error('[Tokenizer] addBosToken is enabled but bos token is missing.');
@@ -621,15 +702,18 @@ export class BundledTokenizer extends BaseTokenizer {
621
702
  const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(tokenizerJson.pre_tokenizer);
622
703
  const configuredAddBosToken = this.addBosToken;
623
704
  const configuredAddEosToken = this.addEosToken;
705
+ const inferredFlags = inferBundledTokenizerBehaviorFlags(tokenizerJson, this.specialTokens);
624
706
  this.addBosToken =
625
707
  tokenizerJson.addBosToken
626
708
  ?? tokenizerJson.add_bos_token
627
709
  ?? configuredAddBosToken
710
+ ?? inferredFlags.addBosToken
628
711
  ?? runtimeDefaults.addBosToken;
629
712
  this.addEosToken =
630
713
  tokenizerJson.addEosToken
631
714
  ?? tokenizerJson.add_eos_token
632
715
  ?? configuredAddEosToken
716
+ ?? inferredFlags.addEosToken
633
717
  ?? runtimeDefaults.addEosToken;
634
718
  if (this.addBosToken && this.specialTokens.bos == null) {
635
719
  throw new Error('[Tokenizer] addBosToken is enabled but bos token is missing.');
@@ -1006,4 +1090,30 @@ export class BundledTokenizer extends BaseTokenizer {
1006
1090
  // Only trim when requested (not during streaming where spaces matter)
1007
1091
  return trim ? result.trim() : result;
1008
1092
  }
1093
+
1094
+ getHotTokenIds(limit) {
1095
+ const resolvedLimit = Math.trunc(Number(limit));
1096
+ if (!Number.isFinite(resolvedLimit) || resolvedLimit <= 0) {
1097
+ return [];
1098
+ }
1099
+ if (this.#type === 'bpe' && (!Array.isArray(this.#scores) || this.#scores.length === 0)) {
1100
+ return rankFallbackBpeHotTokenIds(
1101
+ this.#reverseVocab,
1102
+ resolvedLimit,
1103
+ (tokenId) => this.isSpecialToken(tokenId)
1104
+ );
1105
+ }
1106
+ if (!Array.isArray(this.#scores) || this.#scores.length === 0) {
1107
+ return null;
1108
+ }
1109
+ const ranked = [];
1110
+ for (let id = 0; id < this.#scores.length; id += 1) {
1111
+ if (this.isSpecialToken(id)) {
1112
+ continue;
1113
+ }
1114
+ ranked.push({ id, score: Number(this.#scores[id] ?? 0) });
1115
+ }
1116
+ ranked.sort((a, b) => b.score - a.score || a.id - b.id);
1117
+ return ranked.slice(0, resolvedLimit).map((entry) => entry.id);
1118
+ }
1009
1119
  }
@@ -25,4 +25,6 @@ export declare class SentencePieceTokenizer extends BaseTokenizer {
25
25
  encode(text: string): number[];
26
26
 
27
27
  decode(ids: number[], skipSpecialTokens?: boolean, trim?: boolean): string;
28
+
29
+ getHotTokenIds(limit: number): number[] | null;
28
30
  }
@@ -58,6 +58,22 @@ export class SentencePieceTokenizer extends BaseTokenizer {
58
58
  }
59
59
  }
60
60
 
61
+ getHotTokenIds(limit) {
62
+ const resolvedLimit = Math.trunc(Number(limit));
63
+ if (!Number.isFinite(resolvedLimit) || resolvedLimit <= 0) {
64
+ return [];
65
+ }
66
+ const ranked = [];
67
+ for (const piece of this.#pieces.values()) {
68
+ if (!piece || this.isSpecialToken(piece.id)) {
69
+ continue;
70
+ }
71
+ ranked.push({ id: piece.id, score: Number(piece.score ?? 0) });
72
+ }
73
+ ranked.sort((a, b) => b.score - a.score || a.id - b.id);
74
+ return ranked.slice(0, resolvedLimit).map((entry) => entry.id);
75
+ }
76
+
61
77
 
62
78
  async #parseModelProto(buffer) {
63
79
  const view = new DataView(buffer);
@@ -67,6 +67,8 @@ export interface TokenizerBackend {
67
67
  decode(tokens: number[], skipSpecial?: boolean, trim?: boolean): string;
68
68
  /** Get vocabulary size */
69
69
  getVocabSize(): number;
70
+ /** Get high-priority token IDs for bounded runtime hot caches, or null when unavailable */
71
+ getHotTokenIds?(limit: number): number[] | null;
70
72
  /** Get special token IDs */
71
73
  specialTokens: SpecialTokens;
72
74
  }
@@ -13,6 +13,7 @@
13
13
  import type { MemoryCapabilities } from '../memory/capability.js';
14
14
  import type { HeapManager } from '../memory/heap-manager.js';
15
15
  import type { RDRRManifest } from '../formats/rdrr/index.js';
16
+ import type { RuntimeModelContract } from '../inference/runtime-model.js';
16
17
  import type { WeightBuffer, WeightLayout, CpuWeightBuffer } from '../gpu/weight-buffer.js';
17
18
  import type { ExpertCache, CacheStats } from './experts/expert-cache.js';
18
19
  import type { ExpertWeights } from './weights.js';
@@ -21,6 +22,7 @@ import type { LoadedEmbeddingPostprocessor } from './final-weights-loader.js';
21
22
  import type {
22
23
  TensorLocation,
23
24
  LayerWeights,
25
+ PerLayerInputWeights,
24
26
  LoadProgress,
25
27
  LoadOptions,
26
28
  CustomShardLoader,
@@ -33,6 +35,7 @@ import type {
33
35
  import type { ShardCache } from './shard-cache.js';
34
36
  import type { LoadingConfigSchema } from '../config/schema/loading.schema.js';
35
37
  import type { LoaderDebugConfigSchema } from '../config/schema/debug.schema.js';
38
+ import type { ExecutionV1PerLayerInputsSessionSchema } from '../config/schema/execution-v1.schema.js';
36
39
 
37
40
  // Re-export types for backward compatibility
38
41
  export type {
@@ -59,7 +62,7 @@ export declare class DopplerLoader {
59
62
  isUnifiedMemory: boolean;
60
63
 
61
64
  // Manifest and model info
62
- manifest: RDRRManifest | null;
65
+ manifest: (RDRRManifest | RuntimeModelContract) | null;
63
66
  modelId: string | null;
64
67
  isMoE: boolean;
65
68
 
@@ -71,6 +74,7 @@ export declare class DopplerLoader {
71
74
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
72
75
  finalNorm: GPUBuffer | Float32Array | null;
73
76
  embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
77
+ perLayerInputWeights: PerLayerInputWeights | null;
74
78
 
75
79
  // Memory management
76
80
  heapManager: HeapManager | null;
@@ -92,6 +96,7 @@ export declare class DopplerLoader {
92
96
  // Q4K layout: 'row' = fused kernel compatible (fast), 'col' = dequant fallback
93
97
  q4kLayout: 'row' | 'col' | null;
94
98
  keepF32Weights: boolean;
99
+ q4kMaterializationMode: 'dense' | 'fused' | 'mixed';
95
100
 
96
101
  constructor(loadingConfig?: LoadingConfigSchema);
97
102
 
@@ -101,15 +106,17 @@ export declare class DopplerLoader {
101
106
 
102
107
  setLoaderDebugConfig(loaderDebug: LoaderDebugConfigSchema | null): void;
103
108
 
109
+ setPerLayerInputSession(sessionConfig: ExecutionV1PerLayerInputsSessionSchema | null): void;
110
+
104
111
  setCustomShardLoader(loadShardFn: CustomShardLoader, options?: CustomShardLoaderOptions): void;
105
112
 
106
113
  setTensorsJsonUrl(url: string | null): void;
107
114
 
108
115
  init(): Promise<void>;
109
116
 
110
- setManifest(manifest: RDRRManifest): void;
117
+ setManifest(manifest: RDRRManifest | RuntimeModelContract): void;
111
118
 
112
- loadLoRAWeights(manifest: RDRRManifest): Promise<LoRAAdapter>;
119
+ loadLoRAWeights(manifest: RDRRManifest | RuntimeModelContract): Promise<LoRAAdapter>;
113
120
 
114
121
  load(modelId: string, options?: LoadOptions): Promise<ModelConfig>;
115
122