doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,47 +1,34 @@
1
1
  import { createReadStream } from 'node:fs';
2
2
  import fs from 'node:fs/promises';
3
3
  import { createHash } from 'node:crypto';
4
+ import { totalmem } from 'node:os';
4
5
  import path from 'node:path';
5
6
  import {
6
7
  HEADER_READ_SIZE,
7
- createConverterConfig,
8
- DEFAULT_EXECUTION_V1_SESSION_DEFAULTS,
9
8
  } from '../config/schema/index.js';
9
+ import { getRuntimeConfig } from '../config/runtime.js';
10
10
  import { extractArchitecture } from '../converter/core.js';
11
- import {
12
- inferSourceWeightQuantization,
13
- resolveConversionPlan,
14
- resolveConvertedModelId,
15
- } from '../converter/conversion-plan.js';
16
11
  import { parseGGUFModel } from '../converter/parsers/gguf.js';
17
12
  import { parseTransformerModel } from '../converter/parsers/transformer.js';
18
13
  import { parseGGUFHeader } from '../formats/gguf/types.js';
19
14
  import { parseSafetensorsHeader } from '../formats/safetensors/types.js';
15
+ import { parseTFLiteFromSource } from '../formats/tflite/types.js';
20
16
  import { log } from '../debug/index.js';
17
+ import { formatBytes } from '../storage/quota.js';
21
18
  import {
22
- buildSourceRuntimeBundle,
23
19
  createSourceStorageContext,
24
20
  } from './source-runtime-bundle.js';
25
-
26
- const SUPPORTED_SOURCE_DTYPES = new Set([
27
- 'F32',
28
- 'F16',
29
- 'BF16',
30
- 'Q4_K',
31
- 'Q4_K_M',
32
- 'Q6_K',
33
- ]);
34
-
35
- const SOURCE_RUNTIME_EXECUTION_OVERRIDE = {
36
- steps: [],
37
- };
38
-
39
- function cloneSessionDefaults() {
40
- if (typeof structuredClone === 'function') {
41
- return structuredClone(DEFAULT_EXECUTION_V1_SESSION_DEFAULTS);
42
- }
43
- return JSON.parse(JSON.stringify(DEFAULT_EXECUTION_V1_SESSION_DEFAULTS));
44
- }
21
+ import {
22
+ inferSourceQuantizationForSourceRuntime,
23
+ resolveSourceRuntimeBundleFromParsedArtifact,
24
+ } from './source-artifact-adapter.js';
25
+ import {
26
+ LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
27
+ LITERT_PACKAGE_SOURCE_KIND_TASK,
28
+ appendLiteRTPackageVirtualFiles,
29
+ resolveLiteRTPackageParsedArtifact,
30
+ } from './litert-package-runtime.js';
31
+ const MAX_NODE_READ_BYTES = 64 * 1024 * 1024;
45
32
 
46
33
  function toArrayBuffer(value, label) {
47
34
  if (value instanceof ArrayBuffer) {
@@ -61,23 +48,16 @@ function isGgufPath(filePath) {
61
48
  return String(filePath || '').toLowerCase().endsWith('.gguf');
62
49
  }
63
50
 
64
- function resolveModelIdHint(requestedModelId, plan, sourceKind, sourcePath) {
65
- const explicit = String(requestedModelId || '').trim();
66
- if (explicit) {
67
- return resolveConvertedModelId({
68
- explicitModelId: explicit,
69
- converterConfig: null,
70
- detectedModelId: explicit,
71
- quantizationInfo: plan.quantizationInfo,
72
- }) || explicit;
73
- }
74
- const basename = path.basename(sourcePath, path.extname(sourcePath)) || sourceKind;
75
- return resolveConvertedModelId({
76
- explicitModelId: basename,
77
- converterConfig: null,
78
- detectedModelId: basename,
79
- quantizationInfo: plan.quantizationInfo,
80
- }) || basename;
51
+ function isTflitePath(filePath) {
52
+ return String(filePath || '').toLowerCase().endsWith('.tflite');
53
+ }
54
+
55
+ function isLiteRTTaskPath(filePath) {
56
+ return String(filePath || '').toLowerCase().endsWith('.task');
57
+ }
58
+
59
+ function isLiteRTLMPath(filePath) {
60
+ return String(filePath || '').toLowerCase().endsWith('.litertlm');
81
61
  }
82
62
 
83
63
  async function getPathStats(targetPath, label) {
@@ -115,16 +95,6 @@ async function readJson(filePath, label) {
115
95
  }
116
96
  }
117
97
 
118
- async function readFileBytes(filePath, label) {
119
- try {
120
- const bytes = await fs.readFile(filePath);
121
- return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
122
- } catch (error) {
123
- const message = error instanceof Error ? error.message : String(error);
124
- throw new Error(`Failed to read ${label} "${filePath}": ${message}`);
125
- }
126
- }
127
-
128
98
  async function readRange(filePath, offset, length) {
129
99
  if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
130
100
  return new ArrayBuffer(0);
@@ -140,7 +110,8 @@ async function readRange(filePath, offset, length) {
140
110
  const out = Buffer.allocUnsafe(end - start);
141
111
  let pos = 0;
142
112
  while (pos < out.length) {
143
- const { bytesRead } = await handle.read(out, pos, out.length - pos, start + pos);
113
+ const nextChunkBytes = Math.min(out.length - pos, MAX_NODE_READ_BYTES);
114
+ const { bytesRead } = await handle.read(out, pos, nextChunkBytes, start + pos);
144
115
  if (bytesRead === 0) break;
145
116
  pos += bytesRead;
146
117
  }
@@ -166,6 +137,106 @@ async function readSafetensorsHeaderFromFile(filePath) {
166
137
  );
167
138
  }
168
139
 
140
+ async function collectTokenizerAssets(inputDir) {
141
+ const tokenizerJsonPath = path.join(inputDir, 'tokenizer.json');
142
+ const tokenizerConfigPath = path.join(inputDir, 'tokenizer_config.json');
143
+ const tokenizerModelPath = path.join(inputDir, 'tokenizer.model');
144
+ const tokenizerJson = await fileExists(tokenizerJsonPath)
145
+ ? await readJson(tokenizerJsonPath, 'tokenizer.json')
146
+ : null;
147
+ const tokenizerConfig = await fileExists(tokenizerConfigPath)
148
+ ? await readJson(tokenizerConfigPath, 'tokenizer_config.json')
149
+ : null;
150
+ const hasTokenizerModel = await fileExists(tokenizerModelPath);
151
+ return {
152
+ tokenizerJson,
153
+ tokenizerConfig,
154
+ tokenizerModelName: hasTokenizerModel ? 'tokenizer.model' : null,
155
+ tokenizerJsonPath,
156
+ tokenizerConfigPath,
157
+ tokenizerModelPath: hasTokenizerModel ? tokenizerModelPath : null,
158
+ };
159
+ }
160
+
161
+ async function buildTokenizerAuxiliaryFiles(tokenizerAssets) {
162
+ const auxiliaryFiles = [];
163
+ if (tokenizerAssets.tokenizerJson) {
164
+ auxiliaryFiles.push({
165
+ path: tokenizerAssets.tokenizerJsonPath,
166
+ size: Number((await getPathStats(tokenizerAssets.tokenizerJsonPath, 'tokenizer.json')).size),
167
+ kind: 'tokenizer_json',
168
+ });
169
+ }
170
+ if (tokenizerAssets.tokenizerConfig) {
171
+ auxiliaryFiles.push({
172
+ path: tokenizerAssets.tokenizerConfigPath,
173
+ size: Number((await getPathStats(tokenizerAssets.tokenizerConfigPath, 'tokenizer_config.json')).size),
174
+ kind: 'tokenizer_config',
175
+ });
176
+ }
177
+ if (tokenizerAssets.tokenizerModelPath) {
178
+ auxiliaryFiles.push({
179
+ path: tokenizerAssets.tokenizerModelPath,
180
+ size: Number((await getPathStats(tokenizerAssets.tokenizerModelPath, 'tokenizer.model')).size),
181
+ kind: 'tokenizer_model',
182
+ });
183
+ }
184
+ return auxiliaryFiles;
185
+ }
186
+
187
+ function buildPackageTokenizerVirtualFiles(tokenizerAssets) {
188
+ const virtualFiles = [];
189
+ if (tokenizerAssets.tokenizerJson) {
190
+ virtualFiles.push({
191
+ path: 'tokenizer.json',
192
+ offset: 0,
193
+ size: 0,
194
+ kind: 'tokenizer_json',
195
+ externalPath: tokenizerAssets.tokenizerJsonPath,
196
+ });
197
+ }
198
+ if (tokenizerAssets.tokenizerConfig) {
199
+ virtualFiles.push({
200
+ path: 'tokenizer_config.json',
201
+ offset: 0,
202
+ size: 0,
203
+ kind: 'tokenizer_config',
204
+ externalPath: tokenizerAssets.tokenizerConfigPath,
205
+ });
206
+ }
207
+ if (tokenizerAssets.tokenizerModelPath) {
208
+ virtualFiles.push({
209
+ path: 'TOKENIZER_MODEL',
210
+ offset: 0,
211
+ size: 0,
212
+ kind: 'tokenizer_model',
213
+ externalPath: tokenizerAssets.tokenizerModelPath,
214
+ });
215
+ }
216
+ return virtualFiles;
217
+ }
218
+
219
+ function applyPackageTokenizerAssets(parsedArtifact, tokenizerAssets) {
220
+ const next = {
221
+ ...parsedArtifact,
222
+ };
223
+ if (tokenizerAssets.tokenizerJson && next.tokenizerJson == null) {
224
+ next.tokenizerJson = tokenizerAssets.tokenizerJson;
225
+ next.tokenizerJsonPath = 'tokenizer.json';
226
+ }
227
+ if (tokenizerAssets.tokenizerConfig && next.tokenizerConfigPath == null) {
228
+ next.tokenizerConfigPath = 'tokenizer_config.json';
229
+ }
230
+ if (tokenizerAssets.tokenizerConfig && next.tokenizerConfig == null) {
231
+ next.tokenizerConfig = tokenizerAssets.tokenizerConfig;
232
+ }
233
+ if (tokenizerAssets.tokenizerModelPath && next.tokenizerModelPath == null) {
234
+ next.tokenizerModelName = 'TOKENIZER_MODEL';
235
+ next.tokenizerModelPath = 'TOKENIZER_MODEL';
236
+ }
237
+ return next;
238
+ }
239
+
169
240
  async function parseSafetensorsInput(inputDir) {
170
241
  const configPath = path.join(inputDir, 'config.json');
171
242
  if (!(await fileExists(configPath))) {
@@ -214,17 +285,7 @@ async function parseSafetensorsInput(inputDir) {
214
285
  const architectureHint = parsedTransformer.architectureHint;
215
286
  const embeddingPostprocessor = parsedTransformer.embeddingPostprocessor ?? null;
216
287
  const architecture = extractArchitecture(config, null);
217
-
218
- const tokenizerJsonPath = path.join(inputDir, 'tokenizer.json');
219
- const tokenizerConfigPath = path.join(inputDir, 'tokenizer_config.json');
220
- const tokenizerModelPath = path.join(inputDir, 'tokenizer.model');
221
- const tokenizerJson = await fileExists(tokenizerJsonPath)
222
- ? await readJson(tokenizerJsonPath, 'tokenizer.json')
223
- : null;
224
- const tokenizerConfig = await fileExists(tokenizerConfigPath)
225
- ? await readJson(tokenizerConfigPath, 'tokenizer_config.json')
226
- : null;
227
- const hasTokenizerModel = await fileExists(tokenizerModelPath);
288
+ const tokenizerAssets = await collectTokenizerAssets(inputDir);
228
289
 
229
290
  const sourceFiles = [];
230
291
  const uniquePaths = new Set(tensors.map((tensor) => normalizePath(tensor.sourcePath)));
@@ -241,27 +302,7 @@ async function parseSafetensorsInput(inputDir) {
241
302
  kind: 'safetensors_index',
242
303
  }]
243
304
  : []),
244
- ...(tokenizerJson
245
- ? [{
246
- path: tokenizerJsonPath,
247
- size: Number((await getPathStats(tokenizerJsonPath, 'tokenizer.json')).size),
248
- kind: 'tokenizer_json',
249
- }]
250
- : []),
251
- ...(tokenizerConfig
252
- ? [{
253
- path: tokenizerConfigPath,
254
- size: Number((await getPathStats(tokenizerConfigPath, 'tokenizer_config.json')).size),
255
- kind: 'tokenizer_config',
256
- }]
257
- : []),
258
- ...(hasTokenizerModel
259
- ? [{
260
- path: tokenizerModelPath,
261
- size: Number((await getPathStats(tokenizerModelPath, 'tokenizer.model')).size),
262
- kind: 'tokenizer_model',
263
- }]
264
- : []),
305
+ ...await buildTokenizerAuxiliaryFiles(tokenizerAssets),
265
306
  ];
266
307
 
267
308
  return {
@@ -273,18 +314,226 @@ async function parseSafetensorsInput(inputDir) {
273
314
  architectureHint,
274
315
  embeddingPostprocessor,
275
316
  architecture,
276
- sourceQuantization: inferSourceQuantizationForSourceRuntime(tensors, 'safetensors'),
277
- tokenizerJson,
278
- tokenizerConfig,
279
- tokenizerModelName: hasTokenizerModel ? 'tokenizer.model' : null,
280
- tokenizerJsonPath: tokenizerJsonPath,
281
- tokenizerConfigPath: tokenizerConfigPath,
282
- tokenizerModelPath: hasTokenizerModel ? tokenizerModelPath : null,
317
+ sourceQuantization: inferSourceQuantizationForSourceRuntime(tensors, 'safetensors', {
318
+ logCategory: 'NodeSourceRuntime',
319
+ }),
320
+ tokenizerJson: tokenizerAssets.tokenizerJson,
321
+ tokenizerConfig: tokenizerAssets.tokenizerConfig,
322
+ tokenizerModelName: tokenizerAssets.tokenizerModelName,
323
+ tokenizerJsonPath: tokenizerAssets.tokenizerJsonPath,
324
+ tokenizerConfigPath: tokenizerAssets.tokenizerConfigPath,
325
+ tokenizerModelPath: tokenizerAssets.tokenizerModelPath,
283
326
  sourceFiles,
284
327
  auxiliaryFiles,
285
328
  };
286
329
  }
287
330
 
331
+ async function parseTfliteInput(tflitePath) {
332
+ const tfliteStats = await getPathStats(tflitePath, 'TFLite file');
333
+ const inputDir = path.dirname(tflitePath);
334
+ const configPath = path.join(inputDir, 'config.json');
335
+ if (!(await fileExists(configPath))) {
336
+ throw new Error(
337
+ `node source runtime: config.json is required next to TFLite source "${tflitePath}".`
338
+ );
339
+ }
340
+
341
+ const config = await readJson(configPath, 'config.json');
342
+ const parsedTFLite = await parseTFLiteFromSource({
343
+ name: path.basename(tflitePath),
344
+ size: Number(tfliteStats.size),
345
+ async readRange(offset, length) {
346
+ return readRange(tflitePath, offset, length);
347
+ },
348
+ });
349
+ const tokenizerAssets = await collectTokenizerAssets(inputDir);
350
+ const tensors = parsedTFLite.tensors.map((tensor) => ({
351
+ ...tensor,
352
+ sourcePath: tflitePath,
353
+ }));
354
+ const architecture = extractArchitecture(config, null);
355
+ const architectureHint = config.architectures?.[0] ?? config.model_type ?? '';
356
+ const auxiliaryFiles = [
357
+ { path: configPath, size: Number((await getPathStats(configPath, 'config.json')).size), kind: 'config' },
358
+ ...await buildTokenizerAuxiliaryFiles(tokenizerAssets),
359
+ ];
360
+
361
+ return {
362
+ sourceKind: 'tflite',
363
+ sourceRoot: inputDir,
364
+ sourcePathForModelId: tflitePath,
365
+ config,
366
+ tensors,
367
+ architectureHint,
368
+ embeddingPostprocessor: null,
369
+ architecture,
370
+ sourceQuantization: parsedTFLite.sourceQuantization,
371
+ tokenizerJson: tokenizerAssets.tokenizerJson,
372
+ tokenizerConfig: tokenizerAssets.tokenizerConfig,
373
+ tokenizerModelName: tokenizerAssets.tokenizerModelName,
374
+ tokenizerJsonPath: tokenizerAssets.tokenizerJsonPath,
375
+ tokenizerConfigPath: tokenizerAssets.tokenizerConfigPath,
376
+ tokenizerModelPath: tokenizerAssets.tokenizerModelPath,
377
+ sourceFiles: [{ path: tflitePath, size: Number(tfliteStats.size) }],
378
+ auxiliaryFiles,
379
+ };
380
+ }
381
+
382
+ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
383
+ const virtualFileMap = new Map(
384
+ (Array.isArray(virtualFiles) ? virtualFiles : []).map((entry) => [entry.path, entry])
385
+ );
386
+
387
+ const resolveVirtualFile = (virtualPath) => {
388
+ const normalized = normalizePath(virtualPath);
389
+ const entry = virtualFileMap.get(normalized);
390
+ if (!entry) {
391
+ throw new Error(`node source runtime: missing package asset "${virtualPath}".`);
392
+ }
393
+ return entry;
394
+ };
395
+
396
+ const readRangeFromVirtualFile = async (virtualPath, offset, length) => {
397
+ const entry = resolveVirtualFile(virtualPath);
398
+ if (entry.externalPath) {
399
+ return readRange(entry.externalPath, offset, length);
400
+ }
401
+ const start = Math.max(0, Math.floor(Number(offset) || 0));
402
+ const requested = Math.max(0, Math.floor(Number(length) || 0));
403
+ const available = Math.max(0, entry.size - start);
404
+ const readLength = Math.min(available, requested);
405
+ if (readLength <= 0) {
406
+ return new ArrayBuffer(0);
407
+ }
408
+ return readRange(packagePath, entry.offset + start, readLength);
409
+ };
410
+
411
+ const streamRange = async function* (virtualPath, offset, length, options = {}) {
412
+ const entry = resolveVirtualFile(virtualPath);
413
+ if (entry.externalPath) {
414
+ const stats = await getPathStats(entry.externalPath, `package sidecar (${entry.externalPath})`);
415
+ const start = Math.max(0, Math.floor(Number(offset) || 0));
416
+ const requested = Math.max(0, Math.floor(Number(length) || 0));
417
+ const available = Math.max(0, Number(stats.size) - start);
418
+ const readLength = Math.min(available, requested);
419
+ if (readLength <= 0) {
420
+ return;
421
+ }
422
+ const chunkBytesRaw = Number(options?.chunkBytes);
423
+ const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
424
+ ? Math.floor(chunkBytesRaw)
425
+ : MAX_NODE_READ_BYTES;
426
+ const stream = createReadStream(entry.externalPath, {
427
+ start,
428
+ end: start + readLength - 1,
429
+ highWaterMark,
430
+ });
431
+ for await (const chunk of stream) {
432
+ yield chunk;
433
+ }
434
+ return;
435
+ }
436
+ const start = Math.max(0, Math.floor(Number(offset) || 0));
437
+ const requested = Math.max(0, Math.floor(Number(length) || 0));
438
+ const available = Math.max(0, entry.size - start);
439
+ const readLength = Math.min(available, requested);
440
+ if (readLength <= 0) {
441
+ return;
442
+ }
443
+ const chunkBytesRaw = Number(options?.chunkBytes);
444
+ const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
445
+ ? Math.floor(chunkBytesRaw)
446
+ : MAX_NODE_READ_BYTES;
447
+ const stream = createReadStream(packagePath, {
448
+ start: entry.offset + start,
449
+ end: entry.offset + start + readLength - 1,
450
+ highWaterMark,
451
+ });
452
+ for await (const chunk of stream) {
453
+ yield chunk;
454
+ }
455
+ };
456
+
457
+ const readText = async (virtualPath) => {
458
+ const entry = resolveVirtualFile(virtualPath);
459
+ const bytes = entry.externalPath
460
+ ? await fs.readFile(entry.externalPath)
461
+ : await readRange(packagePath, entry.offset, entry.size);
462
+ return new TextDecoder().decode(bytes);
463
+ };
464
+
465
+ const readBinary = async (virtualPath) => {
466
+ const entry = resolveVirtualFile(virtualPath);
467
+ return entry.externalPath
468
+ ? fs.readFile(entry.externalPath)
469
+ : readRange(packagePath, entry.offset, entry.size);
470
+ };
471
+
472
+ return {
473
+ virtualFileMap,
474
+ readRange: readRangeFromVirtualFile,
475
+ streamRange,
476
+ readText,
477
+ readBinary,
478
+ };
479
+ }
480
+
481
+ async function addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm) {
482
+ const readers = createNodeVirtualFileReaders(packagePath, virtualFiles);
483
+ const hashedEntries = [];
484
+ for (const entry of Array.isArray(entries) ? entries : []) {
485
+ const virtualPath = normalizePath(entry?.path);
486
+ if (!virtualPath) {
487
+ continue;
488
+ }
489
+ const descriptor = readers.virtualFileMap.get(virtualPath);
490
+ if (!descriptor) {
491
+ throw new Error(`node source runtime: missing virtual package asset "${virtualPath}" for hashing.`);
492
+ }
493
+ const hash = createHash(hashAlgorithm);
494
+ for await (const chunk of readers.streamRange(virtualPath, 0, descriptor.size)) {
495
+ hash.update(chunk);
496
+ }
497
+ hashedEntries.push({
498
+ ...entry,
499
+ path: virtualPath,
500
+ size: Number.isFinite(entry?.size) ? Math.max(0, Math.floor(Number(entry.size))) : descriptor.size,
501
+ hash: hash.digest('hex'),
502
+ hashAlgorithm,
503
+ });
504
+ }
505
+ return hashedEntries;
506
+ }
507
+
508
+ async function parseLiteRTPackageInput(packagePath, sourceKind) {
509
+ const stats = await getPathStats(packagePath, `LiteRT package (${sourceKind})`);
510
+ const resolved = await resolveLiteRTPackageParsedArtifact({
511
+ sourceKind,
512
+ sourcePathForModelId: packagePath,
513
+ source: {
514
+ name: path.basename(packagePath),
515
+ size: Number(stats.size),
516
+ async readRange(offset, length) {
517
+ return readRange(packagePath, offset, length);
518
+ },
519
+ },
520
+ });
521
+ const tokenizerAssets = await collectTokenizerAssets(path.dirname(packagePath));
522
+ const virtualFiles = appendLiteRTPackageVirtualFiles(
523
+ resolved.virtualFiles,
524
+ buildPackageTokenizerVirtualFiles(tokenizerAssets)
525
+ );
526
+ const parsedArtifact = applyPackageTokenizerAssets(resolved.parsedArtifact, tokenizerAssets);
527
+ return {
528
+ ...parsedArtifact,
529
+ sourceRoot: packagePath,
530
+ storageReaders: createNodeVirtualFileReaders(packagePath, virtualFiles),
531
+ async hashFileEntries(entries, hashAlgorithm) {
532
+ return addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm);
533
+ },
534
+ };
535
+ }
536
+
288
537
  async function parseGgufInput(ggufPath) {
289
538
  const ggufStats = await getPathStats(ggufPath, 'GGUF file');
290
539
  const fileSize = Number(ggufStats.size);
@@ -338,7 +587,9 @@ async function parseGgufInput(ggufPath) {
338
587
  tensors,
339
588
  architectureHint: parsed.architecture,
340
589
  architecture: extractArchitecture({}, parsed.config || {}),
341
- sourceQuantization: parsed.quantization ?? inferSourceQuantizationForSourceRuntime(tensors, 'gguf'),
590
+ sourceQuantization: parsed.quantization ?? inferSourceQuantizationForSourceRuntime(tensors, 'gguf', {
591
+ logCategory: 'NodeSourceRuntime',
592
+ }),
342
593
  tokenizerJson: null,
343
594
  tokenizerConfig: null,
344
595
  tokenizerModelName: null,
@@ -350,54 +601,31 @@ async function parseGgufInput(ggufPath) {
350
601
  };
351
602
  }
352
603
 
353
- function assertSupportedSourceDtypes(tensors, sourceKind) {
354
- const unsupported = new Set();
355
- for (const tensor of tensors) {
356
- const dtype = String(tensor?.dtype || '').trim().toUpperCase();
357
- if (!dtype) {
358
- unsupported.add('(empty)');
359
- continue;
360
- }
361
- if (!SUPPORTED_SOURCE_DTYPES.has(dtype)) {
362
- unsupported.add(dtype);
604
+ function buildNodeFileReaders() {
605
+ const readRangeFromFile = async (filePath, offset, length) => readRange(filePath, offset, length);
606
+ const streamRange = async function* (filePath, offset, length, options = {}) {
607
+ if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
608
+ return;
363
609
  }
364
- }
365
- if (unsupported.size > 0) {
366
- throw new Error(
367
- `Unsupported ${sourceKind} tensor dtypes for direct-source runtime: ` +
368
- `${Array.from(unsupported).sort((a, b) => a.localeCompare(b)).join(', ')}. ` +
369
- 'Convert to RDRR first for this model.'
370
- );
371
- }
372
- }
373
-
374
- function inferSourceQuantizationForSourceRuntime(tensors, sourceKind) {
375
- try {
376
- return inferSourceWeightQuantization(tensors);
377
- } catch (error) {
378
- const dtypes = new Set();
379
- for (const tensor of tensors) {
380
- const dtype = String(tensor?.dtype || '').trim().toUpperCase();
381
- if (dtype) dtypes.add(dtype);
610
+ const stats = await getPathStats(filePath, `stream source asset (${filePath})`);
611
+ const start = Math.max(0, Math.floor(offset));
612
+ const end = Math.min(Number(stats.size), start + Math.floor(length));
613
+ if (end <= start) {
614
+ return;
382
615
  }
383
- const hasLowPrecision = dtypes.has('F16') || dtypes.has('BF16');
384
- const onlyLowAndF32 = dtypes.size > 0 && Array.from(dtypes).every(
385
- (dtype) => dtype === 'F16' || dtype === 'BF16' || dtype === 'F32'
386
- );
387
- if (hasLowPrecision && onlyLowAndF32) {
388
- log.warn(
389
- 'NodeSourceRuntime',
390
- `Mixed ${sourceKind} tensor dtypes detected (${Array.from(dtypes).sort((a, b) => a.localeCompare(b)).join(', ')}). ` +
391
- 'Using F32 source quantization for direct-source parity.'
392
- );
393
- return 'F32';
616
+ const chunkBytesRaw = Number(options?.chunkBytes);
617
+ const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
618
+ ? Math.floor(chunkBytesRaw)
619
+ : MAX_NODE_READ_BYTES;
620
+ const stream = createReadStream(filePath, {
621
+ start,
622
+ end: end - 1,
623
+ highWaterMark,
624
+ });
625
+ for await (const chunk of stream) {
626
+ yield chunk;
394
627
  }
395
- throw error;
396
- }
397
- }
398
-
399
- function buildNodeFileReaders() {
400
- const readRangeFromFile = async (filePath, offset, length) => readRange(filePath, offset, length);
628
+ };
401
629
  const readText = async (filePath) => {
402
630
  try {
403
631
  return await fs.readFile(filePath, 'utf8');
@@ -414,44 +642,12 @@ function buildNodeFileReaders() {
414
642
  };
415
643
  return {
416
644
  readRange: readRangeFromFile,
645
+ streamRange,
417
646
  readText,
418
647
  readBinary,
419
648
  };
420
649
  }
421
650
 
422
- // Source dtype → compute precision mapping for source-runtime inference.
423
- // BF16/F32 sources require f32 compute (BF16 has no native WebGPU support).
424
- // Quantized formats require f32 compute for dequantization accuracy.
425
- // F16 sources can use f16 compute directly.
426
- const SOURCE_QUANT_COMPUTE_MAP = {
427
- 'F16': 'f16',
428
- 'BF16': 'f32',
429
- 'F32': 'f32',
430
- 'Q4_K': 'f32',
431
- 'Q4_K_M': 'f32',
432
- 'Q6_K': 'f32',
433
- };
434
- const SOURCE_COMPUTE_DEFAULT = 'f16';
435
-
436
- function resolveSourceRuntimeComputePrecision(tensors, sourceQuantization) {
437
- const dtypes = new Set();
438
- for (const tensor of Array.isArray(tensors) ? tensors : []) {
439
- const dtype = String(tensor?.dtype || '').trim().toUpperCase();
440
- if (dtype) {
441
- dtypes.add(dtype);
442
- }
443
- }
444
- // If any tensor requires f32 compute, use f32 for all.
445
- for (const dtype of dtypes) {
446
- if (SOURCE_QUANT_COMPUTE_MAP[dtype] === 'f32') {
447
- return 'f32';
448
- }
449
- }
450
-
451
- const normalized = String(sourceQuantization || '').trim().toUpperCase();
452
- return SOURCE_QUANT_COMPUTE_MAP[normalized] ?? SOURCE_COMPUTE_DEFAULT;
453
- }
454
-
455
651
  async function addHashesToFileEntries(entries, hashAlgorithm) {
456
652
  const normalized = [];
457
653
  for (const entry of Array.isArray(entries) ? entries : []) {
@@ -487,6 +683,115 @@ async function computeFileHash(filePath, hashAlgorithm) {
487
683
  });
488
684
  }
489
685
 
686
+ function resolveLoadingConfig(runtimeConfig) {
687
+ const loadingConfig = runtimeConfig?.loading ?? getRuntimeConfig().loading;
688
+ if (!loadingConfig || typeof loadingConfig !== 'object') {
689
+ throw new Error('node source runtime: runtime.loading is required.');
690
+ }
691
+ return loadingConfig;
692
+ }
693
+
694
+ function resolveMemoryBudgetConfig(loadingConfig) {
695
+ const budget = loadingConfig?.memoryManagement?.budget;
696
+ if (!budget || typeof budget !== 'object') {
697
+ throw new Error('node source runtime: runtime.loading.memoryManagement.budget is required.');
698
+ }
699
+ return budget;
700
+ }
701
+
702
+ function resolveResidentBudgetBytes(loadingConfig) {
703
+ const budget = resolveMemoryBudgetConfig(loadingConfig);
704
+ if (budget.enabled !== true) {
705
+ return null;
706
+ }
707
+
708
+ const explicitMaxResidentBytes = budget.maxResidentBytes;
709
+ if (explicitMaxResidentBytes != null) {
710
+ const normalized = Number(explicitMaxResidentBytes);
711
+ if (!Number.isFinite(normalized) || normalized <= 0) {
712
+ throw new Error(
713
+ 'node source runtime: runtime.loading.memoryManagement.budget.maxResidentBytes ' +
714
+ 'must be a positive number or null.'
715
+ );
716
+ }
717
+ return Math.floor(normalized);
718
+ }
719
+
720
+ const systemMemoryFraction = Number(budget.systemMemoryFraction);
721
+ const reserveBytes = Number(budget.reserveBytes);
722
+ const minimumBudgetBytes = Number(budget.minimumBudgetBytes);
723
+ if (!Number.isFinite(systemMemoryFraction) || systemMemoryFraction <= 0 || systemMemoryFraction > 1) {
724
+ throw new Error(
725
+ 'node source runtime: runtime.loading.memoryManagement.budget.systemMemoryFraction ' +
726
+ 'must be within (0, 1].'
727
+ );
728
+ }
729
+ if (!Number.isFinite(reserveBytes) || reserveBytes < 0) {
730
+ throw new Error(
731
+ 'node source runtime: runtime.loading.memoryManagement.budget.reserveBytes ' +
732
+ 'must be a non-negative number.'
733
+ );
734
+ }
735
+ if (!Number.isFinite(minimumBudgetBytes) || minimumBudgetBytes <= 0) {
736
+ throw new Error(
737
+ 'node source runtime: runtime.loading.memoryManagement.budget.minimumBudgetBytes ' +
738
+ 'must be a positive number.'
739
+ );
740
+ }
741
+
742
+ const derived = Math.floor(totalmem() * systemMemoryFraction) - Math.floor(reserveBytes);
743
+ return Math.max(Math.floor(minimumBudgetBytes), derived);
744
+ }
745
+
746
+ function estimateSourceRuntimeTransientBytes(parsed, loadingConfig) {
747
+ const maxTensorBytes = (Array.isArray(parsed?.tensors) ? parsed.tensors : []).reduce((maxBytes, tensor) => {
748
+ const size = Number(tensor?.size);
749
+ return Number.isFinite(size) && size > maxBytes ? Math.floor(size) : maxBytes;
750
+ }, 0);
751
+ const streamConfig = loadingConfig?.storage?.backend?.streaming ?? {};
752
+ const readChunkBytes = Number(streamConfig.readChunkBytes);
753
+ const maxInFlightBytes = Number(streamConfig.maxInFlightBytes);
754
+ const streamWindowBytes = Math.max(
755
+ Number.isFinite(readChunkBytes) && readChunkBytes > 0 ? Math.floor(readChunkBytes) : 0,
756
+ Number.isFinite(maxInFlightBytes) && maxInFlightBytes > 0 ? Math.floor(maxInFlightBytes) : 0,
757
+ MAX_NODE_READ_BYTES
758
+ );
759
+ return Math.max(maxTensorBytes, streamWindowBytes);
760
+ }
761
+
762
+ function assertSourceRuntimeFitsResidentBudget(parsed, loadingConfig, residentBudgetBytes) {
763
+ if (!Number.isFinite(residentBudgetBytes) || residentBudgetBytes <= 0) {
764
+ return;
765
+ }
766
+
767
+ const currentRssBytes = typeof process !== 'undefined' && typeof process.memoryUsage === 'function'
768
+ ? process.memoryUsage().rss
769
+ : 0;
770
+ const estimatedTransientBytes = estimateSourceRuntimeTransientBytes(parsed, loadingConfig);
771
+ const projectedResidentBytes = currentRssBytes + estimatedTransientBytes;
772
+ if (projectedResidentBytes <= residentBudgetBytes) {
773
+ return;
774
+ }
775
+
776
+ const sourceFiles = Array.isArray(parsed?.sourceFiles) ? parsed.sourceFiles : [];
777
+ const totalSourceBytes = sourceFiles.reduce((totalBytes, entry) => {
778
+ const size = Number(entry?.size);
779
+ return Number.isFinite(size) ? totalBytes + Math.max(0, Math.floor(size)) : totalBytes;
780
+ }, 0);
781
+ const largestSourceBytes = sourceFiles.reduce((maxBytes, entry) => {
782
+ const size = Number(entry?.size);
783
+ return Number.isFinite(size) && size > maxBytes ? Math.floor(size) : maxBytes;
784
+ }, 0);
785
+
786
+ throw new Error(
787
+ 'node source runtime: direct-source load exceeds resident memory budget. ' +
788
+ `rss=${formatBytes(currentRssBytes)}, transient=${formatBytes(estimatedTransientBytes)}, ` +
789
+ `projected=${formatBytes(projectedResidentBytes)}, budget=${formatBytes(residentBudgetBytes)}, ` +
790
+ `largestSource=${formatBytes(largestSourceBytes)}, totalSource=${formatBytes(totalSourceBytes)}. ` +
791
+ 'Lower the model working set or adjust runtime.loading.memoryManagement.budget.'
792
+ );
793
+ }
794
+
490
795
  export async function resolveNodeSourceRuntimeBundle(options = {}) {
491
796
  const inputPath = normalizePath(options.inputPath);
492
797
  if (!inputPath) {
@@ -498,10 +803,21 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
498
803
 
499
804
  let parsed = null;
500
805
  if (stats.isFile()) {
501
- if (!isGgufPath(resolvedInputPath)) {
806
+ if (isTflitePath(resolvedInputPath)) {
807
+ parsed = await parseTfliteInput(resolvedInputPath);
808
+ }
809
+ if (!parsed && isLiteRTTaskPath(resolvedInputPath)) {
810
+ parsed = await parseLiteRTPackageInput(resolvedInputPath, LITERT_PACKAGE_SOURCE_KIND_TASK);
811
+ }
812
+ if (!parsed && isLiteRTLMPath(resolvedInputPath)) {
813
+ parsed = await parseLiteRTPackageInput(resolvedInputPath, LITERT_PACKAGE_SOURCE_KIND_LITERTLM);
814
+ }
815
+ if (!parsed && !isGgufPath(resolvedInputPath)) {
502
816
  return null;
503
817
  }
504
- parsed = await parseGgufInput(resolvedInputPath);
818
+ if (!parsed) {
819
+ parsed = await parseGgufInput(resolvedInputPath);
820
+ }
505
821
  } else if (stats.isDirectory()) {
506
822
  if (await fileExists(path.join(resolvedInputPath, 'manifest.json'))) {
507
823
  return null;
@@ -520,6 +836,51 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
520
836
  `node source runtime: multiple GGUF files found in "${resolvedInputPath}": ${ggufFiles.join(', ')}.`
521
837
  );
522
838
  }
839
+ if (!parsed) {
840
+ const tfliteFiles = entries
841
+ .filter((entry) => entry.isFile() && isTflitePath(entry.name))
842
+ .map((entry) => entry.name)
843
+ .sort((left, right) => left.localeCompare(right));
844
+ if (tfliteFiles.length === 1) {
845
+ parsed = await parseTfliteInput(path.join(resolvedInputPath, tfliteFiles[0]));
846
+ } else if (tfliteFiles.length > 1) {
847
+ throw new Error(
848
+ `node source runtime: multiple TFLite files found in "${resolvedInputPath}": ${tfliteFiles.join(', ')}.`
849
+ );
850
+ }
851
+ }
852
+ if (!parsed) {
853
+ const taskFiles = entries
854
+ .filter((entry) => entry.isFile() && isLiteRTTaskPath(entry.name))
855
+ .map((entry) => entry.name)
856
+ .sort((left, right) => left.localeCompare(right));
857
+ if (taskFiles.length === 1) {
858
+ parsed = await parseLiteRTPackageInput(
859
+ path.join(resolvedInputPath, taskFiles[0]),
860
+ LITERT_PACKAGE_SOURCE_KIND_TASK
861
+ );
862
+ } else if (taskFiles.length > 1) {
863
+ throw new Error(
864
+ `node source runtime: multiple LiteRT task files found in "${resolvedInputPath}": ${taskFiles.join(', ')}.`
865
+ );
866
+ }
867
+ }
868
+ if (!parsed) {
869
+ const litertLmFiles = entries
870
+ .filter((entry) => entry.isFile() && isLiteRTLMPath(entry.name))
871
+ .map((entry) => entry.name)
872
+ .sort((left, right) => left.localeCompare(right));
873
+ if (litertLmFiles.length === 1) {
874
+ parsed = await parseLiteRTPackageInput(
875
+ path.join(resolvedInputPath, litertLmFiles[0]),
876
+ LITERT_PACKAGE_SOURCE_KIND_LITERTLM
877
+ );
878
+ } else if (litertLmFiles.length > 1) {
879
+ throw new Error(
880
+ `node source runtime: multiple LiteRT-LM files found in "${resolvedInputPath}": ${litertLmFiles.join(', ')}.`
881
+ );
882
+ }
883
+ }
523
884
  }
524
885
  } else {
525
886
  return null;
@@ -529,84 +890,48 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
529
890
  return null;
530
891
  }
531
892
 
532
- assertSupportedSourceDtypes(parsed.tensors, parsed.sourceKind);
533
-
534
- const converterConfig = createConverterConfig({
535
- quantization: {
536
- computePrecision: resolveSourceRuntimeComputePrecision(parsed.tensors, parsed.sourceQuantization),
537
- },
538
- output: {
539
- modelBaseId: options.modelId || null,
540
- },
541
- inference: {
542
- sessionDefaults: cloneSessionDefaults(),
543
- execution: SOURCE_RUNTIME_EXECUTION_OVERRIDE,
544
- },
545
- });
546
- const plan = resolveConversionPlan({
547
- rawConfig: parsed.config,
548
- tensors: parsed.tensors,
549
- converterConfig,
550
- sourceQuantization: parsed.sourceQuantization,
551
- modelKind: 'transformer',
552
- architectureHint: parsed.architectureHint,
553
- architectureConfig: parsed.architecture,
554
- });
555
-
556
- const modelId = resolveModelIdHint(
557
- options.modelId || null,
558
- plan,
559
- parsed.sourceKind,
560
- parsed.sourcePathForModelId
561
- );
562
- const hashAlgorithm = converterConfig.manifest.hashAlgorithm;
563
- const sourceFiles = await addHashesToFileEntries(parsed.sourceFiles, hashAlgorithm);
564
- const auxiliaryFiles = await addHashesToFileEntries(parsed.auxiliaryFiles, hashAlgorithm);
565
- const { manifest, shardSources } = await buildSourceRuntimeBundle({
566
- modelId,
567
- modelName: modelId,
568
- modelType: plan.modelType,
569
- sourceKind: parsed.sourceKind,
570
- architecture: parsed.architecture,
571
- architectureHint: parsed.architectureHint,
572
- rawConfig: parsed.config,
573
- inference: plan.manifestInference,
574
- tensors: parsed.tensors,
575
- embeddingPostprocessor: parsed.embeddingPostprocessor ?? null,
576
- sourceFiles,
577
- auxiliaryFiles,
578
- sourceQuantization: parsed.sourceQuantization,
579
- quantizationInfo: plan.quantizationInfo,
580
- hashAlgorithm,
581
- tokenizerJson: parsed.tokenizerJson,
582
- tokenizerConfig: parsed.tokenizerConfig,
583
- tokenizerModelName: parsed.tokenizerModelName,
584
- tokenizerJsonPath: parsed.tokenizerJsonPath,
585
- tokenizerConfigPath: parsed.tokenizerConfigPath,
586
- tokenizerModelPath: parsed.tokenizerModelPath,
893
+ const loadingConfig = resolveLoadingConfig(options.runtimeConfig ?? null);
894
+ const resolvedMemoryBudgetBytes = resolveResidentBudgetBytes(loadingConfig);
895
+ assertSourceRuntimeFitsResidentBudget(parsed, loadingConfig, resolvedMemoryBudgetBytes);
896
+ const {
897
+ model,
898
+ shardSources,
899
+ sourceKind,
900
+ } = await resolveSourceRuntimeBundleFromParsedArtifact({
901
+ parsedArtifact: parsed,
902
+ requestedModelId: options.modelId || null,
903
+ runtimeLabel: 'node source runtime',
904
+ logCategory: 'NodeSourceRuntime',
905
+ hashFileEntries: typeof parsed?.hashFileEntries === 'function'
906
+ ? (entries, hashAlgorithm) => parsed.hashFileEntries(entries, hashAlgorithm)
907
+ : addHashesToFileEntries,
587
908
  });
588
909
 
589
- const readers = buildNodeFileReaders();
910
+ const readers = parsed?.storageReaders ?? buildNodeFileReaders();
590
911
  const storageContext = createSourceStorageContext({
591
- manifest,
912
+ model,
592
913
  shardSources,
593
914
  readRange: readers.readRange,
915
+ streamRange: readers.streamRange,
594
916
  readText: readers.readText,
595
917
  readBinary: readers.readBinary,
596
918
  tokenizerJsonPath: parsed.tokenizerJsonPath,
597
919
  tokenizerModelPath: parsed.tokenizerModelPath,
598
920
  verifyHashes,
921
+ sourceHashesTrusted: true,
599
922
  });
600
923
 
601
924
  log.info(
602
925
  'NodeSourceRuntime',
603
- `Source runtime ready: ${manifest.modelId} (${parsed.sourceKind}, ${parsed.tensors.length} tensors)`
926
+ `Source runtime ready: ${model.modelId} (${sourceKind}, ${parsed.tensors.length} tensors)`
604
927
  );
605
928
 
606
929
  return {
607
- manifest,
930
+ model,
931
+ manifest: model,
608
932
  storageContext,
609
- sourceKind: parsed.sourceKind,
933
+ sourceKind,
610
934
  sourceRoot: parsed.sourceRoot,
935
+ resolvedMemoryBudgetBytes,
611
936
  };
612
937
  }