doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,677 +0,0 @@
1
- import {
2
- createConverterConfig,
3
- HEADER_READ_SIZE,
4
- } from '../../config/schema/index.js';
5
- import { DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS } from '../../config/schema/execution-v1.schema.js';
6
- import { extractArchitecture } from '../../converter/core.js';
7
- import {
8
- inferSourceWeightQuantization,
9
- resolveConversionPlan,
10
- resolveConvertedModelId,
11
- } from '../../converter/conversion-plan.js';
12
- import { parseGGUFModel } from '../../converter/parsers/gguf.js';
13
- import { parseTransformerModel } from '../../converter/parsers/transformer.js';
14
- import { parseGGUFHeader } from '../../formats/gguf/types.js';
15
- import { parseSafetensorsHeader } from '../../formats/safetensors/types.js';
16
- import { log } from '../../debug/index.js';
17
- import { computeHash } from '../../storage/shard-manager.js';
18
- import {
19
- buildSourceRuntimeBundle,
20
- createSourceStorageContext,
21
- getSourceRuntimeMetadata,
22
- } from '../../tooling/source-runtime-bundle.js';
23
-
24
- const SUPPORTED_SOURCE_DTYPES = new Set([
25
- 'F32',
26
- 'F16',
27
- 'BF16',
28
- 'Q4_K',
29
- 'Q4_K_M',
30
- 'Q6_K',
31
- ]);
32
-
33
- const ZERO_DIGEST = 'sha256:' + '0'.repeat(64);
34
-
35
- const SOURCE_RUNTIME_SESSION_DEFAULTS = {
36
- compute: {
37
- defaults: { ...DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS },
38
- },
39
- kvcache: null,
40
- decodeLoop: null,
41
- };
42
-
43
- const SOURCE_RUNTIME_EXECUTION = {
44
- kernels: {
45
- embed: { kernel: 'gather_f16.wgsl', entry: 'main', digest: ZERO_DIGEST },
46
- },
47
- preLayer: [['embed', 'embed', 'embed_tokens']],
48
- decode: [],
49
- prefill: [],
50
- postLayer: [],
51
- policies: {
52
- unsupportedPrecision: 'error',
53
- dtypeTransition: 'require_cast_step',
54
- unresolvedKernel: 'error',
55
- },
56
- };
57
-
58
- const SOURCE_RUNTIME_INFERENCE = {
59
- attention: {
60
- slidingWindow: null,
61
- attnLogitSoftcapping: null,
62
- queryKeyNorm: false,
63
- attentionOutputGate: false,
64
- causal: true,
65
- attentionBias: false,
66
- },
67
- normalization: {
68
- rmsNormWeightOffset: false,
69
- rmsNormEps: 1e-6,
70
- },
71
- ffn: {
72
- activation: 'gelu',
73
- gatedActivation: true,
74
- swigluLimit: null,
75
- },
76
- rope: {
77
- ropeTheta: 10000,
78
- partialRotaryFactor: 1.0,
79
- ropeInterleaved: false,
80
- },
81
- output: {
82
- scaleEmbeddings: false,
83
- tieWordEmbeddings: false,
84
- embeddingTranspose: false,
85
- embeddingVocabSize: null,
86
- finalLogitSoftcapping: null,
87
- },
88
- chatTemplate: { type: 'none' },
89
- layerPattern: { type: 'uniform' },
90
- };
91
-
92
- function normalizeRelativePath(value) {
93
- return String(value || '')
94
- .replace(/\\/g, '/')
95
- .replace(/^\.\/+/, '')
96
- .replace(/^\/+/, '')
97
- .trim();
98
- }
99
-
100
- function joinPath(base, relativePath) {
101
- const root = String(base || '').replace(/\/+$/, '');
102
- const rel = normalizeRelativePath(relativePath);
103
- return rel ? `${root}/${rel}` : root;
104
- }
105
-
106
- function toArrayBuffer(value, label) {
107
- if (value instanceof ArrayBuffer) {
108
- return value;
109
- }
110
- if (value instanceof Uint8Array) {
111
- return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
112
- }
113
- throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
114
- }
115
-
116
- function toUint8Array(value) {
117
- return value instanceof Uint8Array ? value : new Uint8Array(value);
118
- }
119
-
120
- function decodeText(value) {
121
- return new TextDecoder().decode(value);
122
- }
123
-
124
- function ensureJsonObject(raw, label) {
125
- if (raw == null || typeof raw !== 'object' || Array.isArray(raw)) {
126
- throw new Error(`${label} must be a JSON object.`);
127
- }
128
- return raw;
129
- }
130
-
131
- async function listBridgeFilesRecursive(bridgeClient, rootPath) {
132
- const files = [];
133
- const base = String(rootPath || '').replace(/\/+$/, '');
134
-
135
- async function walk(relativePath = '') {
136
- const absolutePath = relativePath ? joinPath(base, relativePath) : base;
137
- const entries = await bridgeClient.list(absolutePath);
138
- for (const entry of entries) {
139
- const name = String(entry?.name || '');
140
- if (!name) continue;
141
- const childRelative = relativePath
142
- ? `${normalizeRelativePath(relativePath)}/${name}`
143
- : name;
144
- if (entry.isDir) {
145
- await walk(childRelative);
146
- } else {
147
- const normalized = normalizeRelativePath(childRelative);
148
- files.push({
149
- relativePath: normalized,
150
- absolutePath: joinPath(base, normalized),
151
- size: Number(entry?.size) || 0,
152
- });
153
- }
154
- }
155
- }
156
-
157
- await walk('');
158
- files.sort((left, right) => left.relativePath.localeCompare(right.relativePath));
159
- return files;
160
- }
161
-
162
- function indexBridgeFiles(files) {
163
- const map = new Map();
164
- for (const file of files) {
165
- map.set(file.relativePath, file);
166
- }
167
- return map;
168
- }
169
-
170
- function hasModelManifest(fileIndex) {
171
- return fileIndex.has('manifest.json');
172
- }
173
-
174
- function detectBridgeSourceFormat(fileIndex) {
175
- const relativePaths = Array.from(fileIndex.keys());
176
- const ggufFiles = relativePaths.filter((path) => path.toLowerCase().endsWith('.gguf'));
177
- if (ggufFiles.length === 1) {
178
- return { kind: 'gguf', ggufPath: ggufFiles[0] };
179
- }
180
-
181
- const hasConfig = fileIndex.has('config.json');
182
- const hasSafetensors = relativePaths.some((path) => path.toLowerCase().endsWith('.safetensors'));
183
- const hasSafetensorsIndex = fileIndex.has('model.safetensors.index.json');
184
- if (hasConfig && (hasSafetensors || hasSafetensorsIndex)) {
185
- return { kind: 'safetensors', ggufPath: null };
186
- }
187
-
188
- return null;
189
- }
190
-
191
- function assertSupportedSourceDtypes(tensors, sourceKind) {
192
- const unsupported = new Set();
193
- for (const tensor of tensors) {
194
- const dtype = String(tensor?.dtype || '').trim().toUpperCase();
195
- if (!dtype) {
196
- unsupported.add('(empty)');
197
- continue;
198
- }
199
- if (!SUPPORTED_SOURCE_DTYPES.has(dtype)) {
200
- unsupported.add(dtype);
201
- }
202
- }
203
- if (unsupported.size > 0) {
204
- throw new Error(
205
- `Unsupported ${sourceKind} tensor dtypes for direct-source runtime: ` +
206
- `${Array.from(unsupported).sort((a, b) => a.localeCompare(b)).join(', ')}. ` +
207
- 'Convert to RDRR first for this model.'
208
- );
209
- }
210
- }
211
-
212
- async function readBridgeRange(bridgeClient, fileEntry, offset, length) {
213
- return bridgeClient.read(fileEntry.absolutePath, offset, length);
214
- }
215
-
216
- async function readBridgeAllBytes(bridgeClient, fileEntry, label) {
217
- const size = Number(fileEntry?.size) || 0;
218
- if (size < 0) {
219
- throw new Error(`Invalid bridge file size for ${label}.`);
220
- }
221
- return readBridgeRange(bridgeClient, fileEntry, 0, size);
222
- }
223
-
224
- async function readBridgeTextFile(bridgeClient, fileEntry, label) {
225
- const size = Number(fileEntry?.size) || 0;
226
- if (size <= 0) {
227
- throw new Error(`Bridge file "${label}" is empty.`);
228
- }
229
- const bytes = await readBridgeRange(bridgeClient, fileEntry, 0, size);
230
- return decodeText(bytes);
231
- }
232
-
233
- async function readBridgeJsonFile(bridgeClient, fileEntry, label) {
234
- const text = await readBridgeTextFile(bridgeClient, fileEntry, label);
235
- try {
236
- return ensureJsonObject(JSON.parse(text), label);
237
- } catch (error) {
238
- const message = error instanceof Error ? error.message : String(error);
239
- throw new Error(`Invalid JSON in ${label}: ${message}`);
240
- }
241
- }
242
-
243
- async function readSafetensorsHeaderFromBridge(bridgeClient, fileEntry) {
244
- const prefix = await readBridgeRange(bridgeClient, fileEntry, 0, 8);
245
- const prefixBytes = toUint8Array(prefix);
246
- if (prefixBytes.byteLength < 8) {
247
- throw new Error(`Invalid safetensors header prefix for "${fileEntry.relativePath}"`);
248
- }
249
- const headerSize = Number(new DataView(toArrayBuffer(prefixBytes, 'safetensors prefix')).getBigUint64(0, true));
250
- const header = await readBridgeRange(bridgeClient, fileEntry, 8, headerSize);
251
- const full = new Uint8Array(8 + headerSize);
252
- full.set(prefixBytes, 0);
253
- full.set(toUint8Array(header), 8);
254
- return parseSafetensorsHeader(full.buffer);
255
- }
256
-
257
- async function parseBridgeSafetensorsModel(bridgeClient, fileIndex) {
258
- const parsedTransformer = await parseTransformerModel({
259
- async readJson(path, label = 'json') {
260
- const entry = fileIndex.get(normalizeRelativePath(path));
261
- if (!entry) {
262
- throw new Error(`Missing ${label} (${path})`);
263
- }
264
- return readBridgeJsonFile(bridgeClient, entry, `${label} (${path})`);
265
- },
266
- async fileExists(path) {
267
- return fileIndex.has(normalizeRelativePath(path));
268
- },
269
- async loadSingleSafetensors(path) {
270
- const normalized = normalizeRelativePath(path);
271
- const entry = fileIndex.get(normalized);
272
- if (!entry) {
273
- throw new Error(`Missing safetensors file (${path})`);
274
- }
275
- const parsed = await readSafetensorsHeaderFromBridge(bridgeClient, entry);
276
- return parsed.tensors.map((tensor) => ({
277
- ...tensor,
278
- sourcePath: normalized,
279
- }));
280
- },
281
- async loadShardedSafetensors(indexJson) {
282
- const shardFiles = [...new Set(Object.values(indexJson.weight_map || {}))]
283
- .map((path) => normalizeRelativePath(path));
284
- const tensors = [];
285
- for (const shardPath of shardFiles) {
286
- const entry = fileIndex.get(shardPath);
287
- if (!entry) {
288
- throw new Error(`Missing safetensors shard (${shardPath})`);
289
- }
290
- const parsed = await readSafetensorsHeaderFromBridge(bridgeClient, entry);
291
- for (const tensor of parsed.tensors) {
292
- tensors.push({
293
- ...tensor,
294
- sourcePath: shardPath,
295
- });
296
- }
297
- }
298
- return tensors;
299
- },
300
- });
301
-
302
- const config = parsedTransformer.config;
303
- const tensors = parsedTransformer.tensors;
304
- const architectureHint = parsedTransformer.architectureHint;
305
- const embeddingPostprocessor = parsedTransformer.embeddingPostprocessor ?? null;
306
- const architecture = extractArchitecture(config, null);
307
- const tokenizerJson = fileIndex.has('tokenizer.json')
308
- ? await readBridgeJsonFile(bridgeClient, fileIndex.get('tokenizer.json'), 'tokenizer.json')
309
- : null;
310
- const tokenizerConfig = fileIndex.has('tokenizer_config.json')
311
- ? await readBridgeJsonFile(bridgeClient, fileIndex.get('tokenizer_config.json'), 'tokenizer_config.json')
312
- : null;
313
- const tokenizerModelName = fileIndex.has('tokenizer.model') ? 'tokenizer.model' : null;
314
-
315
- return {
316
- sourceKind: 'safetensors',
317
- config,
318
- tensors,
319
- architectureHint,
320
- embeddingPostprocessor,
321
- architecture,
322
- sourceQuantization: inferSourceWeightQuantization(tensors),
323
- tokenizerJson,
324
- tokenizerConfig,
325
- tokenizerModelName,
326
- sourceFiles: Array.from(new Set(tensors.map((tensor) => normalizeRelativePath(tensor.sourcePath))))
327
- .map((path) => {
328
- const entry = fileIndex.get(path);
329
- if (!entry) {
330
- throw new Error(`Missing source file entry for "${path}"`);
331
- }
332
- return { path, size: entry.size };
333
- }),
334
- auxiliaryFiles: [
335
- { path: 'config.json', size: Number(fileIndex.get('config.json')?.size || 0), kind: 'config' },
336
- ...(fileIndex.has('model.safetensors.index.json')
337
- ? [{
338
- path: 'model.safetensors.index.json',
339
- size: Number(fileIndex.get('model.safetensors.index.json')?.size || 0),
340
- kind: 'safetensors_index',
341
- }]
342
- : []),
343
- ...(fileIndex.has('tokenizer.json')
344
- ? [{
345
- path: 'tokenizer.json',
346
- size: Number(fileIndex.get('tokenizer.json')?.size || 0),
347
- kind: 'tokenizer_json',
348
- }]
349
- : []),
350
- ...(fileIndex.has('tokenizer_config.json')
351
- ? [{
352
- path: 'tokenizer_config.json',
353
- size: Number(fileIndex.get('tokenizer_config.json')?.size || 0),
354
- kind: 'tokenizer_config',
355
- }]
356
- : []),
357
- ...(fileIndex.has('tokenizer.model')
358
- ? [{
359
- path: 'tokenizer.model',
360
- size: Number(fileIndex.get('tokenizer.model')?.size || 0),
361
- kind: 'tokenizer_model',
362
- }]
363
- : []),
364
- ],
365
- tokenizerJsonPath: fileIndex.has('tokenizer.json') ? 'tokenizer.json' : null,
366
- tokenizerConfigPath: fileIndex.has('tokenizer_config.json') ? 'tokenizer_config.json' : null,
367
- tokenizerModelPath: fileIndex.has('tokenizer.model') ? 'tokenizer.model' : null,
368
- };
369
- }
370
-
371
- async function parseBridgeGGUFModel(bridgeClient, fileIndex, ggufRelativePath) {
372
- const ggufEntry = fileIndex.get(ggufRelativePath);
373
- if (!ggufEntry) {
374
- throw new Error(`Missing GGUF file (${ggufRelativePath})`);
375
- }
376
-
377
- const ggufSource = {
378
- sourceType: 'bridge-file',
379
- name: ggufRelativePath,
380
- size: ggufEntry.size,
381
- file: {
382
- name: ggufRelativePath,
383
- size: ggufEntry.size,
384
- },
385
- async readRange(offset, length) {
386
- return readBridgeRange(bridgeClient, ggufEntry, offset, length);
387
- },
388
- };
389
-
390
- const parseGGUFHeaderFromSource = async (source) => {
391
- const resolved = source && typeof source.readRange === 'function' ? source : ggufSource;
392
- const readSize = Math.min(resolved.size, HEADER_READ_SIZE);
393
- const buffer = await resolved.readRange(0, readSize);
394
- const info = parseGGUFHeader(toArrayBuffer(buffer, `gguf header (${ggufRelativePath})`));
395
- return {
396
- ...info,
397
- fileSize: resolved.size,
398
- };
399
- };
400
-
401
- const parsedGGUF = await parseGGUFModel({
402
- file: ggufSource,
403
- parseGGUFHeaderFromSource,
404
- normalizeTensorSource(source) {
405
- if (source && typeof source.readRange === 'function' && Number.isFinite(source.size)) {
406
- return source;
407
- }
408
- return ggufSource;
409
- },
410
- onProgress() {},
411
- signal: null,
412
- });
413
-
414
- const tensors = parsedGGUF.tensors.map((tensor) => ({
415
- ...tensor,
416
- sourcePath: ggufRelativePath,
417
- }));
418
- const architecture = extractArchitecture({}, parsedGGUF.config || {});
419
-
420
- return {
421
- sourceKind: 'gguf',
422
- config: parsedGGUF.config,
423
- tensors,
424
- architectureHint: parsedGGUF.architecture,
425
- architecture,
426
- sourceQuantization: parsedGGUF.quantization ?? inferSourceWeightQuantization(tensors),
427
- tokenizerJson: null,
428
- tokenizerConfig: null,
429
- tokenizerModelName: null,
430
- sourceFiles: [{ path: ggufRelativePath, size: ggufEntry.size }],
431
- auxiliaryFiles: [],
432
- tokenizerJsonPath: null,
433
- tokenizerConfigPath: null,
434
- tokenizerModelPath: null,
435
- };
436
- }
437
-
438
- async function parseBridgeSourceModel(bridgeClient, localPath) {
439
- const files = await listBridgeFilesRecursive(bridgeClient, localPath);
440
- const fileIndex = indexBridgeFiles(files);
441
- if (hasModelManifest(fileIndex)) {
442
- return null;
443
- }
444
-
445
- const detected = detectBridgeSourceFormat(fileIndex);
446
- if (!detected) {
447
- return null;
448
- }
449
-
450
- if (detected.kind === 'gguf') {
451
- return parseBridgeGGUFModel(bridgeClient, fileIndex, detected.ggufPath);
452
- }
453
-
454
- return parseBridgeSafetensorsModel(bridgeClient, fileIndex);
455
- }
456
-
457
- function resolveModelIdHint(requestedModelId, plan, sourceKind) {
458
- const explicit = String(requestedModelId || '').trim();
459
- if (explicit) {
460
- return resolveConvertedModelId({
461
- explicitModelId: explicit,
462
- converterConfig: null,
463
- detectedModelId: explicit,
464
- quantizationInfo: plan.quantizationInfo,
465
- }) || explicit;
466
- }
467
- const sourcePrefix = sourceKind === 'gguf' ? 'gguf' : 'safetensors';
468
- return resolveConvertedModelId({
469
- explicitModelId: `${sourcePrefix}-runtime`,
470
- converterConfig: null,
471
- detectedModelId: `${sourcePrefix}-runtime`,
472
- quantizationInfo: plan.quantizationInfo,
473
- }) || `${sourcePrefix}-runtime`;
474
- }
475
-
476
- function createBridgeFileReaders(bridgeClient, fileMap, rootPath) {
477
- const map = fileMap;
478
-
479
- const resolveEntry = (pathHint) => {
480
- const hint = normalizeRelativePath(pathHint);
481
- if (!hint) {
482
- return null;
483
- }
484
- const direct = map.get(hint);
485
- return direct || null;
486
- };
487
-
488
- const readRange = async (relativePath, offset, length) => {
489
- const entry = resolveEntry(relativePath);
490
- if (!entry) {
491
- throw new Error(`Missing source shard file: ${relativePath}`);
492
- }
493
- return bridgeClient.read(entry.absolutePath, offset, length);
494
- };
495
-
496
- const readText = async (pathHint) => {
497
- const entry = resolveEntry(pathHint);
498
- if (!entry) return null;
499
- const bytes = await bridgeClient.read(entry.absolutePath, 0, entry.size);
500
- return decodeText(bytes);
501
- };
502
-
503
- const readBinary = async (pathHint) => {
504
- const entry = resolveEntry(pathHint);
505
- if (!entry) {
506
- throw new Error(`Missing source binary file: ${pathHint}`);
507
- }
508
- return bridgeClient.read(entry.absolutePath, 0, entry.size);
509
- };
510
-
511
- return {
512
- rootPath,
513
- readRange,
514
- readText,
515
- readBinary,
516
- };
517
- }
518
-
519
- async function addHashesToBridgeFiles(bridgeClient, fileIndex, entries, hashAlgorithm) {
520
- const hashedEntries = [];
521
- for (const entry of Array.isArray(entries) ? entries : []) {
522
- const relativePath = normalizeRelativePath(entry?.path);
523
- if (!relativePath) continue;
524
- const fileEntry = fileIndex.get(relativePath);
525
- if (!fileEntry) {
526
- throw new Error(`Missing bridge file entry for "${relativePath}"`);
527
- }
528
- const bytes = await readBridgeAllBytes(bridgeClient, fileEntry, `bridge source asset (${relativePath})`);
529
- hashedEntries.push({
530
- ...entry,
531
- path: relativePath,
532
- size: Number.isFinite(entry?.size) ? Math.max(0, Math.floor(Number(entry.size))) : fileEntry.size,
533
- hash: await computeHash(toUint8Array(bytes), hashAlgorithm),
534
- hashAlgorithm,
535
- });
536
- }
537
- return hashedEntries;
538
- }
539
-
540
- async function resolveBridgeStorageContext(options = {}) {
541
- const bridgeClient = options.bridgeClient;
542
- const localPath = options.localPath;
543
- const manifest = options.manifest;
544
- const sourceRuntime = getSourceRuntimeMetadata(manifest);
545
- if (!sourceRuntime) {
546
- return null;
547
- }
548
- const files = await listBridgeFilesRecursive(bridgeClient, localPath);
549
- const fileMap = indexBridgeFiles(files);
550
- const readers = createBridgeFileReaders(bridgeClient, fileMap, localPath);
551
- return createSourceStorageContext({
552
- manifest,
553
- readRange: readers.readRange,
554
- readText: readers.readText,
555
- readBinary: readers.readBinary,
556
- verifyHashes: options.verifyHashes !== false,
557
- });
558
- }
559
-
560
- export async function resolveBridgeSourceRuntimeBundle(options = {}) {
561
- const bridgeClient = options.bridgeClient;
562
- const localPath = options.localPath;
563
- const requestedModelId = options.modelId || null;
564
- const verifyHashes = options.verifyHashes !== false;
565
- const existingManifest = options.manifest ?? null;
566
-
567
- if (!bridgeClient || typeof bridgeClient.read !== 'function' || typeof bridgeClient.list !== 'function') {
568
- throw new Error('Bridge source runtime requires a connected bridge client with read/list support.');
569
- }
570
- if (!localPath || typeof localPath !== 'string') {
571
- throw new Error('Bridge source runtime requires localPath.');
572
- }
573
-
574
- if (existingManifest && getSourceRuntimeMetadata(existingManifest)) {
575
- const storageContext = await resolveBridgeStorageContext({
576
- bridgeClient,
577
- localPath,
578
- manifest: existingManifest,
579
- verifyHashes,
580
- });
581
- return {
582
- manifest: existingManifest,
583
- storageContext,
584
- sourceKind: getSourceRuntimeMetadata(existingManifest)?.sourceKind ?? 'safetensors',
585
- sourceRoot: localPath,
586
- };
587
- }
588
-
589
- options.onProgress?.({
590
- stage: 'source-discovery',
591
- message: 'Scanning source files via bridge...',
592
- });
593
-
594
- const parsed = await parseBridgeSourceModel(bridgeClient, localPath);
595
- if (!parsed) {
596
- return null;
597
- }
598
-
599
- assertSupportedSourceDtypes(parsed.tensors, parsed.sourceKind);
600
-
601
- const converterConfig = createConverterConfig({
602
- output: {
603
- modelBaseId: requestedModelId || null,
604
- },
605
- inference: SOURCE_RUNTIME_INFERENCE,
606
- sessionDefaults: SOURCE_RUNTIME_SESSION_DEFAULTS,
607
- execution: SOURCE_RUNTIME_EXECUTION,
608
- });
609
- const plan = resolveConversionPlan({
610
- rawConfig: parsed.config,
611
- tensors: parsed.tensors,
612
- converterConfig,
613
- sourceQuantization: parsed.sourceQuantization,
614
- modelKind: 'transformer',
615
- architectureHint: parsed.architectureHint,
616
- architectureConfig: parsed.architecture,
617
- });
618
-
619
- const modelId = resolveModelIdHint(requestedModelId, plan, parsed.sourceKind);
620
- const hashAlgorithm = converterConfig.manifest.hashAlgorithm;
621
- const files = await listBridgeFilesRecursive(bridgeClient, localPath);
622
- const fileMap = indexBridgeFiles(files);
623
- const sourceFiles = await addHashesToBridgeFiles(bridgeClient, fileMap, parsed.sourceFiles, hashAlgorithm);
624
- const auxiliaryFiles = await addHashesToBridgeFiles(
625
- bridgeClient,
626
- fileMap,
627
- parsed.auxiliaryFiles,
628
- hashAlgorithm
629
- );
630
- const { manifest, shardSources } = await buildSourceRuntimeBundle({
631
- modelId,
632
- modelName: modelId,
633
- modelType: plan.modelType,
634
- sourceKind: parsed.sourceKind,
635
- architecture: parsed.architecture,
636
- architectureHint: parsed.architectureHint,
637
- rawConfig: parsed.config,
638
- inference: plan.manifestInference,
639
- tensors: parsed.tensors,
640
- embeddingPostprocessor: parsed.embeddingPostprocessor ?? null,
641
- sourceFiles,
642
- auxiliaryFiles,
643
- sourceQuantization: parsed.sourceQuantization,
644
- quantizationInfo: plan.quantizationInfo,
645
- hashAlgorithm,
646
- tokenizerJson: parsed.tokenizerJson,
647
- tokenizerConfig: parsed.tokenizerConfig,
648
- tokenizerModelName: parsed.tokenizerModelName,
649
- tokenizerJsonPath: parsed.tokenizerJsonPath,
650
- tokenizerConfigPath: parsed.tokenizerConfigPath,
651
- tokenizerModelPath: parsed.tokenizerModelPath,
652
- });
653
-
654
- const readers = createBridgeFileReaders(bridgeClient, fileMap, localPath);
655
- const storageContext = createSourceStorageContext({
656
- manifest,
657
- shardSources,
658
- readRange: readers.readRange,
659
- readText: readers.readText,
660
- readBinary: readers.readBinary,
661
- tokenizerJsonPath: parsed.tokenizerJsonPath,
662
- tokenizerModelPath: parsed.tokenizerModelPath,
663
- verifyHashes,
664
- });
665
-
666
- log.info(
667
- 'DopplerProvider',
668
- `Bridge source runtime ready: ${manifest.modelId} (${parsed.sourceKind}, ${parsed.tensors.length} tensors)`
669
- );
670
-
671
- return {
672
- manifest,
673
- storageContext,
674
- sourceKind: parsed.sourceKind,
675
- sourceRoot: localPath,
676
- };
677
- }