doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,29 +1,5 @@
1
- import { loadLoRAFromManifest, loadLoRAFromUrl } from '../adapters/lora-loader.js';
2
- import { log } from '../debug/index.js';
3
- import { getManifestUrl, parseManifest } from '../formats/rdrr/index.js';
4
- import { createPipeline } from '../generation/index.js';
5
- import { getKernelCapabilities } from '../gpu/device.js';
6
- import { formatChatMessages } from '../inference/pipelines/text/chat-format.js';
7
- import { buildQuickstartModelBaseUrl, listQuickstartModels, resolveQuickstartModel } from './doppler-registry.js';
8
-
9
- /**
10
- * Cache lifetime: entries persist until explicitly evicted via `doppler.evict()`,
11
- * `doppler.evictAll()`, or `clearModelCache()`. The `inFlightLoadCache` holds
12
- * promises for models currently being loaded and auto-clears on load completion
13
- * or failure. Neither cache has a TTL -- callers must manage invalidation.
14
- */
15
- const convenienceModelCache = new Map();
16
- const inFlightLoadCache = new Map();
17
-
18
- /**
19
- * Clear all cached models and in-flight load promises.
20
- * Does not call unload() on cached models -- use `doppler.evictAll()` for graceful cleanup.
21
- */
22
- export function clearModelCache() {
23
- convenienceModelCache.clear();
24
- inFlightLoadCache.clear();
25
- log.debug('doppler', 'Model cache cleared');
26
- }
1
+ import { createDefaultNodeLoadProgressLogger } from './runtime/model-source.js';
2
+ import { createDopplerRuntimeService } from './runtime/index.js';
27
3
 
28
4
  function isNodeRuntime() {
29
5
  return typeof process !== 'undefined'
@@ -45,345 +21,18 @@ async function ensureWebGPUAvailable() {
45
21
  throw new Error('WebGPU is unavailable. Install a Node WebGPU provider or run in a WebGPU-capable browser.');
46
22
  }
47
23
 
48
- function emitLoadProgress(callback, phase, percent, message) {
49
- if (typeof callback !== 'function') return;
50
- callback({ phase, percent, message });
51
- }
24
+ const runtime = createDopplerRuntimeService({
25
+ ensureWebGPUAvailable,
26
+ defaultLoadProgressLogger: createDefaultNodeLoadProgressLogger(),
27
+ });
52
28
 
53
- export function createDefaultNodeLoadProgressLogger() {
54
- return (event) => {
55
- const message = typeof event?.message === 'string' ? event.message.trim() : '';
56
- if (!message) return;
57
- log.info('doppler', message);
58
- };
59
- }
29
+ export const doppler = runtime.doppler;
30
+ export const load = runtime.load;
31
+ export const clearModelCache = runtime.clearModelCache;
32
+ export { createDefaultNodeLoadProgressLogger };
60
33
 
61
34
  export function resolveLoadProgressHandlers(options = {}) {
62
- const onProgress = typeof options?.onProgress === 'function' ? options.onProgress : null;
63
- if (onProgress) {
64
- return {
65
- userProgress: onProgress,
66
- pipelineProgress: onProgress,
67
- };
68
- }
69
- if (isNodeRuntime()) {
70
- return {
71
- userProgress: createDefaultNodeLoadProgressLogger(),
72
- pipelineProgress: null,
73
- };
74
- }
75
- log.debug('doppler', 'resolveLoadProgressHandlers: no progress handler configured, returning null handlers');
76
- return {
77
- userProgress: null,
78
- pipelineProgress: null,
79
- };
80
- }
81
-
82
- async function fetchManifestFromBaseUrl(baseUrl) {
83
- const response = await fetch(getManifestUrl(baseUrl));
84
- if (!response.ok) {
85
- throw new Error(`Failed to fetch manifest from ${baseUrl}: ${response.status}`);
86
- }
87
- return parseManifest(await response.text());
88
- }
89
-
90
- /**
91
- * Resolves a model source descriptor from user input.
92
- *
93
- * Fallback order (first match wins):
94
- * 1. Quickstart registry — `model` is a string, looked up by modelId/alias
95
- * in the quickstart registry (doppler-registry.json). Base URL is built
96
- * from the registry entry's Hugging Face coordinates.
97
- * 2. Explicit URL — `model` is an object with a `.url` string property.
98
- * The URL is used directly as both modelId and baseUrl.
99
- * 3. Inline manifest — `model` is an object with a `.manifest` object.
100
- * Optional `.baseUrl` provides the shard root. When baseUrl is absent,
101
- * the caller must supply shards through a custom loader.
102
- *
103
- * If none of the above match, an error is thrown listing the accepted formats.
104
- *
105
- * @param {string | { url: string } | { manifest: object, baseUrl?: string }} model
106
- * @returns {Promise<{ modelId: string, baseUrl: string | null, manifest: object | null, trace: Array<{ source: string, id: string, outcome: string }> }>}
107
- */
108
- async function resolveModelSource(model) {
109
- const trace = [];
110
-
111
- // 1. Quickstart registry (string model id or alias)
112
- if (typeof model === 'string') {
113
- try {
114
- const entry = await resolveQuickstartModel(model);
115
- trace.push({ source: 'quickstart-registry', id: model, outcome: 'resolved' });
116
- log.debug('doppler', `Model resolved via quickstart-registry: ${entry.modelId}`, { trace });
117
- return {
118
- modelId: entry.modelId,
119
- baseUrl: buildQuickstartModelBaseUrl(entry),
120
- manifest: null,
121
- trace,
122
- };
123
- } catch (registryError) {
124
- trace.push({ source: 'quickstart-registry', id: model, outcome: registryError.message || 'not-found' });
125
- }
126
- }
127
-
128
- // 2. Explicit URL object
129
- if (model && typeof model === 'object' && typeof model.url === 'string' && model.url.trim().length > 0) {
130
- trace.push({ source: 'url', id: model.url.trim(), outcome: 'resolved' });
131
- log.debug('doppler', `Model resolved via explicit url: ${model.url.trim()}`, { trace });
132
- return {
133
- modelId: model.url.trim(),
134
- baseUrl: model.url.trim(),
135
- manifest: null,
136
- trace,
137
- };
138
- }
139
- if (model && typeof model === 'object' && typeof model.url === 'string') {
140
- trace.push({ source: 'url', id: String(model.url), outcome: 'empty-url' });
141
- }
142
-
143
- // 3. Inline manifest object
144
- if (model && typeof model === 'object' && model.manifest && typeof model.manifest === 'object') {
145
- const manifest = model.manifest;
146
- const modelId = typeof manifest.modelId === 'string' && manifest.modelId.length > 0
147
- ? manifest.modelId
148
- : 'manifest';
149
- trace.push({ source: 'inline-manifest', id: modelId, outcome: 'resolved' });
150
- log.debug('doppler', `Model resolved via inline manifest: ${modelId}`, { trace });
151
- return {
152
- modelId,
153
- baseUrl: typeof model.baseUrl === 'string' && model.baseUrl.length > 0 ? model.baseUrl : null,
154
- manifest,
155
- trace,
156
- };
157
- }
158
-
159
- // No source matched — build a diagnostic error with the resolution trace
160
- const traceDescription = trace.length > 0
161
- ? trace.map((entry) => `${entry.source} (${entry.outcome})`).join(', ')
162
- : 'no sources attempted';
163
- throw new Error(
164
- `Model not found. Attempted: ${traceDescription}. ` +
165
- 'doppler.load expects a quickstart registry id, { url }, or { manifest, baseUrl? }.'
166
- );
167
- }
168
-
169
- function countTokens(pipeline, text) {
170
- if (!text || typeof text !== 'string') return 0;
171
- try {
172
- return pipeline?.tokenizer?.encode(text)?.length ?? 0;
173
- } catch {
174
- return 0;
175
- }
176
- }
177
-
178
- function resolveChatPromptForUsage(pipeline, messages) {
179
- const templateType = pipeline?.manifest?.inference?.chatTemplate?.enabled === false
180
- ? null
181
- : (pipeline?.manifest?.inference?.chatTemplate?.type ?? null);
182
- try {
183
- return formatChatMessages(messages, templateType);
184
- } catch {
185
- return messages.map((message) => String(message?.content ?? '')).join('\n');
186
- }
187
- }
188
-
189
- async function collectText(iterable) {
190
- let output = '';
191
- for await (const token of iterable) {
192
- output += token;
193
- }
194
- return output;
195
- }
196
-
197
- function assertNoLoadAffectingOptions(apiName, options) {
198
- if (!options || typeof options !== 'object') {
199
- return;
200
- }
201
- if (
202
- options.runtimeConfig !== undefined
203
- || options.runtimeProfile !== undefined
204
- || options.runtimeConfigUrl !== undefined
205
- ) {
206
- throw new Error(
207
- `${apiName} does not accept load-affecting options. Use doppler.load(model, options) instead.`
208
- );
209
- }
210
- }
211
-
212
- function createModelHandle(pipeline, resolved) {
213
- return {
214
- generate(prompt, options = {}) {
215
- return pipeline.generate(prompt, options);
216
- },
217
- async generateText(prompt, options = {}) {
218
- return collectText(pipeline.generate(prompt, options));
219
- },
220
- chat(messages, options = {}) {
221
- return pipeline.generate(messages, options);
222
- },
223
- async chatText(messages, options = {}) {
224
- const content = await collectText(pipeline.generate(messages, options));
225
- const promptText = resolveChatPromptForUsage(pipeline, messages);
226
- const promptTokens = countTokens(pipeline, promptText);
227
- const completionTokens = countTokens(pipeline, content);
228
- return {
229
- content,
230
- usage: {
231
- promptTokens,
232
- completionTokens,
233
- totalTokens: promptTokens + completionTokens,
234
- },
235
- };
236
- },
237
- async loadLoRA(adapter) {
238
- const lora = typeof adapter === 'string'
239
- ? await loadLoRAFromUrl(adapter)
240
- : await loadLoRAFromManifest(adapter);
241
- pipeline.setLoRAAdapter(lora);
242
- },
243
- async unloadLoRA() {
244
- pipeline.setLoRAAdapter(null);
245
- },
246
- async unload() {
247
- await pipeline.unload();
248
- },
249
- get activeLoRA() {
250
- return pipeline.getActiveLoRA()?.name ?? null;
251
- },
252
- get loaded() {
253
- return pipeline.isLoaded === true;
254
- },
255
- get modelId() {
256
- return resolved.modelId;
257
- },
258
- get manifest() {
259
- return pipeline.manifest;
260
- },
261
- get deviceInfo() {
262
- return getKernelCapabilities()?.adapterInfo ?? null;
263
- },
264
- advanced: {
265
- prefillKV(prompt, options = {}) {
266
- return pipeline.prefillKVOnly(prompt, options);
267
- },
268
- prefillWithLogits(prompt, options = {}) {
269
- return pipeline.prefillWithLogits(prompt, options);
270
- },
271
- decodeStepLogits(currentIds, options = {}) {
272
- return pipeline.decodeStepLogits(currentIds, options);
273
- },
274
- generateWithPrefixKV(prefix, prompt, options = {}) {
275
- return pipeline.generateWithPrefixKV(prefix, prompt, options);
276
- },
277
- },
278
- };
279
- }
280
-
281
- export async function load(model, options = {}) {
282
- const { userProgress, pipelineProgress } = resolveLoadProgressHandlers(options);
283
-
284
- emitLoadProgress(userProgress, 'resolve', 5, 'Resolving model');
285
- const resolved = await resolveModelSource(model);
286
- await ensureWebGPUAvailable();
287
-
288
- emitLoadProgress(userProgress, 'manifest', 15, 'Fetching manifest');
289
- const manifest = resolved.manifest ?? await fetchManifestFromBaseUrl(resolved.baseUrl);
290
-
291
- emitLoadProgress(userProgress, 'load', 25, 'Loading weights');
292
- const pipeline = await createPipeline(manifest, {
293
- baseUrl: resolved.baseUrl ?? undefined,
294
- runtimeConfig: options.runtimeConfig,
295
- onProgress: pipelineProgress
296
- ? (progress) => emitLoadProgress(
297
- pipelineProgress,
298
- 'load',
299
- Math.max(25, Math.min(99, Math.round(progress.percent))),
300
- progress.message || 'Loading weights'
301
- )
302
- : undefined,
303
- });
304
-
305
- emitLoadProgress(userProgress, 'ready', 100, 'Model ready');
306
- return createModelHandle(pipeline, resolved);
35
+ return runtime.resolveLoadProgressHandlers(options);
307
36
  }
308
37
 
309
- async function getCachedModel(model, options = {}) {
310
- const resolved = await resolveModelSource(model);
311
- const cacheKey = resolved.modelId;
312
- const cached = convenienceModelCache.get(cacheKey);
313
- if (cached?.loaded) {
314
- return cached;
315
- }
316
- if (cached && !cached.loaded) {
317
- convenienceModelCache.delete(cacheKey);
318
- }
319
- if (!inFlightLoadCache.has(cacheKey)) {
320
- inFlightLoadCache.set(cacheKey, load(model, options).then((instance) => {
321
- convenienceModelCache.set(cacheKey, instance);
322
- inFlightLoadCache.delete(cacheKey);
323
- return instance;
324
- }).catch((error) => {
325
- inFlightLoadCache.delete(cacheKey);
326
- throw error;
327
- }));
328
- }
329
- return inFlightLoadCache.get(cacheKey);
330
- }
331
-
332
- async function* dopplerGenerate(prompt, options = {}) {
333
- if (!options || typeof options !== 'object' || options.model == null) {
334
- throw new Error('doppler() requires options.model.');
335
- }
336
- assertNoLoadAffectingOptions('doppler()', options);
337
- const model = await getCachedModel(options.model, { onProgress: options.onProgress });
338
- yield* model.generate(prompt, options);
339
- }
340
-
341
- export function doppler(prompt, options) {
342
- return dopplerGenerate(prompt, options);
343
- }
344
-
345
- doppler.load = load;
346
- doppler.text = async function text(prompt, options) {
347
- if (!options || typeof options !== 'object' || options.model == null) {
348
- throw new Error('doppler.text() requires options.model.');
349
- }
350
- assertNoLoadAffectingOptions('doppler.text()', options);
351
- return collectText(doppler(prompt, options));
352
- };
353
- doppler.chat = function chat(messages, options = {}) {
354
- if (!options || typeof options !== 'object' || options.model == null) {
355
- throw new Error('doppler.chat() requires options.model.');
356
- }
357
- assertNoLoadAffectingOptions('doppler.chat()', options);
358
- return (async function* () {
359
- const model = await getCachedModel(options.model, { onProgress: options.onProgress });
360
- yield* model.chat(messages, options);
361
- }());
362
- };
363
- doppler.chatText = async function chatText(messages, options = {}) {
364
- if (!options || typeof options !== 'object' || options.model == null) {
365
- throw new Error('doppler.chatText() requires options.model.');
366
- }
367
- assertNoLoadAffectingOptions('doppler.chatText()', options);
368
- const model = await getCachedModel(options.model, { onProgress: options.onProgress });
369
- return model.chatText(messages, options);
370
- };
371
- doppler.evict = async function evict(model) {
372
- const resolved = await resolveModelSource(model);
373
- const cached = convenienceModelCache.get(resolved.modelId);
374
- if (!cached) {
375
- return false;
376
- }
377
- await cached.unload();
378
- convenienceModelCache.delete(resolved.modelId);
379
- return true;
380
- };
381
- doppler.evictAll = async function evictAll() {
382
- const cached = [...convenienceModelCache.values()];
383
- convenienceModelCache.clear();
384
- await Promise.allSettled(cached.map((entry) => entry.unload()));
385
- };
386
- doppler.listModels = async function listModels() {
387
- const models = await listQuickstartModels();
388
- return models.map((entry) => entry.modelId);
389
- };
38
+ export default doppler;
@@ -17,7 +17,7 @@
17
17
  ],
18
18
  "hf": {
19
19
  "repoId": "Clocksmith/rdrr",
20
- "revision": "f5698592e69b97c38f21c68e01db2216c2f40965",
20
+ "revision": "f242e44c9a35e72186aa61e4cc0c9873f0596741",
21
21
  "path": "models/gemma-3-270m-it-q4k-ehf16-af32"
22
22
  }
23
23
  },
@@ -33,9 +33,62 @@
33
33
  ],
34
34
  "hf": {
35
35
  "repoId": "Clocksmith/rdrr",
36
- "revision": "9e4ef13a43ed47eda17bdc271ae90632eb22b9c7",
36
+ "revision": "f242e44c9a35e72186aa61e4cc0c9873f0596741",
37
37
  "path": "models/google-embeddinggemma-300m-q4k-ehf16-af32"
38
38
  }
39
+ },
40
+ {
41
+ "modelId": "gemma-3-1b-it-q4k-ehf16-af32",
42
+ "aliases": [
43
+ "gemma3-1b",
44
+ "google/gemma-3-1b-it",
45
+ "gemma-3-1b-it-wq4k-ef16-hf16"
46
+ ],
47
+ "modes": [
48
+ "text",
49
+ "vision"
50
+ ],
51
+ "hf": {
52
+ "repoId": "Clocksmith/rdrr",
53
+ "revision": "a6651ea48c5ebc420ebc1c6972979049b72db96e",
54
+ "path": "models/gemma-3-1b-it-q4k-ehf16-af32"
55
+ }
56
+ },
57
+ {
58
+ "modelId": "qwen-3-5-0-8b-q4k-ehaf16",
59
+ "aliases": [
60
+ "qwen3-0.8b",
61
+ "qwen-3-5-0-8b",
62
+ "Qwen/Qwen3.5-0.8B",
63
+ "qwen-3-5-0-8b-wq4k-ef16-hf16-f16"
64
+ ],
65
+ "modes": [
66
+ "text",
67
+ "vision"
68
+ ],
69
+ "hf": {
70
+ "repoId": "Clocksmith/rdrr",
71
+ "revision": "0b7e01c70500b78cbe69c66cb5601dd00941f9c9",
72
+ "path": "models/qwen-3-5-0-8b-q4k-ehaf16"
73
+ }
74
+ },
75
+ {
76
+ "modelId": "qwen-3-5-2b-q4k-ehaf16",
77
+ "aliases": [
78
+ "qwen3-2b",
79
+ "qwen-3-5-2b",
80
+ "Qwen/Qwen3.5-2B",
81
+ "qwen-3-5-2b-wq4k-ef16-hf16-f16"
82
+ ],
83
+ "modes": [
84
+ "text",
85
+ "vision"
86
+ ],
87
+ "hf": {
88
+ "repoId": "Clocksmith/rdrr",
89
+ "revision": "1e14085a6d8e14035ec64966d6cd44250be3df83",
90
+ "path": "models/qwen-3-5-2b-q4k-ehaf16"
91
+ }
39
92
  }
40
93
  ]
41
94
  }
@@ -0,0 +1,63 @@
1
+ import type { ChatMessage } from '../../inference/pipelines/text/chat-format.js';
2
+ import type { GenerateOptions } from '../../generation/index.js';
3
+ import type { LogitsStepResult, PrefillResult } from '../../inference/pipelines/text/types.d.ts';
4
+ import type {
5
+ DopplerLoadOptions,
6
+ DopplerLoadProgress,
7
+ DopplerModelSource,
8
+ } from './model-source.js';
9
+ import type { DopplerModelHandle } from './model-session.js';
10
+
11
+ export type DopplerGenerateOptions = Omit<GenerateOptions, 'stopTokens'>;
12
+
13
+ export interface DopplerChatResponse {
14
+ content: string;
15
+ usage: {
16
+ promptTokens: number;
17
+ completionTokens: number;
18
+ totalTokens: number;
19
+ };
20
+ }
21
+
22
+ export interface DopplerNamespace {
23
+ (prompt: string, options: DopplerCallOptions): AsyncGenerator<string, void, void>;
24
+ load(model: DopplerModelSource, options?: DopplerLoadOptions): Promise<DopplerModelHandle>;
25
+ text(prompt: string, options: DopplerCallOptions): Promise<string>;
26
+ chat(messages: ChatMessage[], options: DopplerCallOptions): AsyncGenerator<string, void, void>;
27
+ chatText(messages: ChatMessage[], options: DopplerCallOptions): Promise<DopplerChatResponse>;
28
+ evict(model: DopplerModelSource): Promise<boolean>;
29
+ evictAll(): Promise<void>;
30
+ listModels(): Promise<string[]>;
31
+ }
32
+
33
+ export interface DopplerCallOptions extends DopplerGenerateOptions {
34
+ model: DopplerModelSource;
35
+ onProgress?: (event: DopplerLoadProgress) => void;
36
+ }
37
+
38
+ export interface DopplerRuntimeService {
39
+ doppler: DopplerNamespace;
40
+ load(model: DopplerModelSource, options?: DopplerLoadOptions): Promise<DopplerModelHandle>;
41
+ clearModelCache(): void;
42
+ resolveLoadProgressHandlers(options?: DopplerLoadOptions): {
43
+ userProgress: ((event: DopplerLoadProgress) => void) | null;
44
+ pipelineProgress: ((event: DopplerLoadProgress) => void) | null;
45
+ };
46
+ createDefaultNodeLoadProgressLogger(): (event: DopplerLoadProgress) => void;
47
+ }
48
+
49
+ export declare function createDopplerRuntimeService(options: {
50
+ ensureWebGPUAvailable: () => Promise<void>;
51
+ defaultLoadProgressLogger?: ((event: DopplerLoadProgress) => void) | null;
52
+ }): DopplerRuntimeService;
53
+
54
+ export type {
55
+ DopplerLoadOptions,
56
+ DopplerLoadProgress,
57
+ DopplerModelSource,
58
+ DopplerModelSourceResolution,
59
+ } from './model-source.js';
60
+
61
+ export type { DopplerModelHandle } from './model-session.js';
62
+ export type { PrefillResult, LogitsStepResult } from '../../inference/pipelines/text/types.d.ts';
63
+ export type { LoRAManifest, ExtensionBridgeClient } from './types.js';
@@ -0,0 +1,173 @@
1
+ import { log } from '../../debug/index.js';
2
+ import { createPipeline } from '../../generation/index.js';
3
+ import { listQuickstartModels } from '../doppler-registry.js';
4
+ import {
5
+ createDefaultNodeLoadProgressLogger,
6
+ fetchManifestFromBaseUrl,
7
+ resolveLoadProgressHandlers,
8
+ resolveModelSource,
9
+ } from './model-source.js';
10
+ import { assertSupportedGenerationOptions, createModelHandle } from './model-session.js';
11
+
12
+ function emitLoadProgress(callback, phase, percent, message) {
13
+ if (typeof callback !== 'function') return;
14
+ callback({ phase, percent, message });
15
+ }
16
+
17
+ function assertDopplerOptions(options, apiName) {
18
+ if (!options || typeof options !== 'object') {
19
+ return;
20
+ }
21
+ if (
22
+ options.runtimeConfig !== undefined
23
+ || options.runtimeProfile !== undefined
24
+ || options.runtimeConfigUrl !== undefined
25
+ ) {
26
+ throw new Error(
27
+ `${apiName} does not accept load-affecting options. Use doppler.load(model, options) instead.`
28
+ );
29
+ }
30
+ }
31
+
32
+ export function createDopplerRuntimeService({
33
+ ensureWebGPUAvailable,
34
+ defaultLoadProgressLogger = null,
35
+ } = {}) {
36
+ if (typeof ensureWebGPUAvailable !== 'function') {
37
+ throw new Error('createDopplerRuntimeService requires ensureWebGPUAvailable.');
38
+ }
39
+
40
+ const convenienceModelCache = new Map();
41
+ const inFlightLoadCache = new Map();
42
+
43
+ function clearModelCache() {
44
+ convenienceModelCache.clear();
45
+ inFlightLoadCache.clear();
46
+ log.debug('doppler', 'Model cache cleared');
47
+ }
48
+
49
+ async function load(model, options = {}) {
50
+ const { userProgress, pipelineProgress } = resolveLoadProgressHandlers(options, defaultLoadProgressLogger);
51
+
52
+ emitLoadProgress(userProgress, 'resolve', 5, 'Resolving model');
53
+ const resolved = await resolveModelSource(model);
54
+ await ensureWebGPUAvailable();
55
+
56
+ emitLoadProgress(userProgress, 'manifest', 15, 'Fetching manifest');
57
+ const manifest = resolved.manifest ?? await fetchManifestFromBaseUrl(resolved.baseUrl);
58
+
59
+ emitLoadProgress(userProgress, 'load', 25, 'Loading weights');
60
+ const pipeline = await createPipeline(manifest, {
61
+ baseUrl: resolved.baseUrl ?? undefined,
62
+ runtimeConfig: options.runtimeConfig,
63
+ onProgress: pipelineProgress
64
+ ? (progress) => emitLoadProgress(
65
+ pipelineProgress,
66
+ 'load',
67
+ Math.max(25, Math.min(99, Math.round(progress.percent))),
68
+ progress.message || 'Loading weights'
69
+ )
70
+ : undefined,
71
+ });
72
+
73
+ emitLoadProgress(userProgress, 'ready', 100, 'Model ready');
74
+ return createModelHandle(pipeline, resolved);
75
+ }
76
+
77
+ async function getCachedModel(model, options = {}) {
78
+ const resolved = await resolveModelSource(model);
79
+ const cacheKey = resolved.modelId;
80
+ const cached = convenienceModelCache.get(cacheKey);
81
+ if (cached?.loaded) {
82
+ return cached;
83
+ }
84
+ if (cached && !cached.loaded) {
85
+ convenienceModelCache.delete(cacheKey);
86
+ }
87
+ if (!inFlightLoadCache.has(cacheKey)) {
88
+ inFlightLoadCache.set(cacheKey, load(model, options).then((instance) => {
89
+ convenienceModelCache.set(cacheKey, instance);
90
+ inFlightLoadCache.delete(cacheKey);
91
+ return instance;
92
+ }).catch((error) => {
93
+ inFlightLoadCache.delete(cacheKey);
94
+ throw error;
95
+ }));
96
+ }
97
+ return inFlightLoadCache.get(cacheKey);
98
+ }
99
+
100
+ async function* dopplerGenerate(prompt, options = {}) {
101
+ if (!options || typeof options !== 'object' || options.model == null) {
102
+ throw new Error('doppler() requires options.model.');
103
+ }
104
+ assertDopplerOptions(options, 'doppler()');
105
+ assertSupportedGenerationOptions(options);
106
+ const model = await getCachedModel(options.model, { onProgress: options.onProgress });
107
+ yield* model.generate(prompt, options);
108
+ }
109
+
110
+ function doppler(prompt, options) {
111
+ return dopplerGenerate(prompt, options);
112
+ }
113
+
114
+ doppler.load = load;
115
+ doppler.text = async function text(prompt, options = {}) {
116
+ if (!options || typeof options !== 'object' || options.model == null) {
117
+ throw new Error('doppler.text() requires options.model.');
118
+ }
119
+ assertDopplerOptions(options, 'doppler.text()');
120
+ assertSupportedGenerationOptions(options);
121
+ const model = await getCachedModel(options.model, { onProgress: options.onProgress });
122
+ return model.generateText(prompt, options);
123
+ };
124
+ doppler.chat = function chat(messages, options = {}) {
125
+ if (!options || typeof options !== 'object' || options.model == null) {
126
+ throw new Error('doppler.chat() requires options.model.');
127
+ }
128
+ assertDopplerOptions(options, 'doppler.chat()');
129
+ assertSupportedGenerationOptions(options);
130
+ return (async function* run() {
131
+ const model = await getCachedModel(options.model, { onProgress: options.onProgress });
132
+ yield* model.chat(messages, options);
133
+ }());
134
+ };
135
+ doppler.chatText = async function chatText(messages, options = {}) {
136
+ if (!options || typeof options !== 'object' || options.model == null) {
137
+ throw new Error('doppler.chatText() requires options.model.');
138
+ }
139
+ assertDopplerOptions(options, 'doppler.chatText()');
140
+ assertSupportedGenerationOptions(options);
141
+ const model = await getCachedModel(options.model, { onProgress: options.onProgress });
142
+ return model.chatText(messages, options);
143
+ };
144
+ doppler.evict = async function evict(model) {
145
+ const resolved = await resolveModelSource(model);
146
+ const cached = convenienceModelCache.get(resolved.modelId);
147
+ if (!cached) {
148
+ return false;
149
+ }
150
+ await cached.unload();
151
+ convenienceModelCache.delete(resolved.modelId);
152
+ return true;
153
+ };
154
+ doppler.evictAll = async function evictAll() {
155
+ const cached = [...convenienceModelCache.values()];
156
+ convenienceModelCache.clear();
157
+ await Promise.allSettled(cached.map((entry) => entry.unload()));
158
+ };
159
+ doppler.listModels = async function listModels() {
160
+ const models = await listQuickstartModels();
161
+ return models.map((entry) => entry.modelId);
162
+ };
163
+
164
+ return {
165
+ doppler,
166
+ load,
167
+ clearModelCache,
168
+ resolveLoadProgressHandlers(options = {}) {
169
+ return resolveLoadProgressHandlers(options, defaultLoadProgressLogger);
170
+ },
171
+ createDefaultNodeLoadProgressLogger,
172
+ };
173
+ }