doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -0,0 +1,299 @@
1
+ #!/usr/bin/env node
2
+
3
+ import fs from 'node:fs/promises';
4
+ import path from 'node:path';
5
+ import process from 'node:process';
6
+ import { fileURLToPath } from 'node:url';
7
+ import { DOPPLER_VERSION, doppler } from '../index.js';
8
+ import { listQuickstartModels, resolveQuickstartModel } from '../client/doppler-registry.js';
9
+
10
+ const QUICKSTART_CONFIG_PATH = fileURLToPath(new URL('./config/doppler-quickstart.json', import.meta.url));
11
+ const JSON_SCHEMA_VERSION = 1;
12
+
13
+ function usage() {
14
+ return [
15
+ 'Usage:',
16
+ ' npx doppler-gpu [prompt] [--model <id>] [--max-tokens <n>] [--temperature <n>] [--json]',
17
+ ' npx doppler-gpu --list-models',
18
+ ' npx doppler-gpu --help',
19
+ '',
20
+ 'Examples:',
21
+ ' npx doppler-gpu',
22
+ ' npx doppler-gpu "Summarize WebGPU in one sentence"',
23
+ ' npx doppler-gpu --model gemma3-270m --prompt "Write a haiku about GPUs"',
24
+ ' npx doppler-gpu --list-models',
25
+ '',
26
+ 'Notes:',
27
+ ' - First run downloads model files from the quickstart registry.',
28
+ ' - This quickstart bin is separate from the heavier tooling CLI (`doppler`).',
29
+ ' - For benchmark/debug/verify workflows, use `doppler <command> --config ...`.',
30
+ ].join('\n');
31
+ }
32
+
33
+ function asStringOrNull(value) {
34
+ if (value === undefined || value === null) return null;
35
+ if (typeof value === 'string') {
36
+ const trimmed = value.trim();
37
+ return trimmed.length > 0 ? trimmed : null;
38
+ }
39
+ return String(value);
40
+ }
41
+
42
+ function parseNumberFlag(value, label) {
43
+ const normalized = asStringOrNull(value);
44
+ if (normalized === null) return null;
45
+ const parsed = Number(normalized);
46
+ if (!Number.isFinite(parsed)) {
47
+ throw new Error(`${label} must be a number.`);
48
+ }
49
+ return parsed;
50
+ }
51
+
52
+ export function parseQuickstartArgs(argv) {
53
+ const flags = {
54
+ model: null,
55
+ prompt: null,
56
+ maxTokens: null,
57
+ temperature: null,
58
+ json: false,
59
+ help: false,
60
+ listModels: false,
61
+ };
62
+ const positional = [];
63
+
64
+ for (let index = 0; index < argv.length; index += 1) {
65
+ const token = argv[index];
66
+ if (token === '--help' || token === '-h') {
67
+ flags.help = true;
68
+ continue;
69
+ }
70
+ if (token === '--json') {
71
+ flags.json = true;
72
+ continue;
73
+ }
74
+ if (token === '--list-models') {
75
+ flags.listModels = true;
76
+ continue;
77
+ }
78
+ if (!token.startsWith('--')) {
79
+ positional.push(token);
80
+ continue;
81
+ }
82
+
83
+ const nextValue = argv[index + 1];
84
+ if (nextValue === undefined || nextValue.startsWith('--')) {
85
+ throw new Error(`Missing value for ${token}.`);
86
+ }
87
+
88
+ if (token === '--model') {
89
+ flags.model = nextValue;
90
+ index += 1;
91
+ continue;
92
+ }
93
+ if (token === '--prompt') {
94
+ flags.prompt = nextValue;
95
+ index += 1;
96
+ continue;
97
+ }
98
+ if (token === '--max-tokens') {
99
+ flags.maxTokens = nextValue;
100
+ index += 1;
101
+ continue;
102
+ }
103
+ if (token === '--temperature') {
104
+ flags.temperature = nextValue;
105
+ index += 1;
106
+ continue;
107
+ }
108
+
109
+ throw new Error(`Unknown flag ${token}.`);
110
+ }
111
+
112
+ return {
113
+ ...flags,
114
+ positionalPrompt: positional.length > 0 ? positional.join(' ') : null,
115
+ };
116
+ }
117
+
118
+ export async function readQuickstartConfig() {
119
+ const raw = await fs.readFile(QUICKSTART_CONFIG_PATH, 'utf8');
120
+ const parsed = JSON.parse(raw);
121
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
122
+ throw new Error('doppler-gpu quickstart config must be a JSON object.');
123
+ }
124
+ if (!parsed.defaults || typeof parsed.defaults !== 'object' || Array.isArray(parsed.defaults)) {
125
+ throw new Error('doppler-gpu quickstart config must define a defaults object.');
126
+ }
127
+ return parsed;
128
+ }
129
+
130
+ function resolvePrompt(parsed, defaults) {
131
+ const positionalPrompt = asStringOrNull(parsed.positionalPrompt);
132
+ if (positionalPrompt !== null) {
133
+ return positionalPrompt;
134
+ }
135
+ const flagPrompt = asStringOrNull(parsed.prompt);
136
+ if (flagPrompt !== null) {
137
+ return flagPrompt;
138
+ }
139
+ return asStringOrNull(defaults.prompt);
140
+ }
141
+
142
+ export async function resolveQuickstartSettings(argv = process.argv.slice(2)) {
143
+ const parsed = parseQuickstartArgs(argv);
144
+ if (parsed.help) {
145
+ return { action: 'help' };
146
+ }
147
+ if (parsed.listModels) {
148
+ return {
149
+ action: 'list-models',
150
+ json: parsed.json === true,
151
+ };
152
+ }
153
+
154
+ const config = await readQuickstartConfig();
155
+ const defaults = config.defaults || {};
156
+ const model = asStringOrNull(parsed.model) ?? asStringOrNull(defaults.model);
157
+ const prompt = resolvePrompt(parsed, defaults);
158
+ const maxTokens = parseNumberFlag(parsed.maxTokens, '--max-tokens') ?? parseNumberFlag(defaults.maxTokens, 'quickstart.defaults.maxTokens');
159
+ const temperature = parseNumberFlag(parsed.temperature, '--temperature') ?? parseNumberFlag(defaults.temperature, 'quickstart.defaults.temperature');
160
+ const topK = parseNumberFlag(defaults.topK, 'quickstart.defaults.topK');
161
+
162
+ if (model === null) {
163
+ throw new Error('doppler-gpu quickstart requires a default or explicit model id.');
164
+ }
165
+ if (prompt === null) {
166
+ throw new Error('doppler-gpu quickstart requires a default or explicit prompt.');
167
+ }
168
+ if (maxTokens === null) {
169
+ throw new Error('doppler-gpu quickstart requires a default or explicit max token count.');
170
+ }
171
+ if (temperature === null) {
172
+ throw new Error('doppler-gpu quickstart requires a default or explicit temperature.');
173
+ }
174
+ if (topK === null) {
175
+ throw new Error('doppler-gpu quickstart requires a default topK.');
176
+ }
177
+
178
+ return {
179
+ action: 'run',
180
+ json: parsed.json === true,
181
+ model,
182
+ prompt,
183
+ maxTokens,
184
+ temperature,
185
+ topK,
186
+ };
187
+ }
188
+
189
+ function formatOneLine(value) {
190
+ return JSON.stringify(String(value ?? '').replace(/\s+/g, ' ').trim());
191
+ }
192
+
193
+ function formatDeviceSummary(deviceInfo) {
194
+ if (!deviceInfo || typeof deviceInfo !== 'object') {
195
+ return 'unknown-device';
196
+ }
197
+ const vendor = asStringOrNull(deviceInfo.vendor) ?? asStringOrNull(deviceInfo.vendorId);
198
+ const architecture = asStringOrNull(deviceInfo.architecture);
199
+ const description = asStringOrNull(deviceInfo.description);
200
+ const parts = [vendor, architecture, description].filter(Boolean);
201
+ return parts.length > 0 ? parts.join(' / ') : 'unknown-device';
202
+ }
203
+
204
+ export async function runQuickstart(settings) {
205
+ const entry = await resolveQuickstartModel(settings.model);
206
+ if (!entry.modes.includes('text')) {
207
+ throw new Error(
208
+ `Quickstart model "${settings.model}" is not text-generative. `
209
+ + 'Use --list-models to see supported text quickstart models.'
210
+ );
211
+ }
212
+
213
+ const start = Date.now();
214
+ const model = await doppler.load(settings.model, {
215
+ onProgress: settings.json ? () => {} : undefined,
216
+ });
217
+ try {
218
+ const result = await model.chatText([
219
+ {
220
+ role: 'user',
221
+ content: settings.prompt,
222
+ },
223
+ ], {
224
+ maxTokens: settings.maxTokens,
225
+ temperature: settings.temperature,
226
+ topK: settings.topK,
227
+ });
228
+ return {
229
+ ok: true,
230
+ schemaVersion: JSON_SCHEMA_VERSION,
231
+ version: DOPPLER_VERSION,
232
+ requestedModel: settings.model,
233
+ modelId: model.modelId,
234
+ prompt: settings.prompt,
235
+ content: result.content,
236
+ usage: result.usage,
237
+ elapsedMs: Date.now() - start,
238
+ deviceInfo: model.deviceInfo ?? null,
239
+ };
240
+ } finally {
241
+ await model.unload();
242
+ }
243
+ }
244
+
245
+ async function printModelList(jsonOutput) {
246
+ const models = await listQuickstartModels();
247
+ const textModels = models.filter((entry) => entry.modes.includes('text'));
248
+ if (jsonOutput) {
249
+ console.log(JSON.stringify({ ok: true, schemaVersion: JSON_SCHEMA_VERSION, models: textModels }, null, 2));
250
+ return;
251
+ }
252
+ console.log('Text quickstart models:');
253
+ for (const entry of textModels) {
254
+ const aliasSuffix = entry.aliases.length > 0 ? ` (${entry.aliases.join(', ')})` : '';
255
+ console.log(`- ${entry.modelId}${aliasSuffix}`);
256
+ }
257
+ }
258
+
259
+ function printQuickstartResult(result) {
260
+ console.log(result.content);
261
+ console.error(
262
+ `[doppler-gpu] model=${result.modelId} elapsed=${result.elapsedMs}ms device=${formatDeviceSummary(result.deviceInfo)}`
263
+ );
264
+ }
265
+
266
+ export async function main(argv = process.argv.slice(2)) {
267
+ const settings = await resolveQuickstartSettings(argv);
268
+ if (settings.action === 'help') {
269
+ console.log(usage());
270
+ return;
271
+ }
272
+ if (settings.action === 'list-models') {
273
+ await printModelList(settings.json);
274
+ return;
275
+ }
276
+
277
+ const result = await runQuickstart(settings);
278
+ if (settings.json) {
279
+ console.log(JSON.stringify(result, null, 2));
280
+ return;
281
+ }
282
+ console.error(`[doppler-gpu] requestedModel=${settings.model} prompt=${formatOneLine(settings.prompt)}`);
283
+ printQuickstartResult(result);
284
+ }
285
+
286
+ function isMainModule(metaUrl) {
287
+ const entryPath = process.argv[1];
288
+ if (!entryPath) {
289
+ return false;
290
+ }
291
+ return path.resolve(fileURLToPath(metaUrl)) === path.resolve(entryPath);
292
+ }
293
+
294
+ if (isMainModule(import.meta.url)) {
295
+ main().catch((error) => {
296
+ console.error(`[doppler-gpu] ${error?.message || String(error)}`);
297
+ process.exit(1);
298
+ });
299
+ }
@@ -0,0 +1,395 @@
1
+ #!/usr/bin/env node
2
+
3
+ import http from 'node:http';
4
+ import crypto from 'node:crypto';
5
+ import path from 'node:path';
6
+ import process from 'node:process';
7
+ import { fileURLToPath } from 'node:url';
8
+ import { DOPPLER_VERSION, doppler } from '../index.js';
9
+ import { listQuickstartModels } from '../client/doppler-registry.js';
10
+
11
+ const DEFAULT_PORT = 8080;
12
+ const DEFAULT_HOST = '127.0.0.1';
13
+
14
+ function usage() {
15
+ return [
16
+ 'Usage:',
17
+ ' doppler-serve [--model <id>] [--port <n>] [--host <addr>]',
18
+ '',
19
+ 'Options:',
20
+ ' --model <id> Pre-load a model at startup (optional, lazy-loads on request otherwise)',
21
+ ' --port <n> Port to listen on (default: 8080)',
22
+ ' --host <addr> Host to bind to (default: 127.0.0.1)',
23
+ ' --help Show this help',
24
+ '',
25
+ 'Endpoints:',
26
+ ' POST /v1/chat/completions OpenAI-compatible chat completions',
27
+ ' GET /v1/models List available models',
28
+ ' GET /health Health check',
29
+ '',
30
+ 'Examples:',
31
+ ' node src/cli/doppler-serve.js --model gemma3-270m',
32
+ ' node src/cli/doppler-serve.js --model qwen3-0.8b --port 3000',
33
+ '',
34
+ 'Then use with any OpenAI-compatible client:',
35
+ ' curl http://localhost:8080/v1/chat/completions \\',
36
+ ' -H "Content-Type: application/json" \\',
37
+ ' -d \'{"model":"gemma3-270m","messages":[{"role":"user","content":"Hello"}]}\'',
38
+ ].join('\n');
39
+ }
40
+
41
+ export function parseServeArgs(argv) {
42
+ const flags = {
43
+ port: DEFAULT_PORT,
44
+ host: DEFAULT_HOST,
45
+ model: null,
46
+ help: false,
47
+ };
48
+ for (let i = 0; i < argv.length; i += 1) {
49
+ const token = argv[i];
50
+ if (token === '--help' || token === '-h') {
51
+ flags.help = true;
52
+ continue;
53
+ }
54
+ if (!token.startsWith('--')) {
55
+ throw new Error(`Unexpected positional argument: ${token}`);
56
+ }
57
+ if (token !== '--port' && token !== '--host' && token !== '--model') {
58
+ throw new Error(`Unknown flag ${token}.`);
59
+ }
60
+ const nextValue = argv[i + 1];
61
+ if (nextValue === undefined || nextValue.startsWith('--')) {
62
+ throw new Error(`Missing value for ${token}.`);
63
+ }
64
+ if (token === '--port') {
65
+ const parsed = Number(nextValue);
66
+ if (!Number.isFinite(parsed) || parsed < 0 || parsed > 65535 || parsed !== Math.floor(parsed)) {
67
+ throw new Error('--port must be a valid port number (0-65535).');
68
+ }
69
+ flags.port = parsed;
70
+ i += 1;
71
+ continue;
72
+ }
73
+ if (token === '--host') {
74
+ flags.host = nextValue.trim();
75
+ i += 1;
76
+ continue;
77
+ }
78
+ if (token === '--model') {
79
+ flags.model = nextValue.trim();
80
+ i += 1;
81
+ continue;
82
+ }
83
+ throw new Error(`Unknown flag ${token}.`);
84
+ }
85
+ return flags;
86
+ }
87
+
88
+ function generateCompletionId() {
89
+ return `chatcmpl-${crypto.randomBytes(12).toString('base64url')}`;
90
+ }
91
+
92
+ function jsonError(res, statusCode, message, type) {
93
+ const body = JSON.stringify({
94
+ error: {
95
+ message,
96
+ type: type ?? 'invalid_request_error',
97
+ param: null,
98
+ code: null,
99
+ },
100
+ });
101
+ res.writeHead(statusCode, {
102
+ 'Content-Type': 'application/json',
103
+ 'Access-Control-Allow-Origin': '*',
104
+ });
105
+ res.end(body);
106
+ }
107
+
108
+ function jsonResponse(res, statusCode, data) {
109
+ const body = JSON.stringify(data);
110
+ res.writeHead(statusCode, {
111
+ 'Content-Type': 'application/json',
112
+ 'Access-Control-Allow-Origin': '*',
113
+ });
114
+ res.end(body);
115
+ }
116
+
117
+ async function readRequestBody(req) {
118
+ const chunks = [];
119
+ for await (const chunk of req) {
120
+ chunks.push(chunk);
121
+ }
122
+ const raw = Buffer.concat(chunks).toString('utf8');
123
+ if (!raw || raw.trim().length === 0) {
124
+ throw new Error('Request body is empty.');
125
+ }
126
+ try {
127
+ return JSON.parse(raw);
128
+ } catch {
129
+ throw new Error('Request body is not valid JSON.');
130
+ }
131
+ }
132
+
133
+ function validateMessages(messages) {
134
+ if (!Array.isArray(messages) || messages.length === 0) {
135
+ throw new Error('"messages" must be a non-empty array.');
136
+ }
137
+ for (let i = 0; i < messages.length; i += 1) {
138
+ const msg = messages[i];
139
+ if (!msg || typeof msg !== 'object') {
140
+ throw new Error(`messages[${i}] must be an object.`);
141
+ }
142
+ if (typeof msg.role !== 'string' || msg.role.trim().length === 0) {
143
+ throw new Error(`messages[${i}].role must be a non-empty string.`);
144
+ }
145
+ if (typeof msg.content !== 'string') {
146
+ throw new Error(`messages[${i}].content must be a string.`);
147
+ }
148
+ }
149
+ return messages.map((msg) => ({ role: msg.role.trim(), content: msg.content }));
150
+ }
151
+
152
+ function extractGenerationOptions(body) {
153
+ const options = {};
154
+ if (body.max_tokens != null) {
155
+ const n = Number(body.max_tokens);
156
+ if (!Number.isFinite(n) || n < 1) {
157
+ throw new Error('"max_tokens" must be a positive integer.');
158
+ }
159
+ options.maxTokens = Math.floor(n);
160
+ }
161
+ if (body.temperature != null) {
162
+ const t = Number(body.temperature);
163
+ if (!Number.isFinite(t) || t < 0) {
164
+ throw new Error('"temperature" must be a non-negative number.');
165
+ }
166
+ options.temperature = t;
167
+ }
168
+ if (body.top_p != null) {
169
+ const p = Number(body.top_p);
170
+ if (!Number.isFinite(p) || p <= 0 || p > 1) {
171
+ throw new Error('"top_p" must be a number between 0 (exclusive) and 1 (inclusive).');
172
+ }
173
+ options.topP = p;
174
+ }
175
+ if (body.top_k != null) {
176
+ const k = Number(body.top_k);
177
+ if (!Number.isFinite(k) || k < 1) {
178
+ throw new Error('"top_k" must be a positive integer.');
179
+ }
180
+ options.topK = Math.floor(k);
181
+ }
182
+ return options;
183
+ }
184
+
185
+ async function handleChatCompletions(req, res) {
186
+ const body = await readRequestBody(req);
187
+ if (typeof body.model !== 'string' || body.model.trim().length === 0) {
188
+ return jsonError(res, 400, '"model" is required and must be a non-empty string.');
189
+ }
190
+ const modelId = body.model.trim();
191
+ const messages = validateMessages(body.messages);
192
+ const generationOptions = extractGenerationOptions(body);
193
+ const stream = body.stream === true;
194
+ const completionId = generateCompletionId();
195
+ const created = Math.floor(Date.now() / 1000);
196
+
197
+ if (stream) {
198
+ return handleStreamingCompletion(res, modelId, messages, generationOptions, completionId, created);
199
+ }
200
+ return handleNonStreamingCompletion(res, modelId, messages, generationOptions, completionId, created);
201
+ }
202
+
203
+ async function handleNonStreamingCompletion(res, modelId, messages, generationOptions, completionId, created) {
204
+ const result = await doppler.chatText(messages, { model: modelId, ...generationOptions });
205
+ jsonResponse(res, 200, {
206
+ id: completionId,
207
+ object: 'chat.completion',
208
+ created,
209
+ model: modelId,
210
+ choices: [
211
+ {
212
+ index: 0,
213
+ message: {
214
+ role: 'assistant',
215
+ content: result.content,
216
+ },
217
+ finish_reason: 'stop',
218
+ },
219
+ ],
220
+ usage: {
221
+ prompt_tokens: result.usage.promptTokens,
222
+ completion_tokens: result.usage.completionTokens,
223
+ total_tokens: result.usage.totalTokens,
224
+ },
225
+ });
226
+ }
227
+
228
+ async function handleStreamingCompletion(res, modelId, messages, generationOptions, completionId, created) {
229
+ res.writeHead(200, {
230
+ 'Content-Type': 'text/event-stream',
231
+ 'Cache-Control': 'no-cache',
232
+ 'Connection': 'keep-alive',
233
+ 'Access-Control-Allow-Origin': '*',
234
+ });
235
+
236
+ function sendChunk(data) {
237
+ res.write(`data: ${JSON.stringify(data)}\n\n`);
238
+ }
239
+
240
+ sendChunk({
241
+ id: completionId,
242
+ object: 'chat.completion.chunk',
243
+ created,
244
+ model: modelId,
245
+ choices: [
246
+ {
247
+ index: 0,
248
+ delta: { role: 'assistant' },
249
+ finish_reason: null,
250
+ },
251
+ ],
252
+ });
253
+
254
+ const stream = doppler.chat(messages, { model: modelId, ...generationOptions });
255
+ for await (const token of stream) {
256
+ if (res.destroyed) {
257
+ break;
258
+ }
259
+ sendChunk({
260
+ id: completionId,
261
+ object: 'chat.completion.chunk',
262
+ created,
263
+ model: modelId,
264
+ choices: [
265
+ {
266
+ index: 0,
267
+ delta: { content: token },
268
+ finish_reason: null,
269
+ },
270
+ ],
271
+ });
272
+ }
273
+
274
+ sendChunk({
275
+ id: completionId,
276
+ object: 'chat.completion.chunk',
277
+ created,
278
+ model: modelId,
279
+ choices: [
280
+ {
281
+ index: 0,
282
+ delta: {},
283
+ finish_reason: 'stop',
284
+ },
285
+ ],
286
+ });
287
+ res.write('data: [DONE]\n\n');
288
+ res.end();
289
+ }
290
+
291
+ async function handleListModels(res) {
292
+ const models = await listQuickstartModels();
293
+ const textModels = models.filter((entry) => entry.modes.includes('text'));
294
+ jsonResponse(res, 200, {
295
+ object: 'list',
296
+ data: textModels.map((entry) => ({
297
+ id: entry.modelId,
298
+ object: 'model',
299
+ created: 0,
300
+ owned_by: 'doppler',
301
+ })),
302
+ });
303
+ }
304
+
305
+ function handleHealth(res) {
306
+ jsonResponse(res, 200, {
307
+ status: 'ok',
308
+ version: DOPPLER_VERSION,
309
+ });
310
+ }
311
+
312
+ function handleCors(req, res) {
313
+ res.writeHead(204, {
314
+ 'Access-Control-Allow-Origin': '*',
315
+ 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
316
+ 'Access-Control-Allow-Headers': 'Content-Type, Authorization',
317
+ 'Access-Control-Max-Age': '86400',
318
+ });
319
+ res.end();
320
+ }
321
+
322
+ export function createServeHandler() {
323
+ return async (req, res) => {
324
+ if (req.method === 'OPTIONS') {
325
+ return handleCors(req, res);
326
+ }
327
+
328
+ const url = new URL(req.url, `http://${req.headers.host || 'localhost'}`);
329
+ const pathname = url.pathname.replace(/\/+$/, '') || '/';
330
+
331
+ try {
332
+ if (pathname === '/v1/chat/completions' && req.method === 'POST') {
333
+ return await handleChatCompletions(req, res);
334
+ }
335
+ if (pathname === '/v1/models' && req.method === 'GET') {
336
+ return await handleListModels(res);
337
+ }
338
+ if ((pathname === '/health' || pathname === '/') && req.method === 'GET') {
339
+ return handleHealth(res);
340
+ }
341
+ jsonError(res, 404, `Unknown endpoint: ${req.method} ${pathname}`, 'not_found');
342
+ } catch (error) {
343
+ const message = error?.message || String(error);
344
+ console.error(`[doppler-serve] ${req.method} ${pathname}: ${message}`);
345
+ if (!res.headersSent) {
346
+ jsonError(res, 500, message, 'server_error');
347
+ }
348
+ }
349
+ };
350
+ }
351
+
352
+ export async function startServer(settings) {
353
+ if (settings.model) {
354
+ console.error(`[doppler-serve] pre-loading model: ${settings.model}`);
355
+ await doppler.load(settings.model);
356
+ console.error(`[doppler-serve] model ready: ${settings.model}`);
357
+ }
358
+
359
+ const handler = createServeHandler();
360
+ const server = http.createServer(handler);
361
+
362
+ return new Promise((resolve, reject) => {
363
+ server.on('error', reject);
364
+ server.listen(settings.port, settings.host, () => {
365
+ const addr = server.address();
366
+ console.error(`[doppler-serve] listening on http://${addr.address}:${addr.port}`);
367
+ console.error(`[doppler-serve] OpenAI-compatible: POST http://${addr.address}:${addr.port}/v1/chat/completions`);
368
+ resolve(server);
369
+ });
370
+ });
371
+ }
372
+
373
+ export async function main(argv = process.argv.slice(2)) {
374
+ const settings = parseServeArgs(argv);
375
+ if (settings.help) {
376
+ console.log(usage());
377
+ return;
378
+ }
379
+ await startServer(settings);
380
+ }
381
+
382
+ function isMainModule(metaUrl) {
383
+ const entryPath = process.argv[1];
384
+ if (!entryPath) {
385
+ return false;
386
+ }
387
+ return path.resolve(fileURLToPath(metaUrl)) === path.resolve(entryPath);
388
+ }
389
+
390
+ if (isMainModule(import.meta.url)) {
391
+ main().catch((error) => {
392
+ console.error(`[doppler-serve] ${error?.message || String(error)}`);
393
+ process.exit(1);
394
+ });
395
+ }