doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -4,13 +4,22 @@ import http from 'node:http';
4
4
  import crypto from 'node:crypto';
5
5
  import path from 'node:path';
6
6
  import process from 'node:process';
7
- import { fileURLToPath } from 'node:url';
7
+ import { fileURLToPath, pathToFileURL } from 'node:url';
8
8
  import { DOPPLER_VERSION, doppler } from '../index.js';
9
- import { listQuickstartModels } from '../client/doppler-registry.js';
9
+ import { listQuickstartModels, resolveQuickstartModel } from '../client/doppler-registry.js';
10
10
 
11
11
  const DEFAULT_PORT = 8080;
12
12
  const DEFAULT_HOST = '127.0.0.1';
13
13
 
14
+ export class ServeRequestError extends Error {
15
+ constructor(message, statusCode = 400, type = 'invalid_request_error') {
16
+ super(message);
17
+ this.name = 'ServeRequestError';
18
+ this.statusCode = statusCode;
19
+ this.type = type;
20
+ }
21
+ }
22
+
14
23
  function usage() {
15
24
  return [
16
25
  'Usage:',
@@ -18,6 +27,7 @@ function usage() {
18
27
  '',
19
28
  'Options:',
20
29
  ' --model <id> Pre-load a model at startup (optional, lazy-loads on request otherwise)',
30
+ ' --model-url <url> Explicit local or remote RDRR artifact URL for --model',
21
31
  ' --port <n> Port to listen on (default: 8080)',
22
32
  ' --host <addr> Host to bind to (default: 127.0.0.1)',
23
33
  ' --help Show this help',
@@ -29,7 +39,7 @@ function usage() {
29
39
  '',
30
40
  'Examples:',
31
41
  ' node src/cli/doppler-serve.js --model gemma3-270m',
32
- ' node src/cli/doppler-serve.js --model qwen3-0.8b --port 3000',
42
+ ' node src/cli/doppler-serve.js --model gemma4-e2b --port 3000',
33
43
  '',
34
44
  'Then use with any OpenAI-compatible client:',
35
45
  ' curl http://localhost:8080/v1/chat/completions \\',
@@ -43,6 +53,7 @@ export function parseServeArgs(argv) {
43
53
  port: DEFAULT_PORT,
44
54
  host: DEFAULT_HOST,
45
55
  model: null,
56
+ modelUrl: null,
46
57
  help: false,
47
58
  };
48
59
  for (let i = 0; i < argv.length; i += 1) {
@@ -54,7 +65,7 @@ export function parseServeArgs(argv) {
54
65
  if (!token.startsWith('--')) {
55
66
  throw new Error(`Unexpected positional argument: ${token}`);
56
67
  }
57
- if (token !== '--port' && token !== '--host' && token !== '--model') {
68
+ if (token !== '--port' && token !== '--host' && token !== '--model' && token !== '--model-url') {
58
69
  throw new Error(`Unknown flag ${token}.`);
59
70
  }
60
71
  const nextValue = argv[i + 1];
@@ -80,24 +91,81 @@ export function parseServeArgs(argv) {
80
91
  i += 1;
81
92
  continue;
82
93
  }
94
+ if (token === '--model-url') {
95
+ flags.modelUrl = normalizeModelUrl(nextValue.trim());
96
+ i += 1;
97
+ continue;
98
+ }
83
99
  throw new Error(`Unknown flag ${token}.`);
84
100
  }
101
+ if (flags.modelUrl && !flags.model) {
102
+ throw new Error('--model-url requires --model so the served model identity remains explicit.');
103
+ }
85
104
  return flags;
86
105
  }
87
106
 
107
+ function normalizeModelUrl(value) {
108
+ if (!value) {
109
+ throw new Error('--model-url must be non-empty.');
110
+ }
111
+ if (/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value)) {
112
+ return value;
113
+ }
114
+ return pathToFileURL(path.resolve(value)).href;
115
+ }
116
+
88
117
  function generateCompletionId() {
89
118
  return `chatcmpl-${crypto.randomBytes(12).toString('base64url')}`;
90
119
  }
91
120
 
92
- function jsonError(res, statusCode, message, type) {
93
- const body = JSON.stringify({
121
+ function sha256Hex(value) {
122
+ return crypto.createHash('sha256').update(value).digest('hex');
123
+ }
124
+
125
+ function utf8ByteLength(value) {
126
+ return Buffer.byteLength(String(value ?? ''), 'utf8');
127
+ }
128
+
129
+ function stableJson(value) {
130
+ if (value === null || typeof value !== 'object') {
131
+ return JSON.stringify(value);
132
+ }
133
+ if (Array.isArray(value)) {
134
+ return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
135
+ }
136
+ const keys = Object.keys(value).sort();
137
+ const entries = keys
138
+ .filter((key) => value[key] !== undefined)
139
+ .map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`);
140
+ return `{${entries.join(',')}}`;
141
+ }
142
+
143
+ function buildSha256Evidence(value) {
144
+ const text = String(value ?? '');
145
+ return {
146
+ algorithm: 'sha256',
147
+ value: sha256Hex(text),
148
+ bytes: utf8ByteLength(text),
149
+ };
150
+ }
151
+
152
+ function buildJsonSha256Evidence(value) {
153
+ return buildSha256Evidence(stableJson(value));
154
+ }
155
+
156
+ function jsonError(res, statusCode, message, type, extra = null) {
157
+ const payload = {
94
158
  error: {
95
159
  message,
96
160
  type: type ?? 'invalid_request_error',
97
161
  param: null,
98
162
  code: null,
99
163
  },
100
- });
164
+ };
165
+ if (extra && typeof extra === 'object' && !Array.isArray(extra)) {
166
+ Object.assign(payload, extra);
167
+ }
168
+ const body = JSON.stringify(payload);
101
169
  res.writeHead(statusCode, {
102
170
  'Content-Type': 'application/json',
103
171
  'Access-Control-Allow-Origin': '*',
@@ -114,6 +182,10 @@ function jsonResponse(res, statusCode, data) {
114
182
  res.end(body);
115
183
  }
116
184
 
185
+ function requestError(message, statusCode = 400, type = 'invalid_request_error') {
186
+ return new ServeRequestError(message, statusCode, type);
187
+ }
188
+
117
189
  async function readRequestBody(req) {
118
190
  const chunks = [];
119
191
  for await (const chunk of req) {
@@ -121,29 +193,29 @@ async function readRequestBody(req) {
121
193
  }
122
194
  const raw = Buffer.concat(chunks).toString('utf8');
123
195
  if (!raw || raw.trim().length === 0) {
124
- throw new Error('Request body is empty.');
196
+ throw requestError('Request body is empty.');
125
197
  }
126
198
  try {
127
199
  return JSON.parse(raw);
128
200
  } catch {
129
- throw new Error('Request body is not valid JSON.');
201
+ throw requestError('Request body is not valid JSON.');
130
202
  }
131
203
  }
132
204
 
133
205
  function validateMessages(messages) {
134
206
  if (!Array.isArray(messages) || messages.length === 0) {
135
- throw new Error('"messages" must be a non-empty array.');
207
+ throw requestError('"messages" must be a non-empty array.');
136
208
  }
137
209
  for (let i = 0; i < messages.length; i += 1) {
138
210
  const msg = messages[i];
139
211
  if (!msg || typeof msg !== 'object') {
140
- throw new Error(`messages[${i}] must be an object.`);
212
+ throw requestError(`messages[${i}] must be an object.`);
141
213
  }
142
214
  if (typeof msg.role !== 'string' || msg.role.trim().length === 0) {
143
- throw new Error(`messages[${i}].role must be a non-empty string.`);
215
+ throw requestError(`messages[${i}].role must be a non-empty string.`);
144
216
  }
145
217
  if (typeof msg.content !== 'string') {
146
- throw new Error(`messages[${i}].content must be a string.`);
218
+ throw requestError(`messages[${i}].content must be a string.`);
147
219
  }
148
220
  }
149
221
  return messages.map((msg) => ({ role: msg.role.trim(), content: msg.content }));
@@ -153,36 +225,279 @@ function extractGenerationOptions(body) {
153
225
  const options = {};
154
226
  if (body.max_tokens != null) {
155
227
  const n = Number(body.max_tokens);
156
- if (!Number.isFinite(n) || n < 1) {
157
- throw new Error('"max_tokens" must be a positive integer.');
228
+ if (!Number.isFinite(n) || n < 1 || Math.floor(n) !== n) {
229
+ throw requestError('"max_tokens" must be a positive integer.');
158
230
  }
159
- options.maxTokens = Math.floor(n);
231
+ options.maxTokens = n;
160
232
  }
161
233
  if (body.temperature != null) {
162
234
  const t = Number(body.temperature);
163
235
  if (!Number.isFinite(t) || t < 0) {
164
- throw new Error('"temperature" must be a non-negative number.');
236
+ throw requestError('"temperature" must be a non-negative number.');
165
237
  }
166
238
  options.temperature = t;
167
239
  }
168
240
  if (body.top_p != null) {
169
241
  const p = Number(body.top_p);
170
242
  if (!Number.isFinite(p) || p <= 0 || p > 1) {
171
- throw new Error('"top_p" must be a number between 0 (exclusive) and 1 (inclusive).');
243
+ throw requestError('"top_p" must be a number between 0 (exclusive) and 1 (inclusive).');
172
244
  }
173
245
  options.topP = p;
174
246
  }
175
247
  if (body.top_k != null) {
176
248
  const k = Number(body.top_k);
177
- if (!Number.isFinite(k) || k < 1) {
178
- throw new Error('"top_k" must be a positive integer.');
249
+ if (!Number.isFinite(k) || k < 1 || Math.floor(k) !== k) {
250
+ throw requestError('"top_k" must be a positive integer.');
179
251
  }
180
- options.topK = Math.floor(k);
252
+ options.topK = k;
181
253
  }
182
254
  return options;
183
255
  }
184
256
 
185
- async function handleChatCompletions(req, res) {
257
+ function resolveIncludeReceipt(body) {
258
+ const includeReceipt = body.include_receipt;
259
+ const dopplerReceipt = body.doppler_receipt;
260
+ if (includeReceipt == null && dopplerReceipt == null) {
261
+ return false;
262
+ }
263
+ if (includeReceipt != null && typeof includeReceipt !== 'boolean') {
264
+ throw requestError('"include_receipt" must be a boolean when provided.');
265
+ }
266
+ if (dopplerReceipt != null && typeof dopplerReceipt !== 'boolean') {
267
+ throw requestError('"doppler_receipt" must be a boolean when provided.');
268
+ }
269
+ if (includeReceipt != null && dopplerReceipt != null && includeReceipt !== dopplerReceipt) {
270
+ throw requestError('"include_receipt" and "doppler_receipt" must agree when both are provided.');
271
+ }
272
+ return includeReceipt ?? dopplerReceipt;
273
+ }
274
+
275
+ async function resolveTextModelEntry(modelId, resolveModel) {
276
+ let entry;
277
+ try {
278
+ entry = await resolveModel(modelId);
279
+ } catch {
280
+ throw requestError(`Unknown model "${modelId}". Use GET /v1/models for supported chat models.`);
281
+ }
282
+ if (!entry.modes.includes('text')) {
283
+ throw requestError(`Model "${modelId}" is not text-generative. Use GET /v1/models for supported chat models.`);
284
+ }
285
+ return entry;
286
+ }
287
+
288
+ function buildGenerationOptionsReceipt(generationOptions) {
289
+ return {
290
+ maxTokens: generationOptions.maxTokens ?? null,
291
+ temperature: generationOptions.temperature ?? null,
292
+ topP: generationOptions.topP ?? null,
293
+ topK: generationOptions.topK ?? null,
294
+ };
295
+ }
296
+
297
+ function normalizeRuntimeModelSource(runtimeModel, registryEntry) {
298
+ if (typeof runtimeModel === 'string') {
299
+ return {
300
+ kind: 'quickstart-registry',
301
+ modelId: runtimeModel,
302
+ };
303
+ }
304
+ if (runtimeModel && typeof runtimeModel === 'object' && typeof runtimeModel.url === 'string') {
305
+ return {
306
+ kind: 'url',
307
+ url: runtimeModel.url,
308
+ };
309
+ }
310
+ if (runtimeModel && typeof runtimeModel === 'object' && runtimeModel.manifest && typeof runtimeModel.manifest === 'object') {
311
+ return {
312
+ kind: 'inline-manifest',
313
+ modelId: typeof runtimeModel.manifest.modelId === 'string'
314
+ ? runtimeModel.manifest.modelId
315
+ : registryEntry.modelId,
316
+ baseUrl: typeof runtimeModel.baseUrl === 'string' ? runtimeModel.baseUrl : null,
317
+ };
318
+ }
319
+ return {
320
+ kind: 'quickstart-registry',
321
+ modelId: registryEntry.modelId,
322
+ };
323
+ }
324
+
325
+ function buildServeReceiptBase({ requestedModel, registryEntry, messages, generationOptions, runtimeModel = null }) {
326
+ const generation = buildGenerationOptionsReceipt(generationOptions);
327
+ return {
328
+ receiptVersion: 'doppler_serve_receipt_v1',
329
+ schemaVersion: 1,
330
+ surface: 'serve',
331
+ endpoint: '/v1/chat/completions',
332
+ runtime: 'doppler-gpu',
333
+ runtimeVersion: DOPPLER_VERSION,
334
+ runtimePath: 'doppler-gpu.chatText',
335
+ runtimeModelSource: normalizeRuntimeModelSource(runtimeModel ?? registryEntry.modelId, registryEntry),
336
+ modelId: registryEntry.modelId,
337
+ requestedModel,
338
+ resolvedModel: registryEntry.modelId,
339
+ artifact: {
340
+ format: 'rdrr',
341
+ source: 'quickstart-registry',
342
+ sourceCheckpointId: registryEntry.sourceCheckpointId,
343
+ weightPackId: registryEntry.weightPackId,
344
+ manifestVariantId: registryEntry.manifestVariantId,
345
+ artifactCompleteness: registryEntry.artifactCompleteness,
346
+ runtimePromotionState: registryEntry.runtimePromotionState,
347
+ weightsRefAllowed: registryEntry.weightsRefAllowed,
348
+ hf: registryEntry.hf,
349
+ },
350
+ request: {
351
+ messages: {
352
+ count: messages.length,
353
+ digest: buildJsonSha256Evidence(messages),
354
+ },
355
+ generationDigest: buildJsonSha256Evidence(generation),
356
+ },
357
+ generation,
358
+ };
359
+ }
360
+
361
+ export function buildServeReceipt({
362
+ requestedModel,
363
+ registryEntry,
364
+ messages,
365
+ generationOptions,
366
+ outputContent,
367
+ usage,
368
+ runtimeModel = null,
369
+ }) {
370
+ const baseReceipt = buildServeReceiptBase({
371
+ requestedModel,
372
+ registryEntry,
373
+ messages,
374
+ generationOptions,
375
+ runtimeModel,
376
+ });
377
+ const outputText = String(outputContent ?? '');
378
+ return {
379
+ ...baseReceipt,
380
+ status: 'pass',
381
+ output: {
382
+ role: 'assistant',
383
+ digest: buildSha256Evidence(outputText),
384
+ textLength: outputText.length,
385
+ empty: outputText.length === 0,
386
+ },
387
+ transcript: {
388
+ digest: buildJsonSha256Evidence({
389
+ messages,
390
+ generation: baseReceipt.generation,
391
+ output: outputText,
392
+ usage,
393
+ }),
394
+ },
395
+ usage,
396
+ };
397
+ }
398
+
399
+ function normalizeWeightLoadFailure(error) {
400
+ const failure = error?.details?.weightLoadFailure ?? error?.cause?.details?.weightLoadFailure;
401
+ if (!failure || typeof failure !== 'object') {
402
+ return null;
403
+ }
404
+ const deviceLimitFailure = failure.deviceLimitFailure && typeof failure.deviceLimitFailure === 'object'
405
+ ? {
406
+ kind: typeof failure.deviceLimitFailure.kind === 'string' ? failure.deviceLimitFailure.kind : null,
407
+ maxGpuResidentBytes: Number.isFinite(failure.deviceLimitFailure.maxGpuResidentBytes)
408
+ ? failure.deviceLimitFailure.maxGpuResidentBytes
409
+ : null,
410
+ maxStorageBufferBindingSize: Number.isFinite(failure.deviceLimitFailure.maxStorageBufferBindingSize)
411
+ ? failure.deviceLimitFailure.maxStorageBufferBindingSize
412
+ : null,
413
+ maxBufferSize: Number.isFinite(failure.deviceLimitFailure.maxBufferSize)
414
+ ? failure.deviceLimitFailure.maxBufferSize
415
+ : null,
416
+ maxStorageBuffersPerShaderStage: Number.isFinite(failure.deviceLimitFailure.maxStorageBuffersPerShaderStage)
417
+ ? failure.deviceLimitFailure.maxStorageBuffersPerShaderStage
418
+ : null,
419
+ largeWeightMaxBytes: Number.isFinite(failure.deviceLimitFailure.largeWeightMaxBytes)
420
+ ? failure.deviceLimitFailure.largeWeightMaxBytes
421
+ : null,
422
+ embeddingKernel: failure.deviceLimitFailure.embeddingKernel
423
+ && typeof failure.deviceLimitFailure.embeddingKernel === 'object'
424
+ ? {
425
+ kernel: typeof failure.deviceLimitFailure.embeddingKernel.kernel === 'string'
426
+ ? failure.deviceLimitFailure.embeddingKernel.kernel
427
+ : null,
428
+ entry: typeof failure.deviceLimitFailure.embeddingKernel.entry === 'string'
429
+ ? failure.deviceLimitFailure.embeddingKernel.entry
430
+ : null,
431
+ }
432
+ : null,
433
+ splitKernelExpected: typeof failure.deviceLimitFailure.splitKernelExpected === 'boolean'
434
+ ? failure.deviceLimitFailure.splitKernelExpected
435
+ : null,
436
+ activeSplitKernelMaxSections: Number.isFinite(failure.deviceLimitFailure.activeSplitKernelMaxSections)
437
+ ? failure.deviceLimitFailure.activeSplitKernelMaxSections
438
+ : null,
439
+ maxSplitEmbeddingSections: Number.isFinite(failure.deviceLimitFailure.maxSplitEmbeddingSections)
440
+ ? failure.deviceLimitFailure.maxSplitEmbeddingSections
441
+ : null,
442
+ requiredSplitSections: Number.isFinite(failure.deviceLimitFailure.requiredSplitSections)
443
+ ? failure.deviceLimitFailure.requiredSplitSections
444
+ : null,
445
+ }
446
+ : null;
447
+ return {
448
+ tensorName: typeof failure.tensorName === 'string' ? failure.tensorName : null,
449
+ tensorRole: typeof failure.tensorRole === 'string' ? failure.tensorRole : null,
450
+ tensorDtype: typeof failure.tensorDtype === 'string' ? failure.tensorDtype : null,
451
+ tensorShape: Array.isArray(failure.tensorShape) ? [...failure.tensorShape] : null,
452
+ tensorSizeBytes: Number.isFinite(failure.tensorSizeBytes) ? failure.tensorSizeBytes : null,
453
+ tensorLoadStage: typeof failure.tensorLoadStage === 'string' ? failure.tensorLoadStage : null,
454
+ toGPU: typeof failure.toGPU === 'boolean' ? failure.toGPU : null,
455
+ streamedUpload: typeof failure.streamedUpload === 'boolean' ? failure.streamedUpload : null,
456
+ deviceLimitFailure,
457
+ };
458
+ }
459
+
460
+ function normalizeServeFailure(error, registryEntry) {
461
+ const message = error?.message || String(error);
462
+ const pipelineLoadPhase = typeof error?.details?.pipelineLoadPhase === 'string'
463
+ ? error.details.pipelineLoadPhase
464
+ : null;
465
+ return {
466
+ code: pipelineLoadPhase ? 'pipeline-load-failed' : 'runtime-error',
467
+ stage: pipelineLoadPhase ?? 'runtime',
468
+ message,
469
+ modelId: typeof error?.details?.modelId === 'string' ? error.details.modelId : registryEntry.modelId,
470
+ weightLoadFailure: normalizeWeightLoadFailure(error),
471
+ };
472
+ }
473
+
474
+ export function buildServeFailureReceipt({
475
+ requestedModel,
476
+ registryEntry,
477
+ messages,
478
+ generationOptions,
479
+ error,
480
+ runtimeModel = null,
481
+ }) {
482
+ return {
483
+ ...buildServeReceiptBase({ requestedModel, registryEntry, messages, generationOptions, runtimeModel }),
484
+ status: 'diagnostic',
485
+ failure: normalizeServeFailure(error, registryEntry),
486
+ };
487
+ }
488
+
489
+ function resolveServeDependencies(dependencies = {}) {
490
+ return {
491
+ dopplerClient: dependencies.dopplerClient ?? doppler,
492
+ listModels: dependencies.listModels ?? listQuickstartModels,
493
+ resolveModel: dependencies.resolveModel ?? resolveQuickstartModel,
494
+ resolveRuntimeModel: typeof dependencies.resolveRuntimeModel === 'function'
495
+ ? dependencies.resolveRuntimeModel
496
+ : (registryEntry) => registryEntry.modelId,
497
+ };
498
+ }
499
+
500
+ async function handleChatCompletions(req, res, dependencies) {
186
501
  const body = await readRequestBody(req);
187
502
  if (typeof body.model !== 'string' || body.model.trim().length === 0) {
188
503
  return jsonError(res, 400, '"model" is required and must be a non-empty string.');
@@ -190,23 +505,81 @@ async function handleChatCompletions(req, res) {
190
505
  const modelId = body.model.trim();
191
506
  const messages = validateMessages(body.messages);
192
507
  const generationOptions = extractGenerationOptions(body);
508
+ const includeReceipt = resolveIncludeReceipt(body);
193
509
  const stream = body.stream === true;
510
+ if (includeReceipt && stream) {
511
+ throw requestError('"include_receipt" is not supported with streaming responses.');
512
+ }
513
+ const registryEntry = await resolveTextModelEntry(modelId, dependencies.resolveModel);
194
514
  const completionId = generateCompletionId();
195
515
  const created = Math.floor(Date.now() / 1000);
196
516
 
197
517
  if (stream) {
198
- return handleStreamingCompletion(res, modelId, messages, generationOptions, completionId, created);
518
+ return handleStreamingCompletion(
519
+ res,
520
+ modelId,
521
+ registryEntry,
522
+ messages,
523
+ generationOptions,
524
+ completionId,
525
+ created,
526
+ dependencies
527
+ );
199
528
  }
200
- return handleNonStreamingCompletion(res, modelId, messages, generationOptions, completionId, created);
529
+ return handleNonStreamingCompletion(
530
+ res,
531
+ modelId,
532
+ registryEntry,
533
+ messages,
534
+ generationOptions,
535
+ completionId,
536
+ created,
537
+ includeReceipt,
538
+ dependencies
539
+ );
201
540
  }
202
541
 
203
- async function handleNonStreamingCompletion(res, modelId, messages, generationOptions, completionId, created) {
204
- const result = await doppler.chatText(messages, { model: modelId, ...generationOptions });
205
- jsonResponse(res, 200, {
542
+ async function handleNonStreamingCompletion(
543
+ res,
544
+ requestedModel,
545
+ registryEntry,
546
+ messages,
547
+ generationOptions,
548
+ completionId,
549
+ created,
550
+ includeReceipt,
551
+ dependencies
552
+ ) {
553
+ let result;
554
+ const runtimeModel = dependencies.resolveRuntimeModel(registryEntry, requestedModel);
555
+ try {
556
+ result = await dependencies.dopplerClient.chatText(messages, { model: runtimeModel, ...generationOptions });
557
+ } catch (error) {
558
+ if (includeReceipt) {
559
+ return jsonError(
560
+ res,
561
+ 500,
562
+ error?.message || String(error),
563
+ 'server_error',
564
+ {
565
+ doppler_receipt: buildServeFailureReceipt({
566
+ requestedModel,
567
+ registryEntry,
568
+ messages,
569
+ generationOptions,
570
+ error,
571
+ runtimeModel,
572
+ }),
573
+ }
574
+ );
575
+ }
576
+ throw error;
577
+ }
578
+ const body = {
206
579
  id: completionId,
207
580
  object: 'chat.completion',
208
581
  created,
209
- model: modelId,
582
+ model: registryEntry.modelId,
210
583
  choices: [
211
584
  {
212
585
  index: 0,
@@ -222,10 +595,32 @@ async function handleNonStreamingCompletion(res, modelId, messages, generationOp
222
595
  completion_tokens: result.usage.completionTokens,
223
596
  total_tokens: result.usage.totalTokens,
224
597
  },
225
- });
598
+ };
599
+ if (includeReceipt) {
600
+ body.doppler_receipt = buildServeReceipt({
601
+ requestedModel,
602
+ registryEntry,
603
+ messages,
604
+ generationOptions,
605
+ outputContent: result.content,
606
+ usage: result.usage,
607
+ runtimeModel,
608
+ });
609
+ }
610
+ jsonResponse(res, 200, body);
226
611
  }
227
612
 
228
- async function handleStreamingCompletion(res, modelId, messages, generationOptions, completionId, created) {
613
+ async function handleStreamingCompletion(
614
+ res,
615
+ requestedModel,
616
+ registryEntry,
617
+ messages,
618
+ generationOptions,
619
+ completionId,
620
+ created,
621
+ dependencies
622
+ ) {
623
+ const runtimeModel = dependencies.resolveRuntimeModel(registryEntry, requestedModel);
229
624
  res.writeHead(200, {
230
625
  'Content-Type': 'text/event-stream',
231
626
  'Cache-Control': 'no-cache',
@@ -241,7 +636,7 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
241
636
  id: completionId,
242
637
  object: 'chat.completion.chunk',
243
638
  created,
244
- model: modelId,
639
+ model: registryEntry.modelId,
245
640
  choices: [
246
641
  {
247
642
  index: 0,
@@ -251,7 +646,7 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
251
646
  ],
252
647
  });
253
648
 
254
- const stream = doppler.chat(messages, { model: modelId, ...generationOptions });
649
+ const stream = dependencies.dopplerClient.chat(messages, { model: runtimeModel, ...generationOptions });
255
650
  for await (const token of stream) {
256
651
  if (res.destroyed) {
257
652
  break;
@@ -260,7 +655,7 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
260
655
  id: completionId,
261
656
  object: 'chat.completion.chunk',
262
657
  created,
263
- model: modelId,
658
+ model: registryEntry.modelId,
264
659
  choices: [
265
660
  {
266
661
  index: 0,
@@ -275,7 +670,7 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
275
670
  id: completionId,
276
671
  object: 'chat.completion.chunk',
277
672
  created,
278
- model: modelId,
673
+ model: registryEntry.modelId,
279
674
  choices: [
280
675
  {
281
676
  index: 0,
@@ -288,8 +683,8 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
288
683
  res.end();
289
684
  }
290
685
 
291
- async function handleListModels(res) {
292
- const models = await listQuickstartModels();
686
+ async function handleListModels(res, dependencies) {
687
+ const models = await dependencies.listModels();
293
688
  const textModels = models.filter((entry) => entry.modes.includes('text'));
294
689
  jsonResponse(res, 200, {
295
690
  object: 'list',
@@ -298,6 +693,15 @@ async function handleListModels(res) {
298
693
  object: 'model',
299
694
  created: 0,
300
695
  owned_by: 'doppler',
696
+ doppler: {
697
+ sourceCheckpointId: entry.sourceCheckpointId,
698
+ weightPackId: entry.weightPackId,
699
+ manifestVariantId: entry.manifestVariantId,
700
+ artifactCompleteness: entry.artifactCompleteness,
701
+ runtimePromotionState: entry.runtimePromotionState,
702
+ weightsRefAllowed: entry.weightsRefAllowed,
703
+ modes: entry.modes,
704
+ },
301
705
  })),
302
706
  });
303
707
  }
@@ -319,7 +723,8 @@ function handleCors(req, res) {
319
723
  res.end();
320
724
  }
321
725
 
322
- export function createServeHandler() {
726
+ export function createServeHandler(dependencies = {}) {
727
+ const resolvedDependencies = resolveServeDependencies(dependencies);
323
728
  return async (req, res) => {
324
729
  if (req.method === 'OPTIONS') {
325
730
  return handleCors(req, res);
@@ -330,10 +735,10 @@ export function createServeHandler() {
330
735
 
331
736
  try {
332
737
  if (pathname === '/v1/chat/completions' && req.method === 'POST') {
333
- return await handleChatCompletions(req, res);
738
+ return await handleChatCompletions(req, res, resolvedDependencies);
334
739
  }
335
740
  if (pathname === '/v1/models' && req.method === 'GET') {
336
- return await handleListModels(res);
741
+ return await handleListModels(res, resolvedDependencies);
337
742
  }
338
743
  if ((pathname === '/health' || pathname === '/') && req.method === 'GET') {
339
744
  return handleHealth(res);
@@ -343,20 +748,41 @@ export function createServeHandler() {
343
748
  const message = error?.message || String(error);
344
749
  console.error(`[doppler-serve] ${req.method} ${pathname}: ${message}`);
345
750
  if (!res.headersSent) {
346
- jsonError(res, 500, message, 'server_error');
751
+ if (error instanceof ServeRequestError) {
752
+ jsonError(res, error.statusCode, message, error.type);
753
+ } else {
754
+ jsonError(res, 500, message, 'server_error');
755
+ }
347
756
  }
348
757
  }
349
758
  };
350
759
  }
351
760
 
761
+ function createRuntimeModelResolver(localModelSourceByModelId = new Map()) {
762
+ return (registryEntry) => localModelSourceByModelId.get(registryEntry.modelId) ?? registryEntry.modelId;
763
+ }
764
+
765
+ async function resolveLocalModelSourceMap(settings) {
766
+ const localModelSourceByModelId = new Map();
767
+ if (!settings.modelUrl) {
768
+ return localModelSourceByModelId;
769
+ }
770
+ const registryEntry = await resolveQuickstartModel(settings.model);
771
+ localModelSourceByModelId.set(registryEntry.modelId, { url: settings.modelUrl });
772
+ return localModelSourceByModelId;
773
+ }
774
+
352
775
  async function startServer(settings) {
776
+ const localModelSourceByModelId = await resolveLocalModelSourceMap(settings);
777
+ const resolveRuntimeModel = createRuntimeModelResolver(localModelSourceByModelId);
353
778
  if (settings.model) {
354
779
  console.error(`[doppler-serve] pre-loading model: ${settings.model}`);
355
- await doppler.load(settings.model);
780
+ const registryEntry = await resolveQuickstartModel(settings.model);
781
+ await doppler.load(resolveRuntimeModel(registryEntry));
356
782
  console.error(`[doppler-serve] model ready: ${settings.model}`);
357
783
  }
358
784
 
359
- const handler = createServeHandler();
785
+ const handler = createServeHandler({ resolveRuntimeModel });
360
786
  const server = http.createServer(handler);
361
787
 
362
788
  return new Promise((resolve, reject) => {