doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,46 +0,0 @@
1
- export {
2
- // Version
3
- DOPPLER_PROVIDER_VERSION,
4
-
5
- // Types
6
- type TextModelConfig,
7
- type InferredAttentionParams,
8
- type ModelEstimate,
9
- type LoadProgressEvent,
10
- type GenerateOptions,
11
- type ChatMessage,
12
- type ChatResponse,
13
- type DopplerCapabilitiesType,
14
- type DopplerProviderInterface,
15
-
16
- // Capability flags
17
- DopplerCapabilities,
18
-
19
- // Model management
20
- initDoppler,
21
- loadModel,
22
- unloadModel,
23
- loadLoRAAdapter,
24
- activateLoRAFromTrainingOutput,
25
- unloadLoRAAdapter,
26
- getActiveLoRA,
27
- getAvailableModels,
28
- destroyDoppler,
29
- getPipeline,
30
- getCurrentModelId,
31
-
32
- // Generation
33
- generate,
34
- prefillKV,
35
- generateWithPrefixKV,
36
- formatGemmaChat,
37
- formatLlama3Chat,
38
- formatGptOssChat,
39
- formatChatMessages,
40
- buildChatPrompt,
41
- dopplerChat,
42
-
43
- // Main provider
44
- DopplerProvider,
45
- default,
46
- } from './doppler-provider.js';
@@ -1,36 +0,0 @@
1
- // Types and interfaces
2
- export {
3
- DOPPLER_PROVIDER_VERSION,
4
- DopplerCapabilities,
5
- } from './doppler-provider/types.js';
6
-
7
- // Model management
8
- export {
9
- initDoppler,
10
- loadModel,
11
- unloadModel,
12
- loadLoRAAdapter,
13
- activateLoRAFromTrainingOutput,
14
- unloadLoRAAdapter,
15
- getActiveLoRA,
16
- getAvailableModels,
17
- destroyDoppler,
18
- getPipeline,
19
- getCurrentModelId,
20
- } from './doppler-provider/model-manager.js';
21
-
22
- // Generation
23
- export {
24
- generate,
25
- prefillKV,
26
- generateWithPrefixKV,
27
- formatGemmaChat,
28
- formatLlama3Chat,
29
- formatGptOssChat,
30
- formatChatMessages,
31
- buildChatPrompt,
32
- dopplerChat,
33
- } from './doppler-provider/generation.js';
34
-
35
- // Main provider
36
- export { DopplerProvider, default } from './doppler-provider/provider.js';
@@ -1,276 +0,0 @@
1
- /**
2
- * Inference Defaults Config Schema
3
- *
4
- * Default values for inference pipeline: batching, sampling, generation.
5
- * These defaults are used when no model-specific or user overrides are provided.
6
- *
7
- * Note: SamplingDefaultsSchema provides defaults for fields from SamplingSchema
8
- * (in inference.schema.js), plus greedyThreshold and repetitionPenaltyWindow
9
- * which are unique to defaults.
10
- *
11
- * @module config/schema/inference-defaults
12
- */
13
-
14
- import type {
15
- ChatTemplateSchema,
16
- InferenceConfigSchema,
17
- LayerPipelineSchema,
18
- SamplingSchema,
19
- TokenizerConfigSchema,
20
- } from './inference.schema.js';
21
- import type {
22
- ExecutionV1PatchSchema,
23
- ExecutionV1SessionDefaultsSchema,
24
- } from './execution-v1.schema.js';
25
- import type { KVCacheConfigSchema } from './kvcache.schema.js';
26
- import type { MoERuntimeConfigSchema } from './moe.schema.js';
27
- import type { KernelPathRef } from './kernel-path.schema.js';
28
- import type { ManifestInferenceSchema } from './manifest.schema.js';
29
- import type { SpeculativeConfigSchema } from './speculative.schema.js';
30
- import type { DiffusionConfigSchema } from './diffusion.schema.js';
31
- import type { EnergyConfigSchema } from './energy.schema.js';
32
-
33
- // Re-export for convenience
34
- export type { ManifestInferenceSchema };
35
-
36
- /**
37
- * Deep partial type for runtime overrides of model-specific inference config.
38
- * Allows overriding any nested field in ManifestInferenceSchema.
39
- */
40
- export type ModelInferenceOverrides = {
41
- [P in keyof ManifestInferenceSchema]?: ManifestInferenceSchema[P] extends object
42
- ? { [K in keyof ManifestInferenceSchema[P]]?: ManifestInferenceSchema[P][K] }
43
- : ManifestInferenceSchema[P];
44
- };
45
-
46
- /**
47
- * Default batching configuration for inference.
48
- *
49
- * Controls how tokens are batched during generation.
50
- */
51
- export interface BatchingDefaultsSchema {
52
- /** Number of sequences to process in parallel (default: 1) */
53
- batchSize: number;
54
-
55
- /** Maximum tokens to generate per sequence (default: 512) */
56
- maxTokens: number;
57
-
58
- /** When to check for stop conditions: per-token or per-batch (default: 'batch') */
59
- stopCheckMode: 'per-token' | 'batch';
60
-
61
- /**
62
- * Number of GPU decode batches to run before readback.
63
- * null means read back after every batch (default: 1).
64
- */
65
- readbackInterval: number | null;
66
-
67
- /** Ring size for token output buffers (null = disable ring) */
68
- ringTokens: number | null;
69
-
70
- /** Ring size for per-token stop buffers (null = disable ring) */
71
- ringStop: number | null;
72
-
73
- /** Ring size for staging readback buffers (null = disable ring) */
74
- ringStaging: number | null;
75
- }
76
-
77
- /** Default batching configuration */
78
- export declare const DEFAULT_BATCHING_DEFAULTS: BatchingDefaultsSchema;
79
-
80
- /**
81
- * Default compute precision configuration.
82
- *
83
- * Controls dtype for intermediate activations and compute operations.
84
- * F16 reduces memory bandwidth by 2x but may have precision implications.
85
- */
86
- export interface ComputeDefaultsSchema {
87
- /** Dtype for hidden state activations (default: 'f32', experimental: 'f16') */
88
- activationDtype: 'f16' | 'f32';
89
-
90
- /** Parameter count threshold for "large model" classification (default: 4e9 = 4B params) */
91
- largeModelParamThreshold: number;
92
-
93
- /** Multiplier for estimating model params from hidden^2 × layers (default: 12) */
94
- paramEstimationMultiplier: number;
95
-
96
- /** Keep weights in F32 (skip downcast even when F16 is available) */
97
- keepF32Weights: boolean;
98
-
99
- /**
100
- * Range-aware selective widening policy used by finiteness guard.
101
- * When enabled, decode checks for abs(value) > absThreshold and widens
102
- * dangerous tokens to F32 fallback before KV corruption.
103
- */
104
- rangeAwareSelectiveWidening: RangeAwareSelectiveWideningDefaultsSchema;
105
-
106
- /**
107
- * Number of decode tokens to keep in widened (F32) mode after an overflow
108
- * hit before restoring the original dtype path.
109
- */
110
- deferredRoundingWindowTokens: number;
111
- }
112
-
113
- export interface RangeAwareSelectiveWideningDefaultsSchema {
114
- /** Enable range-aware widening guard for F16 activations. */
115
- enabled: boolean;
116
- /** Absolute-value threshold used for early widening checks. */
117
- absThreshold: number;
118
- /** Whether non-finite values should always trigger widening. */
119
- includeNonFinite: boolean;
120
- }
121
-
122
- /** Default compute configuration */
123
- export declare const DEFAULT_COMPUTE_DEFAULTS: ComputeDefaultsSchema;
124
-
125
- /**
126
- * Configuration for oversized weights (embeddings, LM head).
127
- *
128
- * When weights exceed device binding limits, DOPPLER can keep them on CPU
129
- * and stream chunks to the GPU for matmul or gather operations.
130
- */
131
- export interface LargeWeightConfigSchema {
132
- /** Enable CPU-backed chunking for oversized weights */
133
- enabled: boolean;
134
- /** Safety ratio applied to GPU binding limits (0..1). Default: 0.9 */
135
- safetyRatio: number;
136
- /** Prefer uploading F16 chunks when supported (reduces chunk size) */
137
- preferF16: boolean;
138
- /** Optional override for LM head chunk rows (null = auto) */
139
- lmHeadChunkRows?: number | null;
140
- }
141
-
142
- /** Default large-weight configuration */
143
- export declare const DEFAULT_LARGE_WEIGHT_CONFIG: LargeWeightConfigSchema;
144
-
145
- /**
146
- * Default sampling configuration for token selection.
147
- *
148
- * Extends Required<SamplingSchema> with greedyThreshold and repetitionPenaltyWindow
149
- * for runtime decisions.
150
- * SamplingSchema (in inference.schema.js) uses optional fields for partial overrides;
151
- * this schema provides concrete defaults for all sampling parameters.
152
- */
153
- export interface SamplingDefaultsSchema extends Required<SamplingSchema> {
154
- /** Temperature below this uses greedy decoding (default: 0.01) */
155
- greedyThreshold: number;
156
-
157
- /** Number of recent tokens to consider for repetition penalty (default: 100) */
158
- repetitionPenaltyWindow: number;
159
- }
160
-
161
- /** Default sampling configuration */
162
- export declare const DEFAULT_SAMPLING_DEFAULTS: SamplingDefaultsSchema;
163
-
164
- /** Default generation behavior flags used by generator runtime paths. */
165
- export interface GenerationDefaultsSchema {
166
- useSpeculative: boolean;
167
- profile: boolean;
168
- benchmark: boolean;
169
- disableCommandBatching: boolean;
170
- disableMultiTokenDecode: boolean;
171
- embeddingMode: 'last' | 'mean';
172
- }
173
-
174
- /** Default generation behavior flags */
175
- export declare const DEFAULT_GENERATION_CONFIG: GenerationDefaultsSchema;
176
-
177
- /**
178
- * Default tokenizer configuration.
179
- *
180
- * Provides defaults for common tokenizer behavior. Actual token strings
181
- * come from the model's tokenizer config; these control runtime behavior.
182
- */
183
- export interface TokenizerDefaultsSchema {
184
- /** Add BOS token to input (default: true for most models) */
185
- addBosToken: boolean;
186
-
187
- /** Add EOS token to output (default: false, model decides) */
188
- addEosToken: boolean;
189
-
190
- /** Defer token validation until token metadata is fully loaded */
191
- deferSpecialTokens: boolean;
192
- }
193
-
194
- /** Default tokenizer configuration */
195
- export declare const DEFAULT_TOKENIZER_DEFAULTS: TokenizerDefaultsSchema;
196
-
197
- /**
198
- * Complete inference defaults configuration schema.
199
- *
200
- * Combines batching, sampling, compute, and tokenizer defaults for the inference pipeline.
201
- */
202
- export interface InferenceDefaultsConfigSchema {
203
- batching: BatchingDefaultsSchema;
204
- sampling: SamplingDefaultsSchema;
205
- compute: ComputeDefaultsSchema;
206
- tokenizer: TokenizerDefaultsSchema;
207
- /** Handling for oversized embeddings/LM head */
208
- largeWeights: LargeWeightConfigSchema;
209
- /** KV cache configuration */
210
- kvcache: KVCacheConfigSchema;
211
- /** MoE routing and cache configuration */
212
- moe: MoERuntimeConfigSchema;
213
- /** Speculative decoding configuration */
214
- speculative: SpeculativeConfigSchema;
215
- /** Generation behavior defaults */
216
- generation: GenerationDefaultsSchema;
217
- /** Diffusion pipeline configuration */
218
- diffusion: DiffusionConfigSchema;
219
- /** Energy pipeline configuration */
220
- energy: EnergyConfigSchema;
221
- /** Optional default prompt text for test harnesses */
222
- prompt?: string | null;
223
- /** Log token ids in test harness output */
224
- debugTokens?: boolean;
225
- pipeline?: LayerPipelineSchema | null;
226
- /** Session overrides for execution v1 (session-only runtime knobs). */
227
- session?: ExecutionV1SessionDefaultsSchema;
228
- /** Atomic execution graph patch for execution v1. */
229
- executionPatch?: ExecutionV1PatchSchema;
230
- /**
231
- * Kernel path for explicit kernel dispatch ordering.
232
- * Specifies exactly which kernels run, in what order, with what configs.
233
- * Can be a registered kernel-path ID (e.g., 'gemma2-q4k-fused-f32a') or inline KernelPathSchema.
234
- */
235
- kernelPath?: KernelPathRef;
236
- /**
237
- * Internal kernel-path source hint used by execution runtime patching.
238
- * Not intended for direct user configuration.
239
- */
240
- kernelPathSource?: 'runtime' | 'config' | 'model' | 'manifest' | 'none';
241
- /**
242
- * Kernel-path auto-selection policy.
243
- * locked: preserve configured kernelPath as-is.
244
- * capability-aware: allow rule-driven remap for selected sources.
245
- */
246
- kernelPathPolicy?: KernelPathPolicySchema;
247
- /**
248
- * Chat template override for runtime config.
249
- * When set, overrides the manifest chatTemplate.enabled setting.
250
- */
251
- chatTemplate?: ChatTemplateSchema;
252
- /**
253
- * Model-specific inference overrides.
254
- * Allows runtime override of any manifest inference field (attention, normalization, rope, etc.).
255
- * These are merged with manifest.inference via mergeConfig().
256
- */
257
- modelOverrides?: ModelInferenceOverrides;
258
- }
259
-
260
- export interface KernelPathPolicySchema {
261
- mode: 'locked' | 'capability-aware';
262
- sourceScope: Array<'model' | 'manifest' | 'config'>;
263
- /**
264
- * Backward-compatible alias for sourceScope.
265
- * Prefer sourceScope in new configs.
266
- */
267
- allowSources?: Array<'model' | 'manifest' | 'config'>;
268
- onIncompatible: 'error' | 'remap';
269
- }
270
-
271
- /** Default inference configuration */
272
- export declare const DEFAULT_INFERENCE_DEFAULTS_CONFIG: InferenceDefaultsConfigSchema;
273
- export declare const DEFAULT_KERNEL_PATH_POLICY: KernelPathPolicySchema;
274
-
275
- /** Default inference configuration for model-owned manifest fields */
276
- export declare const DEFAULT_MODEL_INFERENCE_DEFAULTS: InferenceConfigSchema;
@@ -1,189 +0,0 @@
1
- import { DEFAULT_KVCACHE_CONFIG } from './kvcache.schema.js';
2
- import { DEFAULT_MOE_RUNTIME_CONFIG } from './moe.schema.js';
3
- import { DEFAULT_SPECULATIVE_CONFIG } from './speculative.schema.js';
4
- import { DEFAULT_RMS_NORM_EPS } from './manifest.schema.js';
5
- import { DEFAULT_DIFFUSION_CONFIG } from './diffusion.schema.js';
6
- import { DEFAULT_ENERGY_CONFIG } from './energy.schema.js';
7
- import {
8
- DEFAULT_EXECUTION_V1_PATCH,
9
- DEFAULT_EXECUTION_V1_SESSION_DEFAULTS,
10
- } from './execution-v1.schema.js';
11
-
12
- // =============================================================================
13
- // Generation Defaults (generator.js runtime options)
14
- // =============================================================================
15
-
16
- export const DEFAULT_GENERATION_CONFIG = {
17
- useSpeculative: false,
18
- profile: false,
19
- benchmark: false,
20
- disableCommandBatching: false,
21
- disableMultiTokenDecode: false,
22
- embeddingMode: 'last',
23
- };
24
-
25
- // =============================================================================
26
- // Batching Defaults
27
- // =============================================================================
28
-
29
- export const DEFAULT_BATCHING_DEFAULTS = {
30
- batchSize: 4, // GPU run-ahead window: record N tokens, submit once, readback once
31
- maxTokens: 256,
32
- stopCheckMode: 'batch',
33
- readbackInterval: 1,
34
- ringTokens: 1,
35
- ringStop: 1,
36
- ringStaging: 1,
37
- };
38
-
39
- // =============================================================================
40
- // Compute Defaults
41
- // =============================================================================
42
-
43
- export const DEFAULT_COMPUTE_DEFAULTS = {
44
- activationDtype: 'f16', // Default to F16 for web inference; fallback to F32 when unsupported
45
- largeModelParamThreshold: 4e9, // 4B parameters
46
- paramEstimationMultiplier: 12, // Rough approximation: 12 * hidden^2 * layers
47
- keepF32Weights: false, // Skip weight downcast (debug/compat)
48
- rangeAwareSelectiveWidening: {
49
- enabled: true,
50
- absThreshold: 65500,
51
- includeNonFinite: true,
52
- },
53
- deferredRoundingWindowTokens: 1,
54
- };
55
-
56
- // =============================================================================
57
- // Large Weight Handling
58
- // =============================================================================
59
-
60
- export const DEFAULT_LARGE_WEIGHT_CONFIG = {
61
- enabled: true,
62
- safetyRatio: 0.9,
63
- preferF16: true,
64
- lmHeadChunkRows: null,
65
- };
66
-
67
- // =============================================================================
68
- // Sampling Defaults
69
- // =============================================================================
70
-
71
- export const DEFAULT_SAMPLING_DEFAULTS = {
72
- temperature: 1.0,
73
- topP: 0.95,
74
- topK: 50,
75
- repetitionPenalty: 1.1,
76
- greedyThreshold: 0.01,
77
- repetitionPenaltyWindow: 100,
78
- };
79
-
80
- // =============================================================================
81
- // Tokenizer Defaults
82
- // =============================================================================
83
-
84
- export const DEFAULT_TOKENIZER_DEFAULTS = {
85
- addBosToken: true,
86
- addEosToken: false,
87
- deferSpecialTokens: true,
88
- addSpacePrefix: null, // null = auto-detect from tokenizer.json
89
- };
90
-
91
- // =============================================================================
92
- // Chat Template Defaults
93
- // =============================================================================
94
-
95
- export const DEFAULT_CHAT_TEMPLATE_CONFIG = {
96
- enabled: undefined,
97
- };
98
-
99
- export const DEFAULT_KERNEL_PATH_POLICY = {
100
- mode: 'locked',
101
- sourceScope: ['model', 'manifest'],
102
- onIncompatible: 'error',
103
- };
104
-
105
- // =============================================================================
106
- // Complete Inference Defaults Config
107
- // =============================================================================
108
-
109
- export const DEFAULT_INFERENCE_DEFAULTS_CONFIG = {
110
- batching: DEFAULT_BATCHING_DEFAULTS,
111
- sampling: DEFAULT_SAMPLING_DEFAULTS,
112
- compute: DEFAULT_COMPUTE_DEFAULTS,
113
- tokenizer: DEFAULT_TOKENIZER_DEFAULTS,
114
- largeWeights: DEFAULT_LARGE_WEIGHT_CONFIG,
115
- kvcache: DEFAULT_KVCACHE_CONFIG,
116
- moe: DEFAULT_MOE_RUNTIME_CONFIG,
117
- speculative: DEFAULT_SPECULATIVE_CONFIG,
118
- generation: DEFAULT_GENERATION_CONFIG,
119
- chatTemplate: DEFAULT_CHAT_TEMPLATE_CONFIG,
120
- diffusion: DEFAULT_DIFFUSION_CONFIG,
121
- energy: DEFAULT_ENERGY_CONFIG,
122
- pipeline: null,
123
- kernelPath: undefined,
124
- kernelPathSource: undefined,
125
- kernelPathPolicy: DEFAULT_KERNEL_PATH_POLICY,
126
- session: DEFAULT_EXECUTION_V1_SESSION_DEFAULTS,
127
- executionPatch: DEFAULT_EXECUTION_V1_PATCH,
128
- };
129
-
130
- // =============================================================================
131
- // Model Inference Defaults
132
- // =============================================================================
133
-
134
- export const DEFAULT_MODEL_INFERENCE_DEFAULTS = {
135
- attention: {
136
- slidingWindow: null,
137
- attnLogitSoftcapping: null,
138
- queryKeyNorm: false,
139
- attentionOutputGate: false,
140
- causal: true,
141
- ropeScalingType: null,
142
- ropeScalingFactor: 1.0,
143
- },
144
- normalization: {
145
- rmsNormWeightOffset: false,
146
- rmsNormEps: DEFAULT_RMS_NORM_EPS,
147
- postAttentionNorm: false,
148
- preFeedforwardNorm: false,
149
- postFeedforwardNorm: false,
150
- },
151
- ffn: {
152
- activation: 'silu',
153
- gatedActivation: true,
154
- },
155
- output: {
156
- finalLogitSoftcapping: null,
157
- tieWordEmbeddings: false,
158
- scaleEmbeddings: false,
159
- embeddingTranspose: false,
160
- embeddingVocabSize: null,
161
- embeddingPostprocessor: null,
162
- },
163
- layerPattern: {
164
- type: 'all_attention',
165
- },
166
- rope: {
167
- ropeTheta: 10000,
168
- ropeLocalTheta: null,
169
- mropeInterleaved: false,
170
- mropeSection: null,
171
- partialRotaryFactor: null,
172
- ropeScalingType: null,
173
- ropeScalingFactor: 1.0,
174
- ropeLocalScalingType: null,
175
- ropeLocalScalingFactor: 1.0,
176
- yarnBetaFast: null,
177
- yarnBetaSlow: null,
178
- yarnOriginalMaxPos: null,
179
- ropeLocalYarnBetaFast: null,
180
- ropeLocalYarnBetaSlow: null,
181
- ropeLocalYarnOriginalMaxPos: null,
182
- },
183
- moe: null,
184
- pipeline: null,
185
- chatTemplate: {
186
- type: null,
187
- },
188
- kernelPath: undefined,
189
- };
@@ -1,8 +0,0 @@
1
- export { DiffusionPipeline, createDiffusionPipeline } from '../inference/pipelines/diffusion/pipeline.js';
2
- export { createDiffusionWeightLoader } from '../inference/pipelines/diffusion/weights.js';
3
- export { mergeDiffusionConfig, initializeDiffusion } from '../inference/pipelines/diffusion/init.js';
4
- export {
5
- computeImageFingerprint,
6
- computeImageRegressionMetrics,
7
- assertImageRegressionWithinTolerance,
8
- } from './image-regression.js';
@@ -1,8 +0,0 @@
1
- export { DiffusionPipeline, createDiffusionPipeline } from '../inference/pipelines/diffusion/pipeline.js';
2
- export { createDiffusionWeightLoader } from '../inference/pipelines/diffusion/weights.js';
3
- export { mergeDiffusionConfig, initializeDiffusion } from '../inference/pipelines/diffusion/init.js';
4
- export {
5
- computeImageFingerprint,
6
- computeImageRegressionMetrics,
7
- assertImageRegressionWithinTolerance,
8
- } from './image-regression.js';
@@ -1,2 +0,0 @@
1
- export { EnergyPipeline, createEnergyPipeline } from '../inference/pipelines/energy/pipeline.js';
2
- export { mergeQuintelConfig, computeQuintelEnergy, runQuintelEnergyLoop } from '../inference/pipelines/energy/quintel.js';
@@ -1,2 +0,0 @@
1
- export { EnergyPipeline, createEnergyPipeline } from '../inference/pipelines/energy/pipeline.js';
2
- export { mergeQuintelConfig, computeQuintelEnergy, runQuintelEnergyLoop } from '../inference/pipelines/energy/quintel.js';
@@ -1,2 +0,0 @@
1
- export * from './index.js';
2
- export * from './tooling-exports.js';
@@ -1,2 +0,0 @@
1
- export * from './index.js';
2
- export * from './tooling-exports.js';
File without changes
File without changes